<commit_before>// Copyright (c) 2015 Jan Broer. All rights reserved.
// Use of this source code is governed by The MIT License (MIT) that can be
// found in the LICENSE file.

package main // import "github.com/janeczku/go-dnsmasq"

import (
	"fmt"
	"log/syslog"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"

	log "github.com/Sirupsen/logrus"
	logrus_syslog "github.com/Sirupsen/logrus/hooks/syslog"
	"github.com/codegangsta/cli"
	"github.com/miekg/dns"

	"github.com/janeczku/go-dnsmasq/hostsfile"
	"github.com/janeczku/go-dnsmasq/resolvconf"
	"github.com/janeczku/go-dnsmasq/server"
	"github.com/janeczku/go-dnsmasq/stats"
)

// var Version string
const Version = "0.9.5"

var (
	nameservers   = []string{}
	searchDomains = []string{}
	stubzones     = ""
	hostPort      = ""
	listen        = ""
)

var exitErr error

func init() {
	log.SetOutput(os.Stderr)
}

func main() {
	app := cli.NewApp()
	app.Name = "go-dnsmasq"
	app.Usage = "Lightweight caching DNS proxy for Docker containers"
	app.Version = Version
	app.Author, app.Email = "", ""
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "listen, l",
			Value:  "127.0.0.1:53",
			Usage:  "listen address: ‘host[:port]‘",
			EnvVar: "DNSMASQ_LISTEN",
		},
		cli.BoolFlag{
			Name:   "default-resolver, d",
			Usage:  "make go-dnsmasq the local primary nameserver (updates /etc/resolv.conf)",
			EnvVar: "DNSMASQ_DEFAULT",
		},
		cli.StringFlag{
			Name:   "nameservers, n",
			Value:  "",
			Usage:  "comma-separated list of name servers: ‘host[:port]‘",
			EnvVar: "DNSMASQ_SERVERS",
		},
		cli.StringFlag{
			Name:   "stubzones, z",
			Value:  "",
			Usage:  "domains to resolve using a specific nameserver: ‘fqdn[,fqdn]/host[:port]‘",
			EnvVar: "DNSMASQ_STUB",
		},
		cli.StringFlag{
			Name:   "hostsfile, f",
			Value:  "",
			Usage:  "full path to hostsfile (e.g. ‘/etc/hosts‘)",
			EnvVar: "DNSMASQ_HOSTSFILE",
		},
		cli.IntFlag{
			Name:   "hostsfile-poll, p",
			Value:  0,
			Usage:  "how frequently to poll hostsfile (in seconds, ‘0‘ to disable)",
			EnvVar: "DNSMASQ_POLL",
		},
		cli.StringFlag{
			Name:   "search-domains, s",
			Value:  "",
			Usage:  "specify SEARCH domains taking precedence over /etc/resolv.conf: ‘fqdn[,fqdn]‘",
			EnvVar: "DNSMASQ_SEARCH",
		},
		cli.BoolFlag{
			Name:   "append-search-domains, a",
			Usage:  "enable suffixing single-label queries with SEARCH domains",
			EnvVar: "DNSMASQ_APPEND",
		},
		cli.IntFlag{
			Name:   "rcache, r",
			Value:  0,
			Usage:  "capacity of the response cache (‘0‘ to disable caching)",
			EnvVar: "DNSMASQ_RCACHE",
		},
		cli.IntFlag{
			Name:   "rcache-ttl",
			Value:  server.RCacheTtl,
			Usage:  "TTL of entries in the response cache",
			EnvVar: "DNSMASQ_RCACHE_TTL",
		},
		cli.BoolFlag{
			Name:   "no-rec",
			Usage:  "disable recursion",
			EnvVar: "DNSMASQ_NOREC",
		},
		cli.BoolFlag{
			Name:   "round-robin",
			Usage:  "enable round robin of A/AAAA replies",
			EnvVar: "DNSMASQ_RR",
		},
		cli.BoolFlag{
			Name:   "systemd",
			Usage:  "bind to socket(s) activated by systemd (ignores --listen)",
			EnvVar: "DNSMASQ_SYSTEMD",
		},
		cli.BoolFlag{
			Name:   "verbose",
			Usage:  "enable verbose logging",
			EnvVar: "DNSMASQ_VERBOSE",
		},
		cli.BoolFlag{
			Name:   "syslog",
			Usage:  "enable syslog logging",
			EnvVar: "DNSMASQ_SYSLOG",
		},
	}
	app.Action = func(c *cli.Context) {
		exitReason := make(chan error)
		go func() {
			c := make(chan os.Signal, 1)
			signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
			sig := <-c
			log.Infoln("Application exit requested by signal:", sig)
			exitReason <- nil
		}()

		if c.Bool("verbose") {
			log.SetLevel(log.DebugLevel)
		}

		if c.Bool("syslog") {
			log.SetFormatter(&log.TextFormatter{DisableTimestamp: true, DisableColors: true})
			hook, err := logrus_syslog.NewSyslogHook("", "", syslog.LOG_DAEMON|syslog.LOG_INFO, "go-dnsmasq")
			if err != nil {
				log.Error("Unable to connect to local syslog daemon")
			} else {
				log.AddHook(hook)
			}
		} else {
			log.SetFormatter(&log.TextFormatter{DisableTimestamp: true})
		}

		if ns := c.String("nameservers"); ns != "" {
			for _, hostPort := range strings.Split(ns, ",") {
				if !strings.Contains(hostPort, ":") {
					hostPort += ":53"
				}

				if err := validateHostPort(hostPort); err != nil {
					log.Fatalf("This nameserver is invalid: %s", err)
				}

				nameservers = append(nameservers, hostPort)
			}
		}

		if sd := c.String("search-domains"); sd != "" {
			for _, domain := range strings.Split(sd, ",") {

				if dns.CountLabel(domain) < 2 {
					log.Fatalf("This search domain is not a FQDN: %s", domain)
				}
				domain = dns.Fqdn(strings.ToLower(domain))
				searchDomains = append(searchDomains, domain)
			}
		}

		if listen = c.String("listen"); !strings.Contains(listen, ":") {
			listen += ":53"
		}

		if err := validateHostPort(listen); err != nil {
			log.Fatalf("The listen address is invalid: %s", err)
		}

		config := &server.Config{
			DnsAddr:         listen,
			DefaultResolver: c.Bool("default-resolver"),
			Nameservers:     nameservers,
			Systemd:         c.Bool("systemd"),
			SearchDomains:   searchDomains,
			AppendDomain:    c.Bool("append-search-domains"),
			Hostsfile:       c.String("hostsfile"),
			PollInterval:    c.Int("hostsfile-poll"),
			RoundRobin:      c.Bool("round-robin"),
			NoRec:           c.Bool("no-rec"),
			ReadTimeout:     0,
			RCache:          c.Int("rcache"),
			RCacheTtl:       c.Int("rcache-ttl"),
			Verbose:         c.Bool("verbose"),
		}

		if err := server.SetDefaults(config); err != nil {
			if !config.NoRec && len(config.Nameservers) == 0 {
				log.Fatalf("No nameservers found in resolv.conf and --nameservers option not supplied: %s", err)
			} else if config.AppendDomain && len(config.SearchDomains) == 0 {
				log.Fatalf("No search domains found in resolv.conf and --search-domains option not supplied: %s", err)
			} else {
				log.Warnf("Error parsing resolv.conf: %s", err)
			}
		}

		if stubzones = c.String("stubzones"); stubzones != "" {
			stubmap := make(map[string][]string)
			segments := strings.Split(stubzones, "/")
			if len(segments) != 2 || len(segments[0]) == 0 || len(segments[1]) == 0 {
				log.Fatalf("The --stubzones argument is invalid")
			}

			hostPort = segments[1]
			if !strings.Contains(hostPort, ":") {
				hostPort += ":53"
			}

			if err := validateHostPort(hostPort); err != nil {
				log.Fatalf("This stubzones server address invalid: %s", err)
			}

			for _, sdomain := range strings.Split(segments[0], ",") {
				if dns.CountLabel(sdomain) < 2 {
					log.Fatalf("This stubzones domain is not a FQDN: %s", sdomain)
				}
				sdomain = dns.Fqdn(sdomain)
				stubmap[sdomain] = append(stubmap[sdomain], hostPort)
			}

			config.Stub = &stubmap
		}

		log.Infof("Starting go-dnsmasq server %s ...", Version)
		if config.AppendDomain {
			log.Infof("Search domains in use: %v", config.SearchDomains)
		}

		hf, err := hosts.NewHostsfile(config.Hostsfile, &hosts.Config{
			Poll:    config.PollInterval,
			Verbose: config.Verbose,
		})
		if err != nil {
			log.Fatalf("Error loading hostsfile: %s", err)
		}

		s := server.New(hf, config, Version)

		defer s.Stop()

		stats.Collect()

		if config.DefaultResolver {
			address, _, _ := net.SplitHostPort(config.DnsAddr)
			err := resolvconf.StoreAddress(address)
			if err != nil {
				log.Warnf("Failed to register as default nameserver: %s", err)
			}
			defer resolvconf.Clean()
		}

		go func() {
			if err := s.Run(); err != nil {
				exitReason <- err
			}
		}()

		exitErr = <-exitReason
		if exitErr != nil {
			log.Fatalf("Server error: %s", err)
		}
	}

	app.Run(os.Args)
}

func validateHostPort(hostPort string) error {
	host, port, err := net.SplitHostPort(hostPort)
	if err != nil {
		return err
	}
	if ip := net.ParseIP(host); ip == nil {
		return fmt.Errorf("Bad IP address: %s", host)
	}

	if p, _ := strconv.Atoi(port); p < 1 || p > 65535 {
		return fmt.Errorf("Bad port number %s", port)
	}
	return nil
}
<commit_msg>Bump version 0.9.6<commit_after>// Copyright (c) 2015 Jan Broer. All rights reserved.
// Use of this source code is governed by The MIT License (MIT) that can be
// found in the LICENSE file.

package main // import "github.com/janeczku/go-dnsmasq"

import (
	"fmt"
	"log/syslog"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"

	log "github.com/Sirupsen/logrus"
	logrus_syslog "github.com/Sirupsen/logrus/hooks/syslog"
	"github.com/codegangsta/cli"
	"github.com/miekg/dns"

	"github.com/janeczku/go-dnsmasq/hostsfile"
	"github.com/janeczku/go-dnsmasq/resolvconf"
	"github.com/janeczku/go-dnsmasq/server"
	"github.com/janeczku/go-dnsmasq/stats"
)

// var Version string
const Version = "0.9.6"

var (
	nameservers   = []string{}
	searchDomains = []string{}
	stubzones     = ""
	hostPort      = ""
	listen        = ""
)

var exitErr error

func init() {
	log.SetOutput(os.Stderr)
}

func main() {
	app := cli.NewApp()
	app.Name = "go-dnsmasq"
	app.Usage = "Lightweight caching DNS proxy for Docker containers"
	app.Version = Version
	app.Author, app.Email = "", ""
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "listen, l",
			Value:  "127.0.0.1:53",
			Usage:  "listen address: ‘host[:port]‘",
			EnvVar: "DNSMASQ_LISTEN",
		},
		cli.BoolFlag{
			Name:   "default-resolver, d",
			Usage:  "make go-dnsmasq the local primary nameserver (updates /etc/resolv.conf)",
			EnvVar: "DNSMASQ_DEFAULT",
		},
		cli.StringFlag{
			Name:   "nameservers, n",
			Value:  "",
			Usage:  "comma-separated list of name servers: ‘host[:port]‘",
			EnvVar: "DNSMASQ_SERVERS",
		},
		cli.StringFlag{
			Name:   "stubzones, z",
			Value:  "",
			Usage:  "domains to resolve using a specific nameserver: ‘fqdn[,fqdn]/host[:port]‘",
			EnvVar: "DNSMASQ_STUB",
		},
		cli.StringFlag{
			Name:   "hostsfile, f",
			Value:  "",
			Usage:  "full path to hostsfile (e.g. ‘/etc/hosts‘)",
			EnvVar: "DNSMASQ_HOSTSFILE",
		},
		cli.IntFlag{
			Name:   "hostsfile-poll, p",
			Value:  0,
			Usage:  "how frequently to poll hostsfile (in seconds, ‘0‘ to disable)",
			EnvVar: "DNSMASQ_POLL",
		},
		cli.StringFlag{
			Name:   "search-domains, s",
			Value:  "",
			Usage:  "specify SEARCH domains taking precedence over /etc/resolv.conf: ‘fqdn[,fqdn]‘",
			EnvVar: "DNSMASQ_SEARCH",
		},
		cli.BoolFlag{
			Name:   "append-search-domains, a",
			Usage:  "enable suffixing single-label queries with SEARCH domains",
			EnvVar: "DNSMASQ_APPEND",
		},
		cli.IntFlag{
			Name:   "rcache, r",
			Value:  0,
			Usage:  "capacity of the response cache (‘0‘ to disable caching)",
			EnvVar: "DNSMASQ_RCACHE",
		},
		cli.IntFlag{
			Name:   "rcache-ttl",
			Value:  server.RCacheTtl,
			Usage:  "TTL of entries in the response cache",
			EnvVar: "DNSMASQ_RCACHE_TTL",
		},
		cli.BoolFlag{
			Name:   "no-rec",
			Usage:  "disable recursion",
			EnvVar: "DNSMASQ_NOREC",
		},
		cli.BoolFlag{
			Name:   "round-robin",
			Usage:  "enable round robin of A/AAAA replies",
			EnvVar: "DNSMASQ_RR",
		},
		cli.BoolFlag{
			Name:   "systemd",
			Usage:  "bind to socket(s) activated by systemd (ignores --listen)",
			EnvVar: "DNSMASQ_SYSTEMD",
		},
		cli.BoolFlag{
			Name:   "verbose",
			Usage:  "enable verbose logging",
			EnvVar: "DNSMASQ_VERBOSE",
		},
		cli.BoolFlag{
			Name:   "syslog",
			Usage:  "enable syslog logging",
			EnvVar: "DNSMASQ_SYSLOG",
		},
	}
	app.Action = func(c *cli.Context) {
		exitReason := make(chan error)
		go func() {
			c := make(chan os.Signal, 1)
			signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
			sig := <-c
			log.Infoln("Application exit requested by signal:", sig)
			exitReason <- nil
		}()

		if c.Bool("verbose") {
			log.SetLevel(log.DebugLevel)
		}

		if c.Bool("syslog") {
			log.SetFormatter(&log.TextFormatter{DisableTimestamp: true, DisableColors: true})
			hook, err := logrus_syslog.NewSyslogHook("", "", syslog.LOG_DAEMON|syslog.LOG_INFO, "go-dnsmasq")
			if err != nil {
				log.Error("Unable to connect to local syslog daemon")
			} else {
				log.AddHook(hook)
			}
		} else {
			log.SetFormatter(&log.TextFormatter{DisableTimestamp: true})
		}

		if ns := c.String("nameservers"); ns != "" {
			for _, hostPort := range strings.Split(ns, ",") {
				if !strings.Contains(hostPort, ":") {
					hostPort += ":53"
				}

				if err := validateHostPort(hostPort); err != nil {
					log.Fatalf("This nameserver is invalid: %s", err)
				}

				nameservers = append(nameservers, hostPort)
			}
		}

		if sd := c.String("search-domains"); sd != "" {
			for _, domain := range strings.Split(sd, ",") {

				if dns.CountLabel(domain) < 2 {
					log.Fatalf("This search domain is not a FQDN: %s", domain)
				}
				domain = dns.Fqdn(strings.ToLower(domain))
				searchDomains = append(searchDomains, domain)
			}
		}

		if listen = c.String("listen"); !strings.Contains(listen, ":") {
			listen += ":53"
		}

		if err := validateHostPort(listen); err != nil {
			log.Fatalf("The listen address is invalid: %s", err)
		}

		config := &server.Config{
			DnsAddr:         listen,
			DefaultResolver: c.Bool("default-resolver"),
			Nameservers:     nameservers,
			Systemd:         c.Bool("systemd"),
			SearchDomains:   searchDomains,
			AppendDomain:    c.Bool("append-search-domains"),
			Hostsfile:       c.String("hostsfile"),
			PollInterval:    c.Int("hostsfile-poll"),
			RoundRobin:      c.Bool("round-robin"),
			NoRec:           c.Bool("no-rec"),
			ReadTimeout:     0,
			RCache:          c.Int("rcache"),
			RCacheTtl:       c.Int("rcache-ttl"),
			Verbose:         c.Bool("verbose"),
		}

		if err := server.SetDefaults(config); err != nil {
			if !config.NoRec && len(config.Nameservers) == 0 {
				log.Fatalf("No nameservers found in resolv.conf and --nameservers option not supplied: %s", err)
			} else if config.AppendDomain && len(config.SearchDomains) == 0 {
				log.Fatalf("No search domains found in resolv.conf and --search-domains option not supplied: %s", err)
			} else {
				log.Warnf("Error parsing resolv.conf: %s", err)
			}
		}

		if stubzones = c.String("stubzones"); stubzones != "" {
			stubmap := make(map[string][]string)
			segments := strings.Split(stubzones, "/")
			if len(segments) != 2 || len(segments[0]) == 0 || len(segments[1]) == 0 {
				log.Fatalf("The --stubzones argument is invalid")
			}

			hostPort = segments[1]
			if !strings.Contains(hostPort, ":") {
				hostPort += ":53"
			}

			if err := validateHostPort(hostPort); err != nil {
				log.Fatalf("This stubzones server address invalid: %s", err)
			}

			for _, sdomain := range strings.Split(segments[0], ",") {
				if dns.CountLabel(sdomain) < 2 {
					log.Fatalf("This stubzones domain is not a FQDN: %s", sdomain)
				}
				sdomain = dns.Fqdn(sdomain)
				stubmap[sdomain] = append(stubmap[sdomain], hostPort)
			}

			config.Stub = &stubmap
		}

		log.Infof("Starting go-dnsmasq server %s ...", Version)
		if config.AppendDomain {
			log.Infof("Search domains in use: %v", config.SearchDomains)
		}

		hf, err := hosts.NewHostsfile(config.Hostsfile, &hosts.Config{
			Poll:    config.PollInterval,
			Verbose: config.Verbose,
		})
		if err != nil {
			log.Fatalf("Error loading hostsfile: %s", err)
		}

		s := server.New(hf, config, Version)

		defer s.Stop()

		stats.Collect()

		if config.DefaultResolver {
			address, _, _ := net.SplitHostPort(config.DnsAddr)
			err := resolvconf.StoreAddress(address)
			if err != nil {
				log.Warnf("Failed to register as default nameserver: %s", err)
			}
			defer resolvconf.Clean()
		}

		go func() {
			if err := s.Run(); err != nil {
				exitReason <- err
			}
		}()

		exitErr = <-exitReason
		if exitErr != nil {
			log.Fatalf("Server error: %s", err)
		}
	}

	app.Run(os.Args)
}

func validateHostPort(hostPort string) error {
	host, port, err := net.SplitHostPort(hostPort)
	if err != nil {
		return err
	}
	if ip := net.ParseIP(host); ip == nil {
		return fmt.Errorf("Bad IP address: %s", host)
	}

	if p, _ := strconv.Atoi(port); p < 1 || p > 65535 {
		return fmt.Errorf("Bad port number %s", port)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import "net/http"

func main() {
	http.HandleFunc("/", hello)
	http.ListenAndServe(":8080", nil)
}

func hello(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("hello!"))
}
<commit_msg>Added weather route<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"strings"
)

func main() {
	http.HandleFunc("/", hello)

	http.HandleFunc("/weather/", func(w http.ResponseWriter, r *http.Request) {
		city := strings.SplitN(r.URL.Path, "/", 3)[2]

		data, err := query(city)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		json.NewEncoder(w).Encode(data)
	})

	http.ListenAndServe(":8080", nil)
}

func hello(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("hello!"))
}

type weatherData struct {
	Name string `json:"name"`
	Main struct {
		Kelvin float64 `json:"temp"`
	} `json:"main"`
}

func query(city string) (weatherData, error) {
	api_key, _ := os.LookupEnv("OPENWEATHER_API_KEY")
	url := fmt.Sprintf("http://api.openweathermap.org/data/2.5/weather?APPID=%s&q=%s", api_key, city)
	resp, err := http.Get(url)
	if err != nil {
		return weatherData{}, err
	}

	defer resp.Body.Close()

	var d weatherData

	if err := json.NewDecoder(resp.Body).Decode(&d); err != nil {
		return weatherData{}, err
	}

	return d, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/fluffle/goirc/client"
	"github.com/fluffle/goirc/state"
)

const (
	name    = "gibot"
	version = name + " v0.0"
	quit    = name + " exited"

	timeout = 20 * time.Second
	ping    = 2 * time.Minute
	split   = 100

	nick   = "[" + name + "]"
	server = "irc.freenode.net:7000"
	ssl    = true
)

type repo struct {
	Url      string
	IssuesRe *regexp.Regexp
	PullsRe  *regexp.Regexp
}

func newRepo(url string, aliases ...string) repo {
	return repo{
		Url:      url,
		IssuesRe: regexp.MustCompile(`(` + strings.Join(aliases, "|") + `)#([1-9][0-9]*)`),
		PullsRe:  regexp.MustCompile(`(` + strings.Join(aliases, "|") + `)!([1-9][0-9]*)`),
	}
}

func main() {
	repos := []repo{
		newRepo("https://gitlab.com/fdroid/fdroidclient",
			"", "c", "client", "fdroidclient"),
		newRepo("https://gitlab.com/fdroid/fdroidserver",
			"s", "server", "fdroidserver"),
		newRepo("https://gitlab.com/fdroid/fdroiddata",
			"d", "data", "fdroiddata"),
	}
	channels := map[string]struct{}{
		"#fdroid":     struct{}{},
		"#fdroid-dev": struct{}{},
	}

	c := client.Client(&client.Config{
		Me: &state.Nick{
			Nick:  nick,
			Ident: name,
			Name:  name,
		},
		PingFreq:    ping,
		NewNick:     func(s string) string { return s + "_" },
		Recover:     (*client.Conn).LogPanic,
		SplitLen:    split,
		Timeout:     timeout,
		Server:      server,
		SSL:         ssl,
		Version:     version,
		QuitMessage: quit,
	})
	c.HandleFunc(client.CONNECTED, func(conn *client.Conn, line *client.Line) {
		log.Printf("Connected.")
		for channel := range channels {
			conn.Join(channel)
		}
	})
	c.HandleFunc(client.DISCONNECTED, func(conn *client.Conn, line *client.Line) {
		log.Fatalf("Disconnected!")
	})
	c.HandleFunc(client.PRIVMSG, func(conn *client.Conn, line *client.Line) {
		target := line.Args[0]
		if _, e := channels[target]; !e {
			return
		}
		message := line.Args[1]
		for _, p := range repos {
			for _, issue := range p.IssuesRe.FindAllStringSubmatch(message, -1) {
				n := issue[2]
				notice := fmt.Sprintf("%s/issues/%s", p.Url, n)
				conn.Notice(target, notice)
			}
			for _, issue := range p.PullsRe.FindAllStringSubmatch(message, -1) {
				n := issue[2]
				notice := fmt.Sprintf("%s/merge_requests/%s", p.Url, n)
				conn.Notice(target, notice)
			}
		}
	})

	log.Printf("Connecting...")
	if err := c.Connect(); err != nil {
		log.Fatalf("Connection error: %v", err)
	}
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	<-sigc
	log.Printf("Quitting.")
	c.Quit()
}
<commit_msg>Wait two seconds between every notice<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/fluffle/goirc/client"
	"github.com/fluffle/goirc/state"
)

const (
	name    = "gibot"
	version = name + " v0.0"
	quit    = name + " exited"

	wait    = 2 * time.Second
	timeout = 20 * time.Second
	ping    = 2 * time.Minute
	split   = 100

	nick   = "[" + name + "]"
	server = "irc.freenode.net:7000"
	ssl    = true
)

type repo struct {
	Url      string
	IssuesRe *regexp.Regexp
	PullsRe  *regexp.Regexp
}

func newRepo(url string, aliases ...string) repo {
	return repo{
		Url:      url,
		IssuesRe: regexp.MustCompile(`(` + strings.Join(aliases, "|") + `)#([1-9][0-9]*)`),
		PullsRe:  regexp.MustCompile(`(` + strings.Join(aliases, "|") + `)!([1-9][0-9]*)`),
	}
}

type notice struct {
	channel string
	message string
}

type writer struct {
	notc chan notice
	conn *client.Conn
}

func (w *writer) work() {
	for {
		not := <-w.notc
		w.conn.Notice(not.channel, not.message)
		time.Sleep(wait)
	}
}

func main() {
	repos := []repo{
		newRepo("https://gitlab.com/fdroid/fdroidclient",
			"", "c", "client", "fdroidclient"),
		newRepo("https://gitlab.com/fdroid/fdroidserver",
			"s", "server", "fdroidserver"),
		newRepo("https://gitlab.com/fdroid/fdroiddata",
			"d", "data", "fdroiddata"),
	}
	channels := map[string]struct{}{
		"#fdroid":     struct{}{},
		"#fdroid-dev": struct{}{},
	}

	c := client.Client(&client.Config{
		Me: &state.Nick{
			Nick:  nick,
			Ident: name,
			Name:  name,
		},
		PingFreq:    ping,
		NewNick:     func(s string) string { return s + "_" },
		Recover:     (*client.Conn).LogPanic,
		SplitLen:    split,
		Timeout:     timeout,
		Server:      server,
		SSL:         ssl,
		Version:     version,
		QuitMessage: quit,
	})

	w := writer{
		notc: make(chan notice),
		conn: c,
	}

	c.HandleFunc(client.CONNECTED, func(conn *client.Conn, line *client.Line) {
		log.Printf("Connected.")
		for channel := range channels {
			conn.Join(channel)
		}
	})
	c.HandleFunc(client.DISCONNECTED, func(_ *client.Conn, line *client.Line) {
		log.Fatalf("Disconnected!")
	})
	c.HandleFunc(client.PRIVMSG, func(_ *client.Conn, line *client.Line) {
		channel := line.Args[0]
		if _, e := channels[channel]; !e {
			return
		}
		orig := line.Args[1]
		for _, p := range repos {
			for _, issue := range p.IssuesRe.FindAllStringSubmatch(orig, -1) {
				n := issue[2]
				message := fmt.Sprintf("%s/issues/%s", p.Url, n)
				go func() {
					w.notc <- notice{
						channel: channel,
						message: message,
					}
				}()
			}
			for _, issue := range p.PullsRe.FindAllStringSubmatch(orig, -1) {
				n := issue[2]
				message := fmt.Sprintf("%s/merge_requests/%s", p.Url, n)
				go func() {
					w.notc <- notice{
						channel: channel,
						message: message,
					}
				}()
			}
		}
	})

	log.Printf("Connecting...")
	if err := c.Connect(); err != nil {
		log.Fatalf("Connection error: %v", err)
	}

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)

	go w.work()

	<-sigc
	log.Printf("Quitting.")
	c.Quit()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"os"
	"regexp"
	"sync"
	"text/template"
	"time"

	"github.com/fatih/color"
	"github.com/spf13/pflag"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/kubernetes"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
	"k8s.io/client-go/tools/clientcmd"
)

func main() {
	var (
		contextName           string
		kubeconfigPath        string
		labelSelectorExpr     string
		namespace             string
		allNamespaces         bool
		quiet                 bool
		timestamps            bool
		raw                   bool
		tmplString            string
		sinceStart            bool
		showVersion           bool
		includePatterns       []*regexp.Regexp
		excludePatternStrings []string
		noColor               bool
	)

	flags := pflag.NewFlagSet("ktail", pflag.ExitOnError)
	flags.Usage = func() {
		flags.PrintDefaults()
	}
	flags.StringVar(&contextName, "context", "", "Kubernetes context name")
	flags.StringVar(&kubeconfigPath, "kubeconfig", "",
		"Path to kubeconfig (only required out-of-cluster)")
	flags.StringVarP(&namespace, "namespace", "n", "", "Kubernetes namespace")
	flags.StringArrayVarP(&excludePatternStrings, "exclude", "x", []string{},
		"Exclude using a regular expression. Pattern can be repeated. Takes priority over"+
			" include patterns and labels.")
	flags.StringVarP(&labelSelectorExpr, "selector", "l", "",
		"Match pods by label (see 'kubectl get -h' for syntax).")
	flags.StringVarP(&tmplString, "template", "t", "",
		"Template to format each line. For example, for"+
			" just the message, use --template '{{ .Message }}'.")
	flags.BoolVar(&allNamespaces, "all-namespaces", false, "Apply to all Kubernetes namespaces")
	flags.BoolVarP(&raw, "raw", "r", false, "Don't format output; output messages only (unless --timestamps)")
	flags.BoolVarP(&timestamps, "timestamps", "T", false, "Include timestamps on each line")
	flags.BoolVarP(&quiet, "quiet", "q", false, "Don't print events about new/deleted pods")
	flags.BoolVarP(&sinceStart, "since-start", "s", false,
		"Start reading log from the beginning of the container's lifetime.")
	flags.BoolVarP(&showVersion, "version", "", false, "Show version.")
	flags.BoolVarP(&noColor, "no-color", "", false, "Disable color.")

	if err := flags.Parse(os.Args[1:]); err != nil {
		fail(err.Error())
		os.Exit(1)
	}

	color.NoColor = noColor

	if showVersion {
		fmt.Printf("ktail %s\n", version)
		os.Exit(0)
	}

	var excludePatterns []*regexp.Regexp
	for _, p := range excludePatternStrings {
		r, err := regexp.Compile(p)
		if err != nil {
			fail("Invalid regexp: %q: %s\n", p, err)
		}
		excludePatterns = append(excludePatterns, r)
	}

	for _, arg := range flags.Args() {
		r, err := regexp.Compile(arg)
		if err != nil {
			fail("Invalid regexp: %q: %s\n", arg, err)
		}
		includePatterns = append(includePatterns, r)
	}

	if kubeconfigPath == "" {
		if os.Getenv("KUBECONFIG") != "" {
			kubeconfigPath = os.Getenv("KUBECONFIG")
		} else {
			kubeconfigPath = clientcmd.RecommendedHomeFile
		}
	}

	labelSelector := labels.Everything()
	if labelSelectorExpr != "" {
		if sel, err := labels.Parse(labelSelectorExpr); err != nil {
			fail(err.Error())
		} else {
			labelSelector = sel
		}
	}

	inclusionMatcher := buildMatcher(includePatterns, labelSelector, true)
	exclusionMatcher := buildMatcher(excludePatterns, nil, false)

	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{
			ExplicitPath: kubeconfigPath,
		},
		&clientcmd.ConfigOverrides{
			CurrentContext: contextName,
		})

	config, err := clientConfig.ClientConfig()
	if err != nil {
		fail(err.Error())
	}

	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		fail(err.Error())
	}

	rawConfig, err := clientConfig.RawConfig()
	if err != nil {
		fail(err.Error())
	}

	if allNamespaces {
		namespace = v1.NamespaceAll
	} else if namespace == "" {
		if rawConfig.Contexts[rawConfig.CurrentContext].Namespace == "" {
			namespace = v1.NamespaceDefault
		} else {
			namespace = rawConfig.Contexts[rawConfig.CurrentContext].Namespace
		}
	}

	var tmpl *template.Template
	if tmplString != "" {
		var err error
		tmpl, err = template.New("line").Parse(tmplString)
		if err != nil {
			fail("invalid template: %s", err)
		}
	}

	formatPod := func(pod *v1.Pod) string {
		if allNamespaces {
			return fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)
		}
		return pod.Name
	}

	formatPodAndContainer := func(pod *v1.Pod, container *v1.Container) string {
		return fmt.Sprintf("%s:%s", formatPod(pod), container.Name)
	}

	var printEvent func(*LogEvent) error

	if tmpl != nil {
		printEvent = func(event *LogEvent) error {
			type templateEvent struct {
				Pod       *v1.Pod
				Container *v1.Container
				Timestamp string
				Message   string
			}

			var buf bytes.Buffer
			if err := tmpl.Execute(&buf, &templateEvent{
				Pod:       event.Pod,
				Container: event.Container,
				Message:   event.Message,
				Timestamp: formatTimestamp(event.Timestamp),
			}); err != nil {
				return err
			}

			_, err := fmt.Fprintln(os.Stdout, string(buf.Bytes()))
			return err
		}
	} else {
		printEvent = func(event *LogEvent) error {
			col := getColorConfig(event.Pod.Name, event.Container.Name)

			var line string
			if !raw {
				if timestamps {
					line = col.metadata.Sprint(formatTimestamp(event.Timestamp))
					line += " "
				}
				if allNamespaces {
					line += col.labels.Sprint(fmt.Sprintf("%s/%s:%s",
						event.Pod.Namespace, event.Pod.Name, event.Container.Name))
				} else {
					line += col.labels.Sprint(fmt.Sprintf("%s:%s", event.Pod.Name, event.Container.Name))
				}
				line += " "
			}

			line += event.Message

			_, err := fmt.Fprintln(os.Stdout, line)
			return err
		}
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	var stdoutMutex sync.Mutex
	controller := NewController(clientset, ControllerOptions{
		Namespace:        namespace,
		InclusionMatcher: inclusionMatcher,
		ExclusionMatcher: exclusionMatcher,
		SinceStart:       sinceStart,
	},
		Callbacks{
			OnEvent: func(event LogEvent) {
				stdoutMutex.Lock()
				defer stdoutMutex.Unlock()
				if err := printEvent(&event); err != nil {
					printError(fmt.Sprintf("Could not write event: %s", err))
					cancel()
				}
			},
			OnEnter: func(pod *v1.Pod, container *v1.Container, initialAddPhase bool) bool {
				if !quiet {
					if initialAddPhase {
						printInfo("Attached to container [%s]", formatPodAndContainer(pod, container))
					} else {
						printInfo("New container [%s]", formatPodAndContainer(pod, container))
					}
				}
				return true
			},
			OnExit: func(pod *v1.Pod, container *v1.Container) {
				if !quiet {
					var status = "unknown"
					for _, containerStatus := range pod.Status.ContainerStatuses {
						if containerStatus.Name == container.Name {
							if containerStatus.State.Running != nil {
								status = "running"
							} else if containerStatus.State.Waiting != nil {
								status = "waiting"
							} else if containerStatus.State.Terminated != nil {
								status = "terminated"
							}
							break
						}
					}
					printInfo(fmt.Sprintf("Container left (%s) [%s]", status,
						formatPodAndContainer(pod, container)))
				}
			},
			OnError: func(pod *v1.Pod, container *v1.Container, err error) {
				printError(fmt.Sprintf("Error while tailing container [%s]: %s",
					formatPodAndContainer(pod, container), err))
			},
		})

	if err := controller.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
		printError(err.Error())
	}
}

func printInfo(format string, args ...interface{}) {
	message := fmt.Sprintf(format, args...)
	_, _ = fmt.Fprintf(os.Stderr, colorInfo(fmt.Sprintf("==> %s\n", message)))
}

func printError(format string, args ...interface{}) {
	message := fmt.Sprintf(format, args...)
	_, _ = fmt.Fprintf(os.Stderr, colorError(fmt.Sprintf("==> %s\n", message)))
}

func formatTimestamp(t *time.Time) string {
	s := t.Local().Format("2006-01-02T15:04:05.999")
	for len(s) < 23 {
		s += "0"
	}
	return s
}

func fail(format string, args ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, format, args...)
	os.Exit(1)
}

var (
	colorInfo  = color.New(color.FgYellow).SprintFunc()
	colorError = color.New(color.FgRed).SprintFunc()
)
<commit_msg>fix: don't return error on --help.<commit_after>package main

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"os"
	"regexp"
	"sync"
	"text/template"
	"time"

	"github.com/fatih/color"
	"github.com/spf13/pflag"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/kubernetes"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
	"k8s.io/client-go/tools/clientcmd"
)

func main() {
	var (
		contextName           string
		kubeconfigPath        string
		labelSelectorExpr     string
		namespace             string
		allNamespaces         bool
		quiet                 bool
		timestamps            bool
		raw                   bool
		tmplString            string
		sinceStart            bool
		showVersion           bool
		includePatterns       []*regexp.Regexp
		excludePatternStrings []string
		noColor               bool
	)

	flags := pflag.NewFlagSet("ktail", pflag.ExitOnError)
	flags.Usage = func() {
		flags.PrintDefaults()
	}
	flags.StringVar(&contextName, "context", "", "Kubernetes context name")
	flags.StringVar(&kubeconfigPath, "kubeconfig", "",
		"Path to kubeconfig (only required out-of-cluster)")
	flags.StringVarP(&namespace, "namespace", "n", "", "Kubernetes namespace")
	flags.StringArrayVarP(&excludePatternStrings, "exclude", "x", []string{},
		"Exclude using a regular expression. Pattern can be repeated. Takes priority over"+
			" include patterns and labels.")
	flags.StringVarP(&labelSelectorExpr, "selector", "l", "",
		"Match pods by label (see 'kubectl get -h' for syntax).")
	flags.StringVarP(&tmplString, "template", "t", "",
		"Template to format each line. For example, for"+
			" just the message, use --template '{{ .Message }}'.")
	flags.BoolVar(&allNamespaces, "all-namespaces", false, "Apply to all Kubernetes namespaces")
	flags.BoolVarP(&raw, "raw", "r", false, "Don't format output; output messages only (unless --timestamps)")
	flags.BoolVarP(&timestamps, "timestamps", "T", false, "Include timestamps on each line")
	flags.BoolVarP(&quiet, "quiet", "q", false, "Don't print events about new/deleted pods")
	flags.BoolVarP(&sinceStart, "since-start", "s", false,
		"Start reading log from the beginning of the container's lifetime.")
	flags.BoolVarP(&showVersion, "version", "", false, "Show version.")
	flags.BoolVarP(&noColor, "no-color", "", false, "Disable color.")

	if err := flags.Parse(os.Args[1:]); err != nil {
		if err == pflag.ErrHelp {
			os.Exit(2)
		}
		fail(err.Error())
		os.Exit(1)
	}

	color.NoColor = noColor

	if showVersion {
		fmt.Printf("ktail %s\n", version)
		os.Exit(0)
	}

	var excludePatterns []*regexp.Regexp
	for _, p := range excludePatternStrings {
		r, err := regexp.Compile(p)
		if err != nil {
			fail("Invalid regexp: %q: %s\n", p, err)
		}
		excludePatterns = append(excludePatterns, r)
	}

	for _, arg := range flags.Args() {
		r, err := regexp.Compile(arg)
		if err != nil {
			fail("Invalid regexp: %q: %s\n", arg, err)
		}
		includePatterns = append(includePatterns, r)
	}

	if kubeconfigPath == "" {
		if os.Getenv("KUBECONFIG") != "" {
			kubeconfigPath = os.Getenv("KUBECONFIG")
		} else {
			kubeconfigPath = clientcmd.RecommendedHomeFile
		}
	}

	labelSelector := labels.Everything()
	if labelSelectorExpr != "" {
		if sel, err := labels.Parse(labelSelectorExpr); err != nil {
			fail(err.Error())
		} else {
			labelSelector = sel
		}
	}

	inclusionMatcher := buildMatcher(includePatterns, labelSelector, true)
	exclusionMatcher := buildMatcher(excludePatterns, nil, false)

	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{
			ExplicitPath: kubeconfigPath,
		},
		&clientcmd.ConfigOverrides{
			CurrentContext: contextName,
		})

	config, err := clientConfig.ClientConfig()
	if err != nil {
		fail(err.Error())
	}

	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		fail(err.Error())
	}

	rawConfig, err := clientConfig.RawConfig()
	if err != nil {
		fail(err.Error())
	}

	if allNamespaces {
		namespace = v1.NamespaceAll
	} else if namespace == "" {
		if rawConfig.Contexts[rawConfig.CurrentContext].Namespace == "" {
			namespace = v1.NamespaceDefault
		} else {
			namespace = rawConfig.Contexts[rawConfig.CurrentContext].Namespace
		}
	}

	var tmpl *template.Template
	if tmplString != "" {
		var err error
		tmpl, err = template.New("line").Parse(tmplString)
		if err != nil {
			fail("invalid template: %s", err)
		}
	}

	formatPod := func(pod *v1.Pod) string {
		if allNamespaces {
			return fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)
		}
		return pod.Name
	}

	formatPodAndContainer := func(pod *v1.Pod, container *v1.Container) string {
		return fmt.Sprintf("%s:%s", formatPod(pod), container.Name)
	}

	var printEvent func(*LogEvent) error

	if tmpl != nil {
		printEvent = func(event *LogEvent) error {
			type templateEvent struct {
				Pod       *v1.Pod
				Container *v1.Container
				Timestamp string
				Message   string
			}

			var buf bytes.Buffer
			if err := tmpl.Execute(&buf, &templateEvent{
				Pod:       event.Pod,
				Container: event.Container,
				Message:   event.Message,
				Timestamp: formatTimestamp(event.Timestamp),
			}); err != nil {
				return err
			}

			_, err := fmt.Fprintln(os.Stdout, string(buf.Bytes()))
			return err
		}
	} else {
		printEvent = func(event *LogEvent) error {
			col := getColorConfig(event.Pod.Name, event.Container.Name)

			var line string
			if !raw {
				if timestamps {
					line = col.metadata.Sprint(formatTimestamp(event.Timestamp))
					line += " "
				}
				if allNamespaces {
					line += col.labels.Sprint(fmt.Sprintf("%s/%s:%s",
						event.Pod.Namespace, event.Pod.Name, event.Container.Name))
				} else {
					line += col.labels.Sprint(fmt.Sprintf("%s:%s", event.Pod.Name, event.Container.Name))
				}
				line += " "
			}

			line += event.Message

			_, err := fmt.Fprintln(os.Stdout, line)
			return err
		}
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	var stdoutMutex sync.Mutex
	controller := NewController(clientset, ControllerOptions{
		Namespace:        namespace,
		InclusionMatcher: inclusionMatcher,
		ExclusionMatcher: exclusionMatcher,
		SinceStart:       sinceStart,
	},
		Callbacks{
			OnEvent: func(event LogEvent) {
				stdoutMutex.Lock()
				defer stdoutMutex.Unlock()
				if err := printEvent(&event); err != nil {
					printError(fmt.Sprintf("Could not write event: %s", err))
					cancel()
				}
			},
			OnEnter: func(pod *v1.Pod, container *v1.Container, initialAddPhase bool) bool {
				if !quiet {
					if initialAddPhase {
						printInfo("Attached to container [%s]", formatPodAndContainer(pod, container))
					} else {
						printInfo("New container [%s]", formatPodAndContainer(pod, container))
					}
				}
				return true
			},
			OnExit: func(pod *v1.Pod, container *v1.Container) {
				if !quiet {
					var status = "unknown"
					for _, containerStatus := range pod.Status.ContainerStatuses {
						if containerStatus.Name == container.Name {
							if containerStatus.State.Running != nil {
								status = "running"
							} else if containerStatus.State.Waiting != nil {
								status = "waiting"
							} else if containerStatus.State.Terminated != nil {
								status = "terminated"
							}
							break
						}
					}
					printInfo(fmt.Sprintf("Container left (%s) [%s]", status,
						formatPodAndContainer(pod, container)))
				}
			},
			OnError: func(pod *v1.Pod, container *v1.Container, err error) {
				printError(fmt.Sprintf("Error while tailing container [%s]: %s",
					formatPodAndContainer(pod, container), err))
			},
		})

	if err := controller.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
		printError(err.Error())
	}
}

func printInfo(format string, args ...interface{}) {
	message := fmt.Sprintf(format, args...)
	_, _ = fmt.Fprintf(os.Stderr, colorInfo(fmt.Sprintf("==> %s\n", message)))
}

func printError(format string, args ...interface{}) {
	message := fmt.Sprintf(format, args...)
	_, _ = fmt.Fprintf(os.Stderr, colorError(fmt.Sprintf("==> %s\n", message)))
}

func formatTimestamp(t *time.Time) string {
	s := t.Local().Format("2006-01-02T15:04:05.999")
	for len(s) < 23 {
		s += "0"
	}
	return s
}

func fail(format string, args ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, format, args...)
	os.Exit(1)
}

var (
	colorInfo  = color.New(color.FgYellow).SprintFunc()
	colorError = color.New(color.FgRed).SprintFunc()
)
<|endoftext|>


<commit_before>package main

import (
	"bitbucket.org/anacrolix/dms/dlna/dms"
	"bitbucket.org/anacrolix/dms/rrcache"
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/signal"
	"os/user"
	"path/filepath"
	"runtime"
	"sync"
	"syscall"
)

type dmsConfig struct {
	Path             string
	IfName           string
	Http             string
	FriendlyName     string
	LogHeaders       bool
	FFprobeCachePath string
}

func (config *dmsConfig) load(configPath string) {
	file, err := os.Open(configPath)
	if err != nil {
		log.Printf("config error (config file: '%s'): %v\n", configPath, err)
		return
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	err = decoder.Decode(&config)
	if err != nil {
		log.Printf("config error: %v\n", err)
		return
	}
}

//default config
var config = &dmsConfig{
	Path:             "",
	IfName:           "",
	Http:             ":1338",
	FriendlyName:     "",
	LogHeaders:       false,
	FFprobeCachePath: getDefaultFFprobeCachePath(),
}

func getDefaultFFprobeCachePath() (path string) {
	_user, err := user.Current()
	if err != nil {
		log.Print(err)
		return
	}
	path = filepath.Join(_user.HomeDir, ".dms-ffprobe-cache")
	return
}

type fFprobeCache struct {
	c *rrcache.RRCache
	sync.Mutex
}

func (fc *fFprobeCache) Get(key interface{}) (value interface{}, ok bool) {
	fc.Lock()
	defer fc.Unlock()
	return fc.c.Get(key)
}

func (fc *fFprobeCache) Set(key interface{}, value interface{}) {
	fc.Lock()
	defer fc.Unlock()
	var size int64
	for _, v := range []interface{}{key, value} {
		b, err := json.Marshal(v)
		if err != nil {
			panic(err)
		}
		size += int64(len(b))
	}
	fc.c.Set(key, value, size)
}

func main() {
	log.SetFlags(log.Ltime | log.Lshortfile)

	path := flag.String("path", config.Path, "browse root path")
	ifName := flag.String("ifname", config.IfName, "specific SSDP network interface")
	http := flag.String("http", config.Http, "http server port")
	friendlyName := flag.String("friendlyName", config.FriendlyName, "server friendly name")
	logHeaders := flag.Bool("logHeaders", config.LogHeaders, "log HTTP headers")
	fFprobeCachePath := flag.String("fFprobeCachePath", config.FFprobeCachePath, "path to FFprobe cache file")
	configFilePath := flag.String("config", "", "json configuration file")

	flag.Parse()
	if flag.NArg() != 0 {
		flag.Usage()
		log.Fatalf("%s: %s\n", "unexpected positional arguments", flag.Args())
	}

	config.Path = *path
	config.IfName = *ifName
	config.Http = *http
	config.FriendlyName = *friendlyName
	config.LogHeaders = *logHeaders
	config.FFprobeCachePath = *fFprobeCachePath

	if len(*configFilePath) > 0 {
		config.load(*configFilePath)
	}

	cache := &fFprobeCache{
		c: rrcache.New(64 << 20),
	}
	if err := loadFFprobeCache(cache, config.FFprobeCachePath); err != nil {
		log.Print(err)
	}

	dmsServer := &dms.Server{
		Interfaces: func(ifName string) (ifs []net.Interface) {
			var err error
			if ifName == "" {
				ifs, err = net.Interfaces()
			} else {
				var if_ *net.Interface
				if_, err = net.InterfaceByName(ifName)
				if if_ != nil {
					ifs = append(ifs, *if_)
				}
			}
			if err != nil {
				log.Fatal(err)
			}
			return
		}(config.IfName),
		HTTPConn: func() net.Listener {
			conn, err := net.Listen("tcp", config.Http)
			if err != nil {
				log.Fatal(err)
			}
			return conn
		}(),
		FriendlyName:   config.FriendlyName,
		RootObjectPath: filepath.Clean(config.Path),
		FFProbeCache:   cache,
		LogHeaders:     config.LogHeaders,
	}
	go func() {
		if err := dmsServer.Serve(); err != nil {
			log.Fatal(err)
		}
	}()
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
	<-sigs
	err := dmsServer.Close()
	if err != nil {
		log.Fatal(err)
	}
	if err := saveFFprobeCache(cache, config.FFprobeCachePath); err != nil {
		log.Print(err)
	}
}

func loadFFprobeCache(cache *fFprobeCache, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	dec := json.NewDecoder(f)
	var items []dms.FfprobeCacheItem
	err = dec.Decode(&items)
	if err != nil {
		return err
	}
	for _, item := range items {
		cache.Set(item.Key, item.Value)
	}
	log.Printf("added %d items from cache", len(items))
	return nil
}

func saveFFprobeCache(cache *fFprobeCache, path string) error {
	cache.Lock()
	items := cache.c.Items()
	cache.Unlock()
	f, err := ioutil.TempFile(filepath.Dir(path), filepath.Base(path))
	if err != nil {
		return err
	}
	enc := json.NewEncoder(f)
	err = enc.Encode(items)
	f.Close()
	if err != nil {
		os.Remove(f.Name())
		return err
	}
	if runtime.GOOS == "windows" {
		err = os.Remove(path)
		if err == os.ErrNotExist {
			err = nil
		}
	}
	if err == nil {
		err = os.Rename(f.Name(), path)
	}
	if err == nil {
		log.Printf("saved cache with %d items", len(items))
	} else {
		os.Remove(f.Name())
	}
	return err
}
<commit_msg>[Refactoring] Transform {load|save}FFprobeCache info methods of fFprobeCache<commit_after>package main

import (
	"bitbucket.org/anacrolix/dms/dlna/dms"
	"bitbucket.org/anacrolix/dms/rrcache"
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/signal"
	"os/user"
	"path/filepath"
	"runtime"
	"sync"
	"syscall"
)

type dmsConfig struct {
	Path             string
	IfName           string
	Http             string
	FriendlyName     string
	LogHeaders       bool
	FFprobeCachePath string
}

func (config *dmsConfig) load(configPath string) {
	file, err := os.Open(configPath)
	if err != nil {
		log.Printf("config error (config file: '%s'): %v\n", configPath, err)
		return
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	err = decoder.Decode(&config)
	if err != nil {
		log.Printf("config error: %v\n", err)
		return
	}
}

//default config
var config = &dmsConfig{
	Path:             "",
	IfName:           "",
	Http:             ":1338",
	FriendlyName:     "",
	LogHeaders:       false,
	FFprobeCachePath: getDefaultFFprobeCachePath(),
}

func getDefaultFFprobeCachePath() (path string) {
	_user, err := user.Current()
	if err != nil {
		log.Print(err)
		return
	}
	path = filepath.Join(_user.HomeDir, ".dms-ffprobe-cache")
	return
}

type fFprobeCache struct {
	c *rrcache.RRCache
	sync.Mutex
}

func (fc *fFprobeCache) Get(key interface{}) (value interface{}, ok bool) {
	fc.Lock()
	defer fc.Unlock()
	return fc.c.Get(key)
}

func (fc *fFprobeCache) Set(key interface{}, value interface{}) {
	fc.Lock()
	defer fc.Unlock()
	var size int64
	for _, v := range []interface{}{key, value} {
		b, err := json.Marshal(v)
		if err != nil {
			panic(err)
		}
		size += int64(len(b))
	}
	fc.c.Set(key, value, size)
}

func main() {
	log.SetFlags(log.Ltime | log.Lshortfile)

	path := flag.String("path", config.Path, "browse root path")
	ifName := flag.String("ifname", config.IfName, "specific SSDP network interface")
	http := flag.String("http", config.Http, "http server port")
	friendlyName := flag.String("friendlyName", config.FriendlyName, "server friendly name")
	logHeaders := flag.Bool("logHeaders", config.LogHeaders, "log HTTP headers")
	fFprobeCachePath := flag.String("fFprobeCachePath", config.FFprobeCachePath, "path to FFprobe cache file")
	configFilePath := flag.String("config", "", "json configuration file")

	flag.Parse()
	if flag.NArg() != 0 {
		flag.Usage()
		log.Fatalf("%s: %s\n", "unexpected positional arguments", flag.Args())
	}

	config.Path = *path
	config.IfName = *ifName
	config.Http = *http
	config.FriendlyName = *friendlyName
	config.LogHeaders = *logHeaders
	config.FFprobeCachePath = *fFprobeCachePath

	if len(*configFilePath) > 0 {
		config.load(*configFilePath)
	}

	cache := &fFprobeCache{
		c: rrcache.New(64 << 20),
	}
	if err := cache.load(config.FFprobeCachePath); err != nil {
		log.Print(err)
	}

	dmsServer := &dms.Server{
		Interfaces: func(ifName string) (ifs []net.Interface) {
			var err error
			if ifName == "" {
				ifs, err = net.Interfaces()
			} else {
				var if_ *net.Interface
				if_, err = net.InterfaceByName(ifName)
				if if_ != nil {
					ifs = append(ifs, *if_)
				}
			}
			if err != nil {
				log.Fatal(err)
			}
			return
		}(config.IfName),
		HTTPConn: func() net.Listener {
			conn, err := net.Listen("tcp", config.Http)
			if err != nil {
				log.Fatal(err)
			}
			return conn
		}(),
		FriendlyName:   config.FriendlyName,
		RootObjectPath: filepath.Clean(config.Path),
		FFProbeCache:   cache,
		LogHeaders:     config.LogHeaders,
	}
	go func() {
		if err := dmsServer.Serve(); err != nil {
			log.Fatal(err)
		}
	}()
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
	<-sigs
	err := dmsServer.Close()
	if err != nil {
		log.Fatal(err)
	}
	if err := cache.save(config.FFprobeCachePath); err != nil {
		log.Print(err)
	}
}

func (cache *fFprobeCache) load(path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	dec := json.NewDecoder(f)
	var items []dms.FfprobeCacheItem
	err = dec.Decode(&items)
	if err != nil {
		return err
	}
	for _, item := range items {
		cache.Set(item.Key, item.Value)
	}
	log.Printf("added %d items from cache", len(items))
	return nil
}

func (cache *fFprobeCache) save(path string) error {
	cache.Lock()
	items := cache.c.Items()
	cache.Unlock()
	f, err := ioutil.TempFile(filepath.Dir(path), filepath.Base(path))
	if err != nil {
		return err
	}
	enc := json.NewEncoder(f)
	err = enc.Encode(items)
	f.Close()
	if err != nil {
		os.Remove(f.Name())
		return err
	}
	if runtime.GOOS == "windows" {
		err = os.Remove(path)
		if err == os.ErrNotExist {
			err = nil
		}
	}
	if err == nil {
		err = os.Rename(f.Name(), path)
	}
	if err == nil {
		log.Printf("saved cache with %d items", len(items))
	} else {
		os.Remove(f.Name())
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"html/template"
	"net/http"
	"time"

	"github.com/gorilla/mux"

	"github.com/droptheplot/rand_chat/app"
	"github.com/droptheplot/rand_chat/env"
	"github.com/droptheplot/rand_chat/telegram"
	"github.com/droptheplot/rand_chat/vk"
)

var templates = template.Must(template.ParseGlob(env.Config.Templates))
var db, logger = env.Init()

func main() {
	r := mux.NewRouter()

	r.HandleFunc("/", IndexHandler).Methods("GET")
	r.HandleFunc("/api/chart", ChartHandler).Methods("GET")
	r.HandleFunc("/api/telegram", TelegramHandler).Methods("POST")
	r.HandleFunc("/api/vk", VKHandler).Methods("POST")

	r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(env.Config.Static))))

	srv := &http.Server{
		Handler:      r,
		Addr:         ":443",
		WriteTimeout: 30 * time.Second,
		ReadTimeout:  30 * time.Second,
	}

	srv.ListenAndServeTLS(env.Config.TLS.Cert, env.Config.TLS.Key)
}

func IndexHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")

	err := templates.ExecuteTemplate(w, "index.html", map[string]string{"Title": "Рандомный чат"})

	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func TelegramHandler(w http.ResponseWriter, r *http.Request) {
	var update telegram.Update
	json.NewDecoder(r.Body).Decode(&update)

	app.Message{
		Text: update.Message.Text,
		User: app.User{
			ID:  update.Message.User.ID,
			App: "telegram",
		},
	}.Handle(db, logger)

	w.WriteHeader(http.StatusOK)
}

func VKHandler(w http.ResponseWriter, r *http.Request) {
	var event vk.Event
	json.NewDecoder(r.Body).Decode(&event)

	if event.Type == "confirmation" {
		w.Write([]byte(env.Config.VK.Confirmation))
		return
	}

	if event.Message.Out == 1 {
		w.Write([]byte("ok"))
		return
	}

	app.Message{
		Text: event.Message.Body,
		User: app.User{
			ID:  event.Message.UserID,
			App: "vk",
		},
	}.Handle(db, logger)

	w.Write([]byte("ok"))
}

func ChartHandler(w http.ResponseWriter, r *http.Request) {
	var charts = make(map[string][]interface{})

	for _, chart := range app.GetCharts(db) {
		charts["dates"] = append(charts["dates"], chart.Date.Format("2 Jan"))
		charts["counts"] = append(charts["counts"], chart.Count)
	}

	result, _ := json.Marshal(charts)

	w.Header().Set("Content-Type", "application/json")
	w.Write(result)
}
<commit_msg>Add RecoveryHandler to http server<commit_after>package main

import (
	"encoding/json"
	"html/template"
	"net/http"
	"time"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"

	"github.com/droptheplot/rand_chat/app"
	"github.com/droptheplot/rand_chat/env"
	"github.com/droptheplot/rand_chat/telegram"
	"github.com/droptheplot/rand_chat/vk"
)

var templates = template.Must(template.ParseGlob(env.Config.Templates))
var db, logger = env.Init()

func main() {
	r := mux.NewRouter()

	r.HandleFunc("/", IndexHandler).Methods("GET")
	r.HandleFunc("/api/chart", ChartHandler).Methods("GET")
	r.HandleFunc("/api/telegram", TelegramHandler).Methods("POST")
	r.HandleFunc("/api/vk", VKHandler).Methods("POST")

	r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(env.Config.Static))))

	srv := &http.Server{
		Handler:      handlers.RecoveryHandler()(r),
		Addr:         ":443",
		WriteTimeout: 30 * time.Second,
		ReadTimeout:  30 * time.Second,
	}

	srv.ListenAndServeTLS(env.Config.TLS.Cert, env.Config.TLS.Key)
}

func IndexHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")

	err := templates.ExecuteTemplate(w, "index.html", map[string]string{"Title": "Рандомный чат"})

	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func TelegramHandler(w http.ResponseWriter, r *http.Request) {
	var update telegram.Update
	json.NewDecoder(r.Body).Decode(&update)

	app.Message{
		Text: update.Message.Text,
		User: app.User{
			ID:  update.Message.User.ID,
			App: "telegram",
		},
	}.Handle(db, logger)

	w.WriteHeader(http.StatusOK)
}

func VKHandler(w http.ResponseWriter, r *http.Request) {
	var event vk.Event
	json.NewDecoder(r.Body).Decode(&event)

	if event.Type == "confirmation" {
		w.Write([]byte(env.Config.VK.Confirmation))
		return
	}

	if event.Message.Out == 1 {
		w.Write([]byte("ok"))
		return
	}

	app.Message{
		Text: event.Message.Body,
		User: app.User{
			ID:  event.Message.UserID,
			App: "vk",
		},
	}.Handle(db, logger)

	w.Write([]byte("ok"))
}

func ChartHandler(w http.ResponseWriter, r *http.Request) {
	var charts = make(map[string][]interface{})

	for _, chart := range app.GetCharts(db) {
		charts["dates"] = append(charts["dates"], chart.Date.Format("2 Jan"))
		charts["counts"] = append(charts["counts"], chart.Count)
	}

	result, _ := json.Marshal(charts)

	w.Header().Set("Content-Type", "application/json")
	w.Write(result)
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
// Copyright © 2015 DisposaBoy <disposaboy@dby.me>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the “Software”), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package main

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

func main() {
	// output GOROOT pkgs
	for _, p := range importPaths(runtime.GOROOT()) {
		fmt.Println(p)
	}
	fmt.Println("unsafe")

	// output GOPATH pkgs
	for _, p := range importPaths(os.Getenv("GOPATH")) {
		fmt.Println(p)
	}
}

func importPaths(envs ...string) (paths []string) {
	roots := map[string]bool{}
	for _, env := range envs {
		for _, root := range filepath.SplitList(env) {
			if root != "" {
				roots[root] = true
			}
		}
	}

	seen := map[string]bool{}
	pfx := strings.HasPrefix
	sfx := strings.HasSuffix
	osArchSfx := runtime.GOOS + "_" + runtime.GOARCH
	for root, _ := range roots {
		root = resolveSymlink(filepath.Join(root, "pkg", osArchSfx))
		walkF := func(p string, info os.FileInfo, err error) error {
			if err != nil || info.IsDir() {
				return nil
			}

			p, e := filepath.Rel(root, p)
			if e != nil || !sfx(p, ".a") {
				return nil
			}

			p = p[:len(p)-2]
			if pfx(p, ".") || pfx(p, "_") || sfx(p, "_test") {
				return nil
			}

			p = path.Clean(filepath.ToSlash(p))
			if !seen[p] {
				seen[p] = true
				paths = append(paths, p)
			}

			return nil
		}
		filepath.Walk(root, walkF)
	}
	return paths
}

func resolveSymlink(path string) string {
	resolvePath, err := filepath.EvalSymlinks(path)
	if err != nil {
		return path
	}
	return resolvePath
}
<commit_msg>Skip vendor (#9)<commit_after>// The MIT License (MIT)
// Copyright © 2015 DisposaBoy <disposaboy@dby.me>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the “Software”), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package main

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

func main() {
	// output GOROOT pkgs
	for _, p := range importPaths(runtime.GOROOT()) {
		fmt.Println(p)
	}
	fmt.Println("unsafe")

	// output GOPATH pkgs
	for _, p := range importPaths(os.Getenv("GOPATH")) {
		fmt.Println(p)
	}
}

func importPaths(envs ...string) (paths []string) {
	roots := map[string]bool{}
	for _, env := range envs {
		for _, root := range filepath.SplitList(env) {
			if root != "" {
				roots[root] = true
			}
		}
	}

	seen := map[string]bool{}
	pfx := strings.HasPrefix
	sfx := strings.HasSuffix
	osArchSfx := runtime.GOOS + "_" + runtime.GOARCH
	for root, _ := range roots {
		root = resolveSymlink(filepath.Join(root, "pkg", osArchSfx))
		walkF := func(p string, info os.FileInfo, err error) error {
			if err != nil || info.IsDir() {
				return nil
			}

			p, e := filepath.Rel(root, p)
			if e != nil || !sfx(p, ".a") {
				return nil
			}

			p = p[:len(p)-2]
			if pfx(p, ".") || pfx(p, "_") || sfx(p, "_test") {
				return nil
			}

			p = path.Clean(filepath.ToSlash(p))
			if !seen[p] {
				if idx := strings.Index(p, "vendor/"); idx == 0 ||
					(idx > 0 && p[idx-1] == '/') {
					return nil
				}
				seen[p] = true
				paths = append(paths, p)
			}

			return nil
		}
		filepath.Walk(root, walkF)
	}
	return paths
}

func resolveSymlink(path string) string {
	resolvePath, err := filepath.EvalSymlinks(path)
	if err != nil {
		return path
	}
	return resolvePath
}
<|endoftext|>


<commit_before>// Benchmark parallel downloading and combination of some pdf files.
package main

import (
  "flag"
  "fmt"
  "time"
  "net/http"
  "io/ioutil"
  "launchpad.net/goamz/aws"
  "launchpad.net/goamz/s3"
)

var (
  verbose bool
  bucketName = "pa-hrsuite-production"
  basedir = "/tmp/"
  keybase = "606/docs/"
  doclist = []string{"100001.pdf","100009.pdf","100030.pdf","100035.pdf",
                     "100035.pdf","100037.pdf","100038.pdf","100093.pdf",}
)

func init(){
  flag.BoolVar(&verbose, "v", false, "Display extra data")
}

func getFile(bucket *s3.Bucket, docname string, c chan int) {
  s3key := keybase + docname
  data, err := bucket.Get(s3key)
  if err != nil { panic(err) }

  path := "/tmp/"+docname
  err = ioutil.WriteFile(path, data, 0644)
  if err != nil { panic(err) }
  c <- len(data)

}

func connect() *s3.Bucket {
  auth, err := aws.EnvAuth()
  if err != nil { panic(err) }
  s := s3.New(auth, aws.USEast)
  return s.Bucket(bucketName)
}

func printSummary(start time.Time, bytes int){
  elapsed := time.Since(start)
  seconds := elapsed.Seconds()
  mbps := float64(bytes) / 1024 / 1024 / seconds
  fmt.Printf("got %d bytes over %d files in %f secs (%f MB/s)\n",
             bytes, len(doclist), seconds, mbps)
}

func combine() {

  flag.Parse()
  start := time.Now()
  bucket := connect()
  c := make(chan int)
  for _,doc := range doclist{
    go getFile(bucket,doc,c)
  }

  totalBytes := 0
  for _,doc := range doclist{
    recieved := <-c
    if verbose{
      fmt.Printf("%s was %d bytes\n", doc,recieved)
    }
    totalBytes += recieved
  }
  printSummary(start,totalBytes)

}

func handle_req(w http.ResponseWriter, r *http.Request) {
  fmt.Printf("got conn\n")
  str := fmt.Sprintf("
Time is %s
", time.Now())
  fmt.Fprintf(w, str)
  fmt.Println("wrote",str)
}
// Exists only to prevent calls to favicon when testing the browser
func noopConn(w http.ResponseWriter, r *http.Request) {}

func main() {
  http.HandleFunc("/favicon.ico", noopConn)
  http.HandleFunc("/", handle_req)
  http.ListenAndServe(":8080", nil)
}
<commit_msg>example of deferring processing<commit_after>// Benchmark parallel downloading and combination of some pdf files.
package main

import (
  "flag"
  "fmt"
  "time"
  "net/http"
  "io/ioutil"
  "launchpad.net/goamz/aws"
  "launchpad.net/goamz/s3"
)

var (
  verbose bool
  bucketName = "pa-hrsuite-production"
  basedir = "/tmp/"
  keybase = "606/docs/"
  doclist = []string{"100001.pdf","100009.pdf","100030.pdf","100035.pdf",
                     "100035.pdf","100037.pdf","100038.pdf","100093.pdf",}
)

func init(){
  flag.BoolVar(&verbose, "v", false, "Display extra data")
}

func getFile(bucket *s3.Bucket, docname string, c chan int) {
  s3key := keybase + docname
  data, err := bucket.Get(s3key)
  if err != nil { panic(err) }

  path := "/tmp/"+docname
  err = ioutil.WriteFile(path, data, 0644)
  if err != nil { panic(err) }
  c <- len(data)

}

func connect() *s3.Bucket {
  auth, err := aws.EnvAuth()
  if err != nil { panic(err) }
  s := s3.New(auth, aws.USEast)
  return s.Bucket(bucketName)
}

func printSummary(start time.Time, bytes int){
  elapsed := time.Since(start)
  seconds := elapsed.Seconds()
  mbps := float64(bytes) / 1024 / 1024 / seconds
  fmt.Printf("got %d bytes over %d files in %f secs (%f MB/s)\n",
             bytes, len(doclist), seconds, mbps)
}

func combine() {

  flag.Parse()
  start := time.Now()
  bucket := connect()
  c := make(chan int)
  for _,doc := range doclist{
    go getFile(bucket,doc,c)
  }

  totalBytes := 0
  for _,doc := range doclist{
    recieved := <-c
    if verbose{
      fmt.Printf("%s was %d bytes\n", doc,recieved)
    }
    totalBytes += recieved
  }
  printSummary(start,totalBytes)

}

// This immediately writes a response to the calling channel, then does some
// slow work (e.g. combining files). HTTP hangs up with the initial response
// even through the goroutine continues to 'work'
func doSomethingInBackground(c chan string) {
  fmt.Println("starting goroutine")
  str := "From goroutine: Time is " + time.Now().String()
  c <- str
  fmt.Println("goroutine sent message, starting slow work")
  time.Sleep(5 * time.Second)
  fmt.Println("done with goroutine")
}

func handle_req(w http.ResponseWriter, r *http.Request) {
  fmt.Printf("got conn\n")
  c := make(chan string)
  go doSomethingInBackground(c)
  response := <-c
  str := fmt.Sprintf("
%s
", response)
  fmt.Fprintf(w, str)
  fmt.Println("wrote",str)
}
// Exists only to prevent calls to favicon when testing the browser
func noopConn(w http.ResponseWriter, r *http.Request) {}

func main() {
  http.HandleFunc("/favicon.ico", noopConn)
  http.HandleFunc("/", handle_req)
  http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package core

import (
	"github.com/tinzenite/channel"
	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

/*
CreateTinzenite makes a directory a new Tinzenite directory. Will return error
if already so.
*/
func CreateTinzenite(dirname, dirpath, peername, username, password string) (*Tinzenite, error) {
	if shared.IsTinzenite(dirpath) {
		return nil, shared.ErrIsTinzenite
	}
	// flag whether we have to clean up after us
	var failed bool
	// make .tinzenite
	err := shared.MakeDotTinzenite(dirpath)
	if err != nil {
		return nil, err
	}
	// if failed was set --> clean up by removing everything
	defer func() {
		if failed {
			shared.RemoveDotTinzenite(dirpath)
		}
	}()
	// get auth data
	auth, err := createAuthentication(dirpath, dirname, username, password)
	if err != nil {
		failed = true
		return nil, err
	}
	// Build
	tinzenite := &Tinzenite{
		Path: dirpath,
		auth: auth}
	// prepare chaninterface
	tinzenite.cInterface = createChannelInterface(tinzenite)
	// build channel
	channel, err := channel.Create(peername, nil, tinzenite.cInterface)
	if err != nil {
		failed = true
		return nil, err
	}
	tinzenite.channel = channel
	// build self peer
	address, err := channel.Address()
	if err != nil {
		failed = true
		return nil, err
	}
	peer, err := shared.CreatePeer(peername, address)
	if err != nil {
		failed = true
		return nil, err
	}
	tinzenite.selfpeer = peer
	tinzenite.allPeers = []*shared.Peer{peer}
	// build model (can block for long!)
	m, err := model.Create(dirpath, peer.Identification)
	if err != nil {
		failed = true
		return nil, err
	}
	tinzenite.model = m
	// store initial copy
	err = tinzenite.Store()
	if err != nil {
		failed = true
		return nil, err
	}
	// save that this directory is now a tinzenite dir
	err = shared.WriteDirectoryList(tinzenite.Path)
	if err != nil {
		failed = true
		return nil, err
	}
	tinzenite.initialize()
	return tinzenite, nil
}

/*
LoadTinzenite will try to load the given directory path as a Tinzenite directory.
If not one it won't work: use CreateTinzenite to create a new peer.
*/
func LoadTinzenite(dirpath, password string) (*Tinzenite, error) {
	if !shared.IsTinzenite(dirpath) {
		return nil, shared.ErrNotTinzenite
	}
	t := &Tinzenite{Path: dirpath}
	// load auth
	auth, err := loadAuthentication(dirpath, password)
	if err != nil {
		return nil, err
	}
	t.auth = auth
	// load model
	model, err := model.Load(dirpath)
	if err != nil {
		return nil, err
	}
	t.model = model
	// load peer list (don't use t.applyPeers – channel not running yet!)
	peers, err := shared.LoadPeers(dirpath)
	if err != nil {
		return nil, err
	}
	t.allPeers = peers
	// load tox dump
	selfToxDump, err := shared.LoadToxDump(dirpath)
	if err != nil {
		return nil, err
	}
	t.selfpeer = selfToxDump.SelfPeer
	// prepare chaninterface
	t.cInterface = createChannelInterface(t)
	// prepare channel
	channel, err := channel.Create(t.selfpeer.Name, selfToxDump.ToxData, t.cInterface)
	if err != nil {
		return nil, err
	}
	t.channel = channel
	t.initialize()
	// empty temp folder to remove orphaned files (ignore error because we don't care if it works)
	err = shared.RemoveDirContents(t.Path + "/" + shared.TINZENITEDIR + "/" + shared.TEMPDIR)
	return t, nil
}
<commit_msg>ups, small cleanup<commit_after>package core

import (
	"github.com/tinzenite/channel"
	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

/*
CreateTinzenite makes a directory a new Tinzenite directory. Will return error
if already so.
*/
func CreateTinzenite(dirname, dirpath, peername, username, password string) (*Tinzenite, error) {
	if shared.IsTinzenite(dirpath) {
		return nil, shared.ErrIsTinzenite
	}
	// flag whether we have to clean up after us
	var failed bool
	// make .tinzenite
	err := shared.MakeDotTinzenite(dirpath)
	if err != nil {
		return nil, err
	}
	// if failed was set --> clean up by removing everything
	defer func() {
		if failed {
			shared.RemoveDotTinzenite(dirpath)
		}
	}()
	// get auth data
	auth, err := createAuthentication(dirpath, dirname, username, password)
	if err != nil {
		failed = true
		return nil, err
	}
	// Build
	tinzenite := &Tinzenite{
		Path: dirpath,
		auth: auth}
	// prepare chaninterface
	tinzenite.cInterface = createChannelInterface(tinzenite)
	// build channel
	channel, err := channel.Create(peername, nil, tinzenite.cInterface)
	if err != nil {
		failed = true
		return nil, err
	}
	tinzenite.channel = channel
	// build self peer
	address, err := channel.Address()
	if err != nil {
		failed = true
		return nil, err
	}
	peer, err := shared.CreatePeer(peername, address)
	if err != nil {
		failed = true
		return nil, err
	}
	tinzenite.selfpeer = peer
	tinzenite.allPeers = []*shared.Peer{peer}
	// build model (can block for long!)
	m, err := model.Create(dirpath, peer.Identification)
	if err != nil {
		failed = true
		return nil, err
	}
	tinzenite.model = m
	// store initial copy
	err = tinzenite.Store()
	if err != nil {
		failed = true
		return nil, err
	}
	// save that this directory is now a tinzenite dir
	err = shared.WriteDirectoryList(tinzenite.Path)
	if err != nil {
		failed = true
		return nil, err
	}
	tinzenite.initialize()
	return tinzenite, nil
}

/*
LoadTinzenite will try to load the given directory path as a Tinzenite directory.
If not one it won't work: use CreateTinzenite to create a new peer.
*/
func LoadTinzenite(dirpath, password string) (*Tinzenite, error) {
	if !shared.IsTinzenite(dirpath) {
		return nil, shared.ErrNotTinzenite
	}
	t := &Tinzenite{Path: dirpath}
	// load auth
	auth, err := loadAuthentication(dirpath, password)
	if err != nil {
		return nil, err
	}
	t.auth = auth
	// load model
	model, err := model.Load(dirpath)
	if err != nil {
		return nil, err
	}
	t.model = model
	// load peer list (don't use t.applyPeers – channel not running yet!)
	peers, err := shared.LoadPeers(dirpath)
	if err != nil {
		return nil, err
	}
	t.allPeers = peers
	// load tox dump
	selfToxDump, err := shared.LoadToxDump(dirpath)
	if err != nil {
		return nil, err
	}
	t.selfpeer = selfToxDump.SelfPeer
	// prepare chaninterface
	t.cInterface = createChannelInterface(t)
	// prepare channel
	channel, err := channel.Create(t.selfpeer.Name, selfToxDump.ToxData, t.cInterface)
	if err != nil {
		return nil, err
	}
	t.channel = channel
	t.initialize()
	// empty temp folder to remove orphaned files (ignore error because we don't care if it works)
	_ = shared.RemoveDirContents(t.Path + "/" + shared.TINZENITEDIR + "/" + shared.TEMPDIR)
	return t, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/dimfeld/httptreemux"
	"github.com/sjug/am-go/server"
)

func userTierHandler(w http.ResponseWriter, r *http.Request, ps map[string]string) {
	// TODO userTierHandler
	fmt.Fprintf(w, "hello, %s!\n", ps["num"])
}

func main() {
	const port = ":8080"
	router := httptreemux.New()
	server.InitDetails(router)
	router.GET("/user/tier/:num", userTierHandler)

	log.Println("Listening on port", port)
	log.Fatal(http.ListenAndServe(port, router))
}
<commit_msg>Second file for other operation<commit_after>package main

import (
	"log"
	"net/http"

	"github.com/dimfeld/httptreemux"
	"github.com/sjug/am-go/server"
)

func main() {
	const port = ":8080"
	router := httptreemux.New()
	server.InitDetails(router)
	//server.InitTier(router)

	log.Println("Listening on port", port)
	log.Fatal(http.ListenAndServe(port, router))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/graylog-labs/leeroy/jenkins"
)

const (
	VERSION        = "v1.1.1"
	DEFAULTCONTEXT = "janky"
)

var (
	certFile   string
	keyFile    string
	port       string
	configFile string
	debug      bool
	version    bool

	config Config
)

type Config struct {
	Jenkins      jenkins.Client `json:"jenkins"`
	BuildCommits string         `json:"build_commits"`
	GHToken      string         `json:"github_token"`
	GHUser       string         `json:"github_user"`
	Builds       []Build        `json:"builds"`
	PushBuilds   []Build        `json:"push_builds"`
	User         string         `json:"user"`
	Pass         string         `json:"pass"`
}

type Build struct {
	Repo         string `json:"github_repo"`
	Ref          string `json:"ref"`
	Job          string `json:"jenkins_job_name"`
	Context      string `json:"context"`
	Custom       bool   `json:"custom"`
	HandleIssues bool   `json:"handle_issues"`
}

func init() {
	// parse flags
	flag.BoolVar(&version, "version", false, "print version and exit")
	flag.BoolVar(&version, "v", false, "print version and exit (shorthand)")
	flag.BoolVar(&debug, "d", false, "run in debug mode")
	flag.StringVar(&certFile, "cert", "", "path to ssl certificate")
	flag.StringVar(&keyFile, "key", "", "path to ssl key")
	flag.StringVar(&port, "port", "80", "port to use")
	flag.StringVar(&configFile, "config", "/etc/leeroy/config.json", "path to config file")
	flag.Parse()
}

func main() {
	// set log level
	if debug {
		log.SetLevel(log.DebugLevel)
	}

	if version {
		fmt.Println(VERSION)
		return
	}

	// read the config file
	if _, err := os.Stat(configFile); os.IsNotExist(err) {
		log.Errorf("config file does not exist: %s", configFile)
		return
	}
	c, err := ioutil.ReadFile(configFile)
	if err != nil {
		log.Errorf("could not read config file: %v", err)
		return
	}
	if err := json.Unmarshal(c, &config); err != nil {
		log.Errorf("error parsing config file as json: %v", err)
		return
	}

	// create mux server
	mux := http.NewServeMux()

	// ping endpoint
	mux.HandleFunc("/ping", pingHandler)

	// jenkins notification endpoint
	mux.HandleFunc("/notification/jenkins", jenkinsHandler)

	// github webhooks endpoint
	mux.HandleFunc("/notification/github", githubHandler)

	// retry build endpoint
	mux.HandleFunc("/build/retry", customBuildHandler)

	// custom build endpoint
	mux.HandleFunc("/build/custom", customBuildHandler)

	// cron endpoint to reschedule bulk jobs
	mux.HandleFunc("/build/cron", cronBuildHandler)

	// set up the server
	server := &http.Server{
		Addr:    ":" + port,
		Handler: mux,
	}

	log.Printf("Starting server on port %q", port)
	if certFile != "" && keyFile != "" {
		log.Fatal(server.ListenAndServeTLS(certFile, keyFile))
	} else {
		log.Fatal(server.ListenAndServe())
	}
}
<commit_msg>Bump version to 1.1.2<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/graylog-labs/leeroy/jenkins"
)

const (
	VERSION        = "v1.1.2"
	DEFAULTCONTEXT = "janky"
)

var (
	certFile   string
	keyFile    string
	port       string
	configFile string
	debug      bool
	version    bool

	config Config
)

type Config struct {
	Jenkins      jenkins.Client `json:"jenkins"`
	BuildCommits string         `json:"build_commits"`
	GHToken      string         `json:"github_token"`
	GHUser       string         `json:"github_user"`
	Builds       []Build        `json:"builds"`
	PushBuilds   []Build        `json:"push_builds"`
	User         string         `json:"user"`
	Pass         string         `json:"pass"`
}

type Build struct {
	Repo         string `json:"github_repo"`
	Ref          string `json:"ref"`
	Job          string `json:"jenkins_job_name"`
	Context      string `json:"context"`
	Custom       bool   `json:"custom"`
	HandleIssues bool   `json:"handle_issues"`
}

func init() {
	// parse flags
	flag.BoolVar(&version, "version", false, "print version and exit")
	flag.BoolVar(&version, "v", false, "print version and exit (shorthand)")
	flag.BoolVar(&debug, "d", false, "run in debug mode")
	flag.StringVar(&certFile, "cert", "", "path to ssl certificate")
	flag.StringVar(&keyFile, "key", "", "path to ssl key")
	flag.StringVar(&port, "port", "80", "port to use")
	flag.StringVar(&configFile, "config", "/etc/leeroy/config.json", "path to config file")
	flag.Parse()
}

func main() {
	// set log level
	if debug {
		log.SetLevel(log.DebugLevel)
	}

	if version {
		fmt.Println(VERSION)
		return
	}

	// read the config file
	if _, err := os.Stat(configFile); os.IsNotExist(err) {
		log.Errorf("config file does not exist: %s", configFile)
		return
	}
	c, err := ioutil.ReadFile(configFile)
	if err != nil {
		log.Errorf("could not read config file: %v", err)
		return
	}
	if err := json.Unmarshal(c, &config); err != nil {
		log.Errorf("error parsing config file as json: %v", err)
		return
	}

	// create mux server
	mux := http.NewServeMux()

	// ping endpoint
	mux.HandleFunc("/ping", pingHandler)

	// jenkins notification endpoint
	mux.HandleFunc("/notification/jenkins", jenkinsHandler)

	// github webhooks endpoint
	mux.HandleFunc("/notification/github", githubHandler)

	// retry build endpoint
	mux.HandleFunc("/build/retry", customBuildHandler)

	// custom build endpoint
	mux.HandleFunc("/build/custom", customBuildHandler)

	// cron endpoint to reschedule bulk jobs
	mux.HandleFunc("/build/cron", cronBuildHandler)

	// set up the server
	server := &http.Server{
		Addr:    ":" + port,
		Handler: mux,
	}

	log.Printf("Starting server on port %q", port)
	if certFile != "" && keyFile != "" {
		log.Fatal(server.ListenAndServeTLS(certFile, keyFile))
	} else {
		log.Fatal(server.ListenAndServe())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"io"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"

	. "agent/types"
	"agent/utils"

	log "github.com/Sirupsen/logrus"
	"github.com/xtaci/kcp-go"
	cli "gopkg.in/urfave/cli.v2"
)

func main() {
	log.SetLevel(log.DebugLevel)

	// to catch all uncaught panic
	defer utils.PrintPanicStack()

	// open profiling
	go http.ListenAndServe("0.0.0.0:6060", nil)
	app := &cli.App{
		Name:    "agent",
		Usage:   "a gateway for games with stream multiplexing",
		Version: "2.0",
		Flags: []cli.Flag{
			&cli.StringFlag{
				Name:  "listen",
				Value: ":8888",
				Usage: "listening address:port",
			},
			&cli.StringSliceFlag{
				Name:  "etcd-hosts",
				Value: cli.NewStringSlice("http://127.0.0.1:2379"),
				Usage: "etcd hosts",
			},
			&cli.StringFlag{
				Name:  "etcd-root",
				Value: "/backends",
				Usage: "etcd root path",
			},
			&cli.StringSliceFlag{
				Name:  "services",
				Value: cli.NewStringSlice("snowflake-10000", "game-10000"),
				Usage: "auto-discovering services",
			},
			&cli.DurationFlag{
				Name:  "read-deadline",
				Value: 15 * time.Second,
				Usage: "per connection read timeout",
			},
			&cli.IntFlag{
				Name:  "txqueuelen",
				Value: 128,
				Usage: "per connection output message queue, packet will be dropped if exceeds",
			},
			&cli.IntFlag{
				Name:  "sockbuf",
				Value: 32767,
				Usage: "per connection tcp socket buffer",
			},
			&cli.IntFlag{
				Name:  "udp-sockbuf",
				Value: 4194304,
				Usage: "UDP listener socket buffer",
			},
			&cli.IntFlag{
				Name:  "udp-sndwnd",
				Value: 32,
				Usage: "per connection UDP send window",
			},
			&cli.IntFlag{
				Name:  "udp-rcvwnd",
				Value: 32,
				Usage: "per connection UDP recv window",
			},
			&cli.IntFlag{
				Name:  "udp-mtu",
				Value: 1280,
				Usage: "MTU of UDP packets, without IP(20) + UDP(8)",
			},
			&cli.IntFlag{
				Name:  "dscp",
				Value: 46,
				Usage: "set DSCP(6bit)",
			},
			&cli.IntFlag{
				Name:  "nodelay",
				Value: 1,
				Usage: "ikcp_nodelay()",
			},
			&cli.IntFlag{
				Name:  "interval",
				Value: 20,
				Usage: "ikcp_nodelay()",
			},
			&cli.IntFlag{
				Name:  "resend",
				Value: 1,
				Usage: "ikcp_nodelay()",
			},
			&cli.IntFlag{
				Name:  "nc",
				Value: 1,
				Usage: "ikcp_nodelay()",
			},
			&cli.IntFlag{
				Name:  "rpm-limit",
				Value: 200,
				Usage: "per connection rpm limit",
			},
		},
		Action: func(c *cli.Context) error {
			log.Println("listen:", c.String("listen"))
			log.Println("etcd-hosts:", c.StringSlice("etcd-hosts"))
			log.Println("etcd-root:", c.String("etcd-root"))
			log.Println("services:", c.StringSlice("services"))
			log.Println("read-deadline:", c.Duration("read-deadline"))
			log.Println("txqueuelen:", c.Int("txqueuelen"))
			log.Println("sockbuf:", c.Int("sockbuf"))
			log.Println("udp-sockbuf:", c.Int("udp-sockbuf"))
			log.Println("udp-sndwnd:", c.Int("udp-sndwnd"))
			log.Println("udp-rcvwnd:", c.Int("udp-rcvwnd"))
			log.Println("udp-mtu:", c.Int("udp-mtu"))
			log.Println("dscp:", c.Int("dscp"))
			log.Println("rpm-limit:", c.Int("rpm-limit"))
			log.Println("nodelay parameters:", c.Int("nodelay"), c.Int("interval"), c.Int("resend"), c.Int("nc"))

			//setup net param
			listen := c.String("listen")
			readDeadline := c.Duration("read-deadline")
			sockbuf := c.Int("sockbuf")
			udp_sockbuf := c.Int("udp-sockbuf")
			txqueuelen := c.Int("txqueuelen")
			dscp := c.Int("dscp")
			sndwnd := c.Int("udp-sndwnd")
			rcvwnd := c.Int("udp-rcvwnd")
			mtu := c.Int("udp-mtu")
			nodelay, interval, resend, nc := c.Int("nodelay"), c.Int("interval"), c.Int("resend"), c.Int("nc")

			// init services
			startup(c)
			// init timer
			initTimer(c.Int("rpm-limit"))

			// listeners
			go tcpServer(listen, readDeadline, sockbuf, txqueuelen)
			go udpServer(listen, readDeadline, udp_sockbuf, dscp, sndwnd, rcvwnd, nodelay, interval, resend, nc, mtu, txqueuelen)

			// wait forever
			select {}
		},
	}
	app.Run(os.Args)
}

func tcpServer(addr string, readDeadline time.Duration, sockbuf, txqueuelen int) {
	// resolve address & start listening
	tcpAddr, err := net.ResolveTCPAddr("tcp4", addr)
	checkError(err)

	listener, err := net.ListenTCP("tcp", tcpAddr)
	checkError(err)

	log.Info("listening on:", listener.Addr())

	// loop accepting
	for {
		conn, err := listener.AcceptTCP()
		if err != nil {
			log.Warning("accept failed:", err)
			continue
		}
		// set socket read buffer
		conn.SetReadBuffer(sockbuf)
		// set socket write buffer
		conn.SetWriteBuffer(sockbuf)
		// start a goroutine for every incoming connection for reading
		go handleClient(conn, readDeadline, txqueuelen)
	}
}

func udpServer(addr string, readDeadline time.Duration,
	sockbuf, dscp, sndwnd, rcvwnd,
	nodelay, interval, resend, nc,
	mtu,
	txqueuelen int) {

	l, err := kcp.Listen(addr)
	checkError(err)
	log.Info("udp listening on:", l.Addr())
	lis := l.(*kcp.Listener)

	if err := lis.SetReadBuffer(sockbuf); err != nil {
		log.Println("SetReadBuffer", err)
	}
	if err := lis.SetWriteBuffer(sockbuf); err != nil {
		log.Println("SetWriteBuffer", err)
	}
	if err := lis.SetDSCP(dscp); err != nil {
		log.Println("SetDSCP", err)
	}

	// loop accepting
	for {
		conn, err := lis.AcceptKCP()
		if err != nil {
			log.Warning("accept failed:", err)
			continue
		}
		// set kcp parameters
		conn.SetWindowSize(sndwnd, rcvwnd)
		conn.SetNoDelay(nodelay, interval, resend, nc)
		conn.SetStreamMode(true)
		conn.SetMtu(mtu)

		// start a goroutine for every incoming connection for reading
		go handleClient(conn, readDeadline, txqueuelen)
	}
}

// PIPELINE #1: handleClient
// the goroutine is used for reading incoming PACKETS
// each packet is defined as :
// | 2B size |     DATA       |
//
func handleClient(conn net.Conn, readDeadline time.Duration, txqueuelen int) {
	defer utils.PrintPanicStack()
	// for reading the 2-Byte header
	header := make([]byte, 2)
	// the input channel for agent()
	in := make(chan []byte)
	defer func() {
		close(in) // session will close
	}()

	// create a new session object for the connection
	// and record it's IP address
	var sess Session
	host, port, err := net.SplitHostPort(conn.RemoteAddr().String())
	if err != nil {
		log.Error("cannot get remote address:", err)
		return
	}
	sess.IP = net.ParseIP(host)
	log.Infof("new connection from:%v port:%v", host, port)

	// session die signal, will be triggered by agent()
	sess.Die = make(chan struct{})

	// create a write buffer
	out := new_buffer(conn, sess.Die, txqueuelen)
	go out.start()

	// start agent for PACKET processing
	wg.Add(1)
	go agent(&sess, in, out)

	// read loop
	for {
		// solve dead link problem:
		// physical disconnection without any communcation between client and server
		// will cause the read to block FOREVER, so a timeout is a rescue.
		conn.SetReadDeadline(time.Now().Add(readDeadline))

		// read 2B header
		n, err := io.ReadFull(conn, header)
		if err != nil {
			log.Warningf("read header failed, ip:%v reason:%v size:%v", sess.IP, err, n)
			return
		}
		size := binary.BigEndian.Uint16(header)

		// alloc a byte slice of the size defined in the header for reading data
		payload := make([]byte, size)
		n, err = io.ReadFull(conn, payload)
		if err != nil {
			log.Warningf("read payload failed, ip:%v reason:%v size:%v", sess.IP, err, n)
			return
		}

		// deliver the data to the input queue of agent()
		select {
		case in <- payload: // payload queued
		case <-sess.Die:
			log.Warningf("connection closed by logic, flag:%v ip:%v", sess.Flag, sess.IP)
			return
		}
	}
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
		os.Exit(-1)
	}
}
<commit_msg>put all parameters in Config<commit_after>package main

import (
	"encoding/binary"
	"io"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"

	. "agent/types"
	"agent/utils"

	log "github.com/Sirupsen/logrus"
	"github.com/xtaci/kcp-go"
	cli "gopkg.in/urfave/cli.v2"
)

type Config struct {
	listen                        string
	readDeadline                  time.Duration
	sockbuf                       int
	udp_sockbuf                   int
	txqueuelen                    int
	dscp                          int
	sndwnd                        int
	rcvwnd                        int
	mtu                           int
	nodelay, interval, resend, nc int
}

func main() {
	log.SetLevel(log.DebugLevel)

	// to catch all uncaught panic
	defer utils.PrintPanicStack()

	// open profiling
	go http.ListenAndServe("0.0.0.0:6060", nil)
	app := &cli.App{
		Name:    "agent",
		Usage:   "a gateway for games with stream multiplexing",
		Version: "2.0",
		Flags: []cli.Flag{
			&cli.StringFlag{
				Name:  "listen",
				Value: ":8888",
				Usage: "listening address:port",
			},
			&cli.StringSliceFlag{
				Name:  "etcd-hosts",
				Value: cli.NewStringSlice("http://127.0.0.1:2379"),
				Usage: "etcd hosts",
			},
			&cli.StringFlag{
				Name:  "etcd-root",
				Value: "/backends",
				Usage: "etcd root path",
			},
			&cli.StringSliceFlag{
				Name:  "services",
				Value: cli.NewStringSlice("snowflake-10000", "game-10000"),
				Usage: "auto-discovering services",
			},
			&cli.DurationFlag{
				Name:  "read-deadline",
				Value: 15 * time.Second,
				Usage: "per connection read timeout",
			},
			&cli.IntFlag{
				Name:  "txqueuelen",
				Value: 128,
				Usage: "per connection output message queue, packet will be dropped if exceeds",
			},
			&cli.IntFlag{
				Name:  "sockbuf",
				Value: 32767,
				Usage: "per connection tcp socket buffer",
			},
			&cli.IntFlag{
				Name:  "udp-sockbuf",
				Value: 4194304,
				Usage: "UDP listener socket buffer",
			},
			&cli.IntFlag{
				Name:  "udp-sndwnd",
				Value: 32,
				Usage: "per connection UDP send window",
			},
			&cli.IntFlag{
				Name:  "udp-rcvwnd",
				Value: 32,
				Usage: "per connection UDP recv window",
			},
			&cli.IntFlag{
				Name:  "udp-mtu",
				Value: 1280,
				Usage: "MTU of UDP packets, without IP(20) + UDP(8)",
			},
			&cli.IntFlag{
				Name:  "dscp",
				Value: 46,
				Usage: "set DSCP(6bit)",
			},
			&cli.IntFlag{
				Name:  "nodelay",
				Value: 1,
				Usage: "ikcp_nodelay()",
			},
			&cli.IntFlag{
				Name:  "interval",
				Value: 20,
				Usage: "ikcp_nodelay()",
			},
			&cli.IntFlag{
				Name:  "resend",
				Value: 1,
				Usage: "ikcp_nodelay()",
			},
			&cli.IntFlag{
				Name:  "nc",
				Value: 1,
				Usage: "ikcp_nodelay()",
			},
			&cli.IntFlag{
				Name:  "rpm-limit",
				Value: 200,
				Usage: "per connection rpm limit",
			},
		},
		Action: func(c *cli.Context) error {
			log.Println("listen:", c.String("listen"))
			log.Println("etcd-hosts:", c.StringSlice("etcd-hosts"))
			log.Println("etcd-root:", c.String("etcd-root"))
			log.Println("services:", c.StringSlice("services"))
			log.Println("read-deadline:", c.Duration("read-deadline"))
			log.Println("txqueuelen:", c.Int("txqueuelen"))
			log.Println("sockbuf:", c.Int("sockbuf"))
			log.Println("udp-sockbuf:", c.Int("udp-sockbuf"))
			log.Println("udp-sndwnd:", c.Int("udp-sndwnd"))
			log.Println("udp-rcvwnd:", c.Int("udp-rcvwnd"))
			log.Println("udp-mtu:", c.Int("udp-mtu"))
			log.Println("dscp:", c.Int("dscp"))
			log.Println("rpm-limit:", c.Int("rpm-limit"))
			log.Println("nodelay parameters:", c.Int("nodelay"), c.Int("interval"), c.Int("resend"), c.Int("nc"))

			//setup net param
			config := &Config{
				listen:       c.String("listen"),
				readDeadline: c.Duration("read-deadline"),
				sockbuf:      c.Int("sockbuf"),
				udp_sockbuf:  c.Int("udp-sockbuf"),
				txqueuelen:   c.Int("txqueuelen"),
				dscp:         c.Int("dscp"),
				sndwnd:       c.Int("udp-sndwnd"),
				rcvwnd:       c.Int("udp-rcvwnd"),
				mtu:          c.Int("udp-mtu"),
				nodelay:      c.Int("nodelay"),
				interval:     c.Int("interval"),
				resend:       c.Int("resend"),
				nc:           c.Int("nc"),
			}

			// init services
			startup(c)
			// init timer
			initTimer(c.Int("rpm-limit"))

			// listeners
			go tcpServer(config)
			go udpServer(config)

			// wait forever
			select {}
		},
	}
	app.Run(os.Args)
}

func tcpServer(config *Config) {
	// resolve address & start listening
	tcpAddr, err := net.ResolveTCPAddr("tcp4", config.listen)
	checkError(err)

	listener, err := net.ListenTCP("tcp", tcpAddr)
	checkError(err)

	log.Info("listening on:", listener.Addr())

	// loop accepting
	for {
		conn, err := listener.AcceptTCP()
		if err != nil {
			log.Warning("accept failed:", err)
			continue
		}
		// set socket read buffer
		conn.SetReadBuffer(config.sockbuf)
		// set socket write buffer
		conn.SetWriteBuffer(config.sockbuf)
		// start a goroutine for every incoming connection for reading
		go handleClient(conn, config)
	}
}

func udpServer(config *Config) {
	l, err := kcp.Listen(config.listen)
	checkError(err)
	log.Info("udp listening on:", l.Addr())
	lis := l.(*kcp.Listener)

	if err := lis.SetReadBuffer(config.sockbuf); err != nil {
		log.Println("SetReadBuffer", err)
	}
	if err := lis.SetWriteBuffer(config.sockbuf); err != nil {
		log.Println("SetWriteBuffer", err)
	}
	if err := lis.SetDSCP(config.dscp); err != nil {
		log.Println("SetDSCP", err)
	}

	// loop accepting
	for {
		conn, err := lis.AcceptKCP()
		if err != nil {
			log.Warning("accept failed:", err)
			continue
		}
		// set kcp parameters
		conn.SetWindowSize(config.sndwnd, config.rcvwnd)
		conn.SetNoDelay(config.nodelay, config.interval, config.resend, config.nc)
		conn.SetStreamMode(true)
		conn.SetMtu(config.mtu)

		// start a goroutine for every incoming connection for reading
		go handleClient(conn, config)
	}
}

// PIPELINE #1: handleClient
// the goroutine is used for reading incoming PACKETS
// each packet is defined as :
// | 2B size |     DATA       |
//
func handleClient(conn net.Conn, config *Config) {
	defer utils.PrintPanicStack()
	// for reading the 2-Byte header
	header := make([]byte, 2)
	// the input channel for agent()
	in := make(chan []byte)
	defer func() {
		close(in) // session will close
	}()

	// create a new session object for the connection
	// and record it's IP address
	var sess Session
	host, port, err := net.SplitHostPort(conn.RemoteAddr().String())
	if err != nil {
		log.Error("cannot get remote address:", err)
		return
	}
	sess.IP = net.ParseIP(host)
	log.Infof("new connection from:%v port:%v", host, port)

	// session die signal, will be triggered by agent()
	sess.Die = make(chan struct{})

	// create a write buffer
	out := new_buffer(conn, sess.Die, config.txqueuelen)
	go out.start()

	// start agent for PACKET processing
	wg.Add(1)
	go agent(&sess, in, out)

	// read loop
	for {
		// solve dead link problem:
		// physical disconnection without any communcation between client and server
		// will cause the read to block FOREVER, so a timeout is a rescue.
		conn.SetReadDeadline(time.Now().Add(config.readDeadline))

		// read 2B header
		n, err := io.ReadFull(conn, header)
		if err != nil {
			log.Warningf("read header failed, ip:%v reason:%v size:%v", sess.IP, err, n)
			return
		}
		size := binary.BigEndian.Uint16(header)

		// alloc a byte slice of the size defined in the header for reading data
		payload := make([]byte, size)
		n, err = io.ReadFull(conn, payload)
		if err != nil {
			log.Warningf("read payload failed, ip:%v reason:%v size:%v", sess.IP, err, n)
			return
		}

		// deliver the data to the input queue of agent()
		select {
		case in <- payload: // payload queued
		case <-sess.Die:
			log.Warningf("connection closed by logic, flag:%v ip:%v", sess.Flag, sess.IP)
			return
		}
	}
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
		os.Exit(-1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/evolsnow/robot/conn"
	"golang.org/x/net/websocket"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var messages = make(chan string)

func main() {
	var configFile string
	var debug bool

	flag.StringVar(&configFile, "c", "config.json", "specify config file")
	flag.BoolVar(&debug, "d", false, "debug mode")
	flag.Parse()
	config, err := ParseConfig(configFile)
	if err != nil {
		log.Fatal("a vailid json config file must exist")
	}

	//connect to redis
	redisPort := strconv.Itoa(config.RedisPort)
	redisServer := net.JoinHostPort(config.RedisAddress, redisPort)
	if !conn.Ping(redisServer, config.RedisPassword) {
		log.Fatal("connect to redis server failed")
	}
	conn.Pool = conn.NewPool(redisServer, config.RedisPassword, config.RedisDB)

	//create robot and run
	robot := newRobot(config.RobotToken, config.RobotName, config.WebHookUrl)
	robot.bot.Debug = debug
	go robot.run()

	//run server and web samaritan
	srvPort := strconv.Itoa(config.Port)
	http.HandleFunc("/ajax", ajax)
	http.Handle("/websocket", websocket.Handler(socketHandler))
	log.Fatal(http.ListenAndServeTLS(net.JoinHostPort(config.Server, srvPort), config.Cert, config.CertKey, nil))

}

//used for web samaritan robot
func socketHandler(ws *websocket.Conn) {
	for {
		var in string
		var ret []string
		if err := websocket.Message.Receive(ws, &in); err != nil {
			log.Println(err)
			return
		}
		ret = receive(in)

		for i := range ret {
			websocket.Message.Send(ws, ret[i])
			time.Sleep(time.Second)
		}
		websocket.Message.Send(ws, "")
	}
}

//when webSocket unavailable, fallback to ajax long polling
func ajax(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Origin", "*")
	if r.Method == "GET" {
		w.Write([]byte(<-messages))
	} else {
		body := r.FormValue("text")
		if body != "" {
			go func(string) {
				ret := receive(body)
				for i := range ret {
					messages <- ret[i]
					time.Sleep(time.Second)
				}
				messages <- ""
			}(body)
		}
	}
}

//receive from client
func receive(in string) (ret []string) {
	fmt.Printf("Received: %s\n", in)
	var response string
	var answer = make(chan string)
	sf := func(c rune) bool {
		return c == ',' || c == '，' || c == ';' || c == '。' || c == '.' || c == '？' || c == '?'
	}
	if chinese(in) {
		go func() {
			answer <- tlAI(in)
		}()
		go func() {
			answer <- qinAI(in)
		}()
		response = <-answer
		// Separate into fields with func.
		ret = strings.FieldsFunc(response, sf)

	} else {
		response = mitAI(in)
		ret = strings.FieldsFunc(response, sf)
	}
	return
}
<commit_msg>replace name<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/evolsnow/robot/conn"
	"golang.org/x/net/websocket"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var messages = make(chan string)

func main() {
	var configFile string
	var debug bool

	flag.StringVar(&configFile, "c", "config.json", "specify config file")
	flag.BoolVar(&debug, "d", false, "debug mode")
	flag.Parse()
	config, err := ParseConfig(configFile)
	if err != nil {
		log.Fatal("a vailid json config file must exist")
	}

	//connect to redis
	redisPort := strconv.Itoa(config.RedisPort)
	redisServer := net.JoinHostPort(config.RedisAddress, redisPort)
	if !conn.Ping(redisServer, config.RedisPassword) {
		log.Fatal("connect to redis server failed")
	}
	conn.Pool = conn.NewPool(redisServer, config.RedisPassword, config.RedisDB)

	//create robot and run
	robot := newRobot(config.RobotToken, config.RobotName, config.WebHookUrl)
	robot.bot.Debug = debug
	go robot.run()

	//run server and web samaritan
	srvPort := strconv.Itoa(config.Port)
	http.HandleFunc("/ajax", ajax)
	http.Handle("/websocket", websocket.Handler(socketHandler))
	log.Fatal(http.ListenAndServeTLS(net.JoinHostPort(config.Server, srvPort), config.Cert, config.CertKey, nil))

}

//used for web samaritan robot
func socketHandler(ws *websocket.Conn) {
	for {
		var in string
		var ret []string
		if err := websocket.Message.Receive(ws, &in); err != nil {
			log.Println(err)
			return
		}
		ret = receive(in)

		for i := range ret {
			websocket.Message.Send(ws, ret[i])
			time.Sleep(time.Second)
		}
		websocket.Message.Send(ws, "")
	}
}

//when webSocket unavailable, fallback to ajax long polling
func ajax(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Origin", "*")
	if r.Method == "GET" {
		w.Write([]byte(<-messages))
	} else {
		body := r.FormValue("text")
		if body != "" {
			go func(string) {
				ret := receive(body)
				for i := range ret {
					messages <- ret[i]
					time.Sleep(time.Second)
				}
				messages <- ""
			}(body)
		}
	}
}

//receive from client
func receive(in string) (ret []string) {
	fmt.Printf("Received: %s\n", in)
	var response string
	var answer = make(chan string)
	sf := func(c rune) bool {
		return c == ',' || c == '，' || c == ';' || c == '。' || c == '.' || c == '？' || c == '?'
	}
	if chinese(in) {
		go func() {
			answer <- tlAI(in)
		}()
		go func() {
			answer <- strings.Replace(qinAI(in), "Jarvis", "samaritan", -1)
		}()
		response = <-answer
		// Separate into fields with func.
		ret = strings.FieldsFunc(response, sf)

	} else {
		response = mitAI(in)
		ret = strings.FieldsFunc(response, sf)
	}
	return
}
<|endoftext|>