<commit_before>package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	_ "expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"log/syslog"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"sync"

	pickle "github.com/kisielk/og-rek"
)

var Debug int

var Config = struct {
	Backends []string
	MaxProcs int
	Port     int

	mu          sync.RWMutex
	metricPaths map[string][]string
}{
	MaxProcs:    1,
	Port:        8080,
	metricPaths: make(map[string][]string),
}

var logger multilog

type serverResponse struct {
	server   string
	response []byte
}

func multiGet(servers []string, uri string) []serverResponse {

	ch := make(chan serverResponse)

	if Debug > 0 {
		logger.Logln("querying servers=", servers, "uri=", uri)
	}

	for _, server := range servers {
		go func(server string, ch chan<- serverResponse) {

			u, err := url.Parse(server + uri)
			if err != nil {
				log.Fatal(err)
			}
			req := http.Request{
				URL:    u,
				Header: make(http.Header),
			}

			resp, err := http.DefaultClient.Do(&req)

			if err != nil || resp.StatusCode != 200 {
				logger.Logln("got status code", resp.StatusCode, "while querying", server, "/", uri)
				ch <- serverResponse{server, nil}
			}
			defer resp.Body.Close()

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				ch <- serverResponse{server, nil}
			}

			ch <- serverResponse{server, body}
		}(server, ch)
	}

	var response []serverResponse

	for i := 0; i < len(servers); i++ {
		r := <-ch
		if r.response != nil {
			response = append(response, r)
		}
	}

	return response
}

func findHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		logger.Logln("request: ", req.URL.RequestURI())
	}

	responses := multiGet(Config.Backends, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		logger.Logln("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		return
	}

	// metric -> [server1, ... ]
	paths := make(map[string][]string)

	var metrics []map[interface{}]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				logger.Logln("\n" + hex.Dump(r.response))
			}
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			logger.Logf("bad type for metric:%t from server:%s: req:%s", metric, r.server, req.URL.RequestURI())
			http.Error(w, fmt.Sprintf("bad type for metric: %t", metric), http.StatusInternalServerError)
			return
		}

		for i, m := range marray {
			mm, ok := m.(map[interface{}]interface{})
			if !ok {
				logger.Logf("bad type for metric[%d]:%t from server:%s: req:%s", i, m, r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric[%d]:%t", i, m), http.StatusInternalServerError)
				return
			}
			name, ok := mm["metric_path"].(string)
			if !ok {
				logger.Logf("bad type for metric_path:%t from server:%s: req:%s", mm["metric_path"], r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric_path: %t", mm["metric_path"]), http.StatusInternalServerError)
				return
			}
			p, ok := paths[name]
			if !ok {
				// we haven't seen this name yet
				// add the metric to the list of metrics to return
				metrics = append(metrics, mm)
			}
			// add the server to the list of servers that know about this metric
			p = append(p, r.server)
			paths[name] = p
		}
	}

	// update our cache of which servers have which metrics
	Config.mu.Lock()
	for k, v := range paths {
		Config.metricPaths[k] = v
	}
	Config.mu.Unlock()

	w.Header().Set("Content-Type", "application/pickle")

	pEnc := pickle.NewEncoder(w)
	pEnc.Encode(metrics)
}

func renderHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		logger.Logln("request: ", req.URL.RequestURI())
	}

	req.ParseForm()
	target := req.FormValue("target")

	if target == "" {
		http.Error(w, "empty target", http.StatusBadRequest)
		return
	}

	var serverList []string
	var ok bool

	Config.mu.RLock()
	// lookup the server list for this metric, or use all the servers if it's unknown
	if serverList, ok = Config.metricPaths[target]; !ok {
		serverList = Config.Backends
	}
	Config.mu.RUnlock()

	responses := multiGet(serverList, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		logger.Logln("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		return
	}

	// nothing to merge
	if len(responses) == 1 {
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
	}

	// decode everything
	var decoded [][]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				logger.Logln("\n" + hex.Dump(r.response))
			}
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			err := fmt.Sprintf("bad type for metric:%d from server:%s req:%s", metric, r.server, req.URL.RequestURI())
			logger.Logln(err)
			http.Error(w, err, http.StatusInternalServerError)
			return
		}
		if len(marray) == 0 {
			continue
		}
		decoded = append(decoded, marray)
	}

	if Debug > 2 {
		logger.Logf("request: %s: %v", req.URL.RequestURI(), decoded)
	}

	if len(decoded) == 0 {
		logger.Logf("no decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
		return
	}

	if len(decoded) == 1 {
		logger.Logf("only one decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		// send back whatever data we have
		e := pickle.NewEncoder(w)
		e.Encode(decoded[0])
		return
	}

	if len(decoded[0]) != 1 {
		err := fmt.Sprintf("bad length for decoded[]:%d from req:%s", len(decoded[0]), req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		return
	}

	base, ok := decoded[0][0].(map[interface{}]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for decoded:%t from req:%s", decoded[0][0], req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		return
	}

	values, ok := base["values"].([]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for values:%t from req:%s", base["values"], req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		return
	}

fixValues:
	for i := 0; i < len(values); i++ {
		if _, ok := values[i].(pickle.None); ok {
			// find one in the other values arrays
			for other := 1; other < len(decoded); other++ {
				m, ok := decoded[other][0].(map[interface{}]interface{})
				if !ok {
					logger.Logln(fmt.Sprintf("bad type for decoded[%d][0]: %t", other, decoded[other][0]))
					break fixValues
				}

				ovalues, ok := m["values"].([]interface{})
				if !ok {
					logger.Logf("bad type for ovalues:%t from req:%s (skipping)", m["values"], req.URL.RequestURI())
					break fixValues
				}

				if len(ovalues) != len(values) {
					logger.Logf("unable to merge ovalues: len(values)=%d but len(ovalues)=%d", req.URL.RequestURI(), len(values), len(ovalues))
					logger.Logf("request: %s: %v", req.URL.RequestURI(), decoded)
					break fixValues
				}

				if _, ok := ovalues[i].(pickle.None); !ok {
					values[i] = ovalues[i]
					break
				}
			}
		}
	}

	// the first response is where we've been filling in our data, so we're ok just to serialize it as our response
	w.Header().Set("Content-Type", "application/pickle")
	e := pickle.NewEncoder(w)
	e.Encode(decoded[0])
}

func stripCommentHeader(cfg []byte) []byte {

	// strip out the comment header block that begins with '#' characters
	// as soon as we see a line that starts with something _other_ than '#', we're done

	idx := 0
	for cfg[0] == '#' {
		idx = bytes.Index(cfg, []byte("\n"))
		if idx == -1 || idx+1 == len(cfg) {
			return nil
		}
		cfg = cfg[idx+1:]
	}

	return cfg
}

func main() {

	configFile := flag.String("c", "", "config file (json)")
	port := flag.Int("p", 0, "port to listen on")
	maxprocs := flag.Int("maxprocs", 0, "GOMAXPROCS")
	flag.IntVar(&Debug, "d", 0, "enable debug logging")
	logStdout := flag.Bool("stdout", false, "write logging output also to stdout (default: only syslog)")

	flag.Parse()

	if *configFile == "" {
		log.Fatal("missing config file")
	}

	cfgjs, err := ioutil.ReadFile(*configFile)
	if err != nil {
		log.Fatal("unable to load config file:", err)
	}

	cfgjs = stripCommentHeader(cfgjs)

	if cfgjs == nil {
		log.Fatal("error removing header comment from ", *configFile)
	}

	err = json.Unmarshal(cfgjs, &Config)
	if err != nil {
		log.Fatal("error parsing config file: ", err)
	}

	if len(Config.Backends) == 0 {
		log.Fatal("no Backends loaded -- exiting")
	}

	// command line overrides config file

	if *port != 0 {
		Config.Port = *port
	}

	if *maxprocs != 0 {
		Config.MaxProcs = *maxprocs
	}

	// set up our logging
	slog, err := syslog.New(syslog.LOG_DAEMON, "carbonzipper")
	if err != nil {
		log.Fatal("can't obtain a syslog connection", err)
	}
	logger = append(logger, &sysLogger{w: slog})

	if *logStdout {
		logger = append(logger, &stdoutLogger{log.New(os.Stdout, "", log.LstdFlags)})
	}

	logger.Logln("setting GOMAXPROCS=", Config.MaxProcs)
	runtime.GOMAXPROCS(Config.MaxProcs)

	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", renderHandler)

	portStr := fmt.Sprintf(":%d", Config.Port)
	logger.Logln("listening on", portStr)
	log.Fatal(http.ListenAndServe(portStr, nil))
}

// trivial logging classes

type Logger interface {
	Log(string)
}

type stdoutLogger struct{ logger *log.Logger }

func (l *stdoutLogger) Log(s string) { l.logger.Print(s) }

type sysLogger struct{ w *syslog.Writer }

func (l *sysLogger) Log(s string) { l.w.Info(s) }

type multilog []Logger

func (ml multilog) Logln(a ...interface{}) {
	s := fmt.Sprintln(a...)
	for _, l := range ml {
		l.Log(s)
	}
}

func (ml multilog) Logf(format string, a ...interface{}) {
	s := fmt.Sprintf(format, a...)
	for _, l := range ml {
		l.Log(s)
	}
}
<commit_msg>track errors and request numbers via expvar<commit_after>package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"log/syslog"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"sync"

	pickle "github.com/kisielk/og-rek"
)

var Debug int

var Config = struct {
	Backends []string
	MaxProcs int
	Port     int

	mu          sync.RWMutex
	metricPaths map[string][]string
}{
	MaxProcs:    1,
	Port:        8080,
	metricPaths: make(map[string][]string),
}

var Metrics = struct {
	Requests *expvar.Int
	Errors   *expvar.Int
}{
	Requests: expvar.NewInt("requests"),
	Errors:   expvar.NewInt("errors"),
}

var logger multilog

type serverResponse struct {
	server   string
	response []byte
}

func multiGet(servers []string, uri string) []serverResponse {

	ch := make(chan serverResponse)

	if Debug > 0 {
		logger.Logln("querying servers=", servers, "uri=", uri)
	}

	for _, server := range servers {
		go func(server string, ch chan<- serverResponse) {

			u, err := url.Parse(server + uri)
			if err != nil {
				log.Fatal(err)
			}
			req := http.Request{
				URL:    u,
				Header: make(http.Header),
			}

			resp, err := http.DefaultClient.Do(&req)

			if err != nil || resp.StatusCode != 200 {
				logger.Logln("got status code", resp.StatusCode, "while querying", server, "/", uri)
				ch <- serverResponse{server, nil}
			}
			defer resp.Body.Close()

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				ch <- serverResponse{server, nil}
			}

			ch <- serverResponse{server, body}
		}(server, ch)
	}

	var response []serverResponse

	for i := 0; i < len(servers); i++ {
		r := <-ch
		if r.response != nil {
			response = append(response, r)
		}
	}

	return response
}

func findHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		logger.Logln("request: ", req.URL.RequestURI())
	}

	Metrics.Requests.Add(1)

	responses := multiGet(Config.Backends, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		logger.Logln("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		return
	}

	// metric -> [server1, ... ]
	paths := make(map[string][]string)

	var metrics []map[interface{}]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				logger.Logln("\n" + hex.Dump(r.response))
			}
			Metrics.Errors.Add(1)
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			logger.Logf("bad type for metric:%t from server:%s: req:%s", metric, r.server, req.URL.RequestURI())
			http.Error(w, fmt.Sprintf("bad type for metric: %t", metric), http.StatusInternalServerError)
			Metrics.Errors.Add(1)
			return
		}

		for i, m := range marray {
			mm, ok := m.(map[interface{}]interface{})
			if !ok {
				logger.Logf("bad type for metric[%d]:%t from server:%s: req:%s", i, m, r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric[%d]:%t", i, m), http.StatusInternalServerError)
				Metrics.Errors.Add(1)
				return
			}
			name, ok := mm["metric_path"].(string)
			if !ok {
				logger.Logf("bad type for metric_path:%t from server:%s: req:%s", mm["metric_path"], r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric_path: %t", mm["metric_path"]), http.StatusInternalServerError)
				Metrics.Errors.Add(1)
				return
			}
			p, ok := paths[name]
			if !ok {
				// we haven't seen this name yet
				// add the metric to the list of metrics to return
				metrics = append(metrics, mm)
			}
			// add the server to the list of servers that know about this metric
			p = append(p, r.server)
			paths[name] = p
		}
	}

	// update our cache of which servers have which metrics
	Config.mu.Lock()
	for k, v := range paths {
		Config.metricPaths[k] = v
	}
	Config.mu.Unlock()

	w.Header().Set("Content-Type", "application/pickle")

	pEnc := pickle.NewEncoder(w)
	pEnc.Encode(metrics)
}

func renderHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		logger.Logln("request: ", req.URL.RequestURI())
	}

	Metrics.Requests.Add(1)

	req.ParseForm()
	target := req.FormValue("target")

	if target == "" {
		http.Error(w, "empty target", http.StatusBadRequest)
		return
	}

	var serverList []string
	var ok bool

	Config.mu.RLock()
	// lookup the server list for this metric, or use all the servers if it's unknown
	if serverList, ok = Config.metricPaths[target]; !ok {
		serverList = Config.Backends
	}
	Config.mu.RUnlock()

	responses := multiGet(serverList, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		logger.Logln("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

	// nothing to merge
	if len(responses) == 1 {
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
	}

	// decode everything
	var decoded [][]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				logger.Logln("\n" + hex.Dump(r.response))
			}
			Metrics.Errors.Add(1)
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			err := fmt.Sprintf("bad type for metric:%d from server:%s req:%s", metric, r.server, req.URL.RequestURI())
			logger.Logln(err)
			http.Error(w, err, http.StatusInternalServerError)
			Metrics.Errors.Add(1)
			return
		}
		if len(marray) == 0 {
			continue
		}
		decoded = append(decoded, marray)
	}

	if Debug > 2 {
		logger.Logf("request: %s: %v", req.URL.RequestURI(), decoded)
	}

	if len(decoded) == 0 {
		logger.Logf("no decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
		return
	}

	if len(decoded) == 1 {
		logger.Logf("only one decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		// send back whatever data we have
		e := pickle.NewEncoder(w)
		e.Encode(decoded[0])
		return
	}

	if len(decoded[0]) != 1 {
		err := fmt.Sprintf("bad length for decoded[]:%d from req:%s", len(decoded[0]), req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

	base, ok := decoded[0][0].(map[interface{}]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for decoded:%t from req:%s", decoded[0][0], req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

	values, ok := base["values"].([]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for values:%t from req:%s", base["values"], req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

fixValues:
	for i := 0; i < len(values); i++ {
		if _, ok := values[i].(pickle.None); ok {
			// find one in the other values arrays
			for other := 1; other < len(decoded); other++ {
				m, ok := decoded[other][0].(map[interface{}]interface{})
				if !ok {
					logger.Logln(fmt.Sprintf("bad type for decoded[%d][0]: %t", other, decoded[other][0]))
					Metrics.Errors.Add(1)
					break fixValues
				}

				ovalues, ok := m["values"].([]interface{})
				if !ok {
					logger.Logf("bad type for ovalues:%t from req:%s (skipping)", m["values"], req.URL.RequestURI())
					Metrics.Errors.Add(1)
					break fixValues
				}

				if len(ovalues) != len(values) {
					logger.Logf("unable to merge ovalues: len(values)=%d but len(ovalues)=%d", req.URL.RequestURI(), len(values), len(ovalues))
					logger.Logf("request: %s: %v", req.URL.RequestURI(), decoded)
					Metrics.Errors.Add(1)
					break fixValues
				}

				if _, ok := ovalues[i].(pickle.None); !ok {
					values[i] = ovalues[i]
					break
				}
			}
		}
	}

	// the first response is where we've been filling in our data, so we're ok just to serialize it as our response
	w.Header().Set("Content-Type", "application/pickle")
	e := pickle.NewEncoder(w)
	e.Encode(decoded[0])
}

func stripCommentHeader(cfg []byte) []byte {

	// strip out the comment header block that begins with '#' characters
	// as soon as we see a line that starts with something _other_ than '#', we're done

	idx := 0
	for cfg[0] == '#' {
		idx = bytes.Index(cfg, []byte("\n"))
		if idx == -1 || idx+1 == len(cfg) {
			return nil
		}
		cfg = cfg[idx+1:]
	}

	return cfg
}

func main() {

	configFile := flag.String("c", "", "config file (json)")
	port := flag.Int("p", 0, "port to listen on")
	maxprocs := flag.Int("maxprocs", 0, "GOMAXPROCS")
	flag.IntVar(&Debug, "d", 0, "enable debug logging")
	logStdout := flag.Bool("stdout", false, "write logging output also to stdout (default: only syslog)")

	flag.Parse()

	if *configFile == "" {
		log.Fatal("missing config file")
	}

	cfgjs, err := ioutil.ReadFile(*configFile)
	if err != nil {
		log.Fatal("unable to load config file:", err)
	}

	cfgjs = stripCommentHeader(cfgjs)

	if cfgjs == nil {
		log.Fatal("error removing header comment from ", *configFile)
	}

	err = json.Unmarshal(cfgjs, &Config)
	if err != nil {
		log.Fatal("error parsing config file: ", err)
	}

	if len(Config.Backends) == 0 {
		log.Fatal("no Backends loaded -- exiting")
	}

	// command line overrides config file

	if *port != 0 {
		Config.Port = *port
	}

	if *maxprocs != 0 {
		Config.MaxProcs = *maxprocs
	}

	// set up our logging
	slog, err := syslog.New(syslog.LOG_DAEMON, "carbonzipper")
	if err != nil {
		log.Fatal("can't obtain a syslog connection", err)
	}
	logger = append(logger, &sysLogger{w: slog})

	if *logStdout {
		logger = append(logger, &stdoutLogger{log.New(os.Stdout, "", log.LstdFlags)})
	}

	logger.Logln("setting GOMAXPROCS=", Config.MaxProcs)
	runtime.GOMAXPROCS(Config.MaxProcs)

	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", renderHandler)

	portStr := fmt.Sprintf(":%d", Config.Port)
	logger.Logln("listening on", portStr)
	log.Fatal(http.ListenAndServe(portStr, nil))
}

// trivial logging classes

type Logger interface {
	Log(string)
}

type stdoutLogger struct{ logger *log.Logger }

func (l *stdoutLogger) Log(s string) { l.logger.Print(s) }

type sysLogger struct{ w *syslog.Writer }

func (l *sysLogger) Log(s string) { l.w.Info(s) }

type multilog []Logger

func (ml multilog) Logln(a ...interface{}) {
	s := fmt.Sprintln(a...)
	for _, l := range ml {
		l.Log(s)
	}
}

func (ml multilog) Logf(format string, a ...interface{}) {
	s := fmt.Sprintf(format, a...)
	for _, l := range ml {
		l.Log(s)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
)

func main() {
	var resp EC2Response

	err := json.NewDecoder(os.Stdin).Decode(&resp)
	if err != nil {
		log.Fatal(fmt.Errorf("error reading json string: %s", err))
	}

	var output string
	for _, res := range resp.Reservations {
		if output == "" {
			output = fmt.Sprintf("%30s%14s%11s%16s%18s\n\n",
				"Server",
				"ID",
				"State",
				"Private IP",
				"Public IP",
			)
		}
		for _, inst := range res.Instances {
			output += fmt.Sprintf("%30s%14s%11s%16s%18s\n",
				getInstanceNameFromTags(inst.Tags),
				inst.ID,
				inst.State.Name,
				inst.PrivateIP,
				inst.PublicIP,
			)
		}
	}

	if output != "" {
		fmt.Println(output)
	}
}

//getInstanceNameFromTags ...
func getInstanceNameFromTags(tags []EC2InstanceTag) string {
	if len(tags) > 0 {
		for _, tag := range tags {
			if tag.Key == "Name" {
				return tag.Value
			}
		}
	}
	return "---"
}

//EC2Response ...
type EC2Response struct {
	Reservations []EC2Result `json:"Reservations"`
}

//EC2Result ...
type EC2Result struct {
	Instances []EC2Instance `json:"Instances"`
}

//EC2Instance ...
type EC2Instance struct {
	ID        string           `json:"InstanceId"`
	PrivateIP string           `json:"PrivateIpAddress"`
	PublicIP  string           `json:"PublicIpAddress"`
	Tags      []EC2InstanceTag `json:"Tags"`
	State     EC2InstanceState `json:"State"`
}

//EC2InstanceState ...
type EC2InstanceState struct {
	Name string `json:"Name"`
}

//EC2InstanceTag ...
type EC2InstanceTag struct {
	Key   string `json:"Key"`
	Value string `json:"Value"`
}
<commit_msg>factors in autoscaling group name into what the displayed instance name is<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
)

func main() {
	var resp EC2Response

	err := json.NewDecoder(os.Stdin).Decode(&resp)
	if err != nil {
		log.Fatal(fmt.Errorf("error reading json string: %s", err))
	}

	var output string
	for _, res := range resp.Reservations {
		if output == "" {
			output = fmt.Sprintf("%30s%14s%11s%16s%18s\n\n",
				"Server",
				"ID",
				"State",
				"Private IP",
				"Public IP",
			)
		}
		for _, inst := range res.Instances {
			output += fmt.Sprintf("%30s%14s%11s%16s%18s\n",
				getInstanceNameFromTags(inst.Tags),
				inst.ID,
				inst.State.Name,
				inst.PrivateIP,
				inst.PublicIP,
			)
		}
	}

	if output != "" {
		fmt.Println(output)
	}
}

//getInstanceNameFromTags ...
func getInstanceNameFromTags(tags []EC2InstanceTag) string {
	if len(tags) > 0 {
		for _, tag := range tags {
			switch tag.Key {
			case "Name":
				fallthrough
			case "aws:autoscaling:groupName":
				return tag.Value
			}
		}
	}
	return "---"
}

//EC2Response ...
type EC2Response struct {
	Reservations []EC2Result `json:"Reservations"`
}

//EC2Result ...
type EC2Result struct {
	Instances []EC2Instance `json:"Instances"`
}

//EC2Instance ...
type EC2Instance struct {
	ID        string           `json:"InstanceId"`
	PrivateIP string           `json:"PrivateIpAddress"`
	PublicIP  string           `json:"PublicIpAddress"`
	Tags      []EC2InstanceTag `json:"Tags"`
	State     EC2InstanceState `json:"State"`
}

//EC2InstanceState ...
type EC2InstanceState struct {
	Name string `json:"Name"`
}

//EC2InstanceTag ...
type EC2InstanceTag struct {
	Key   string `json:"Key"`
	Value string `json:"Value"`
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"

	"github.com/rs/cors"
	"gopkg.in/gographics/imagick.v3/imagick"

	_ "net/http/pprof"
)

var (
	appconfig = appConfig{}
	dao       = dbDao{}
)

func init() {
	loadConfig("config/app.json", &appconfig)
	dao.Server = appconfig.DBServer
	dao.Database = appconfig.DBName
	dao.connect()
	imagick.Initialize()
	defer imagick.Terminate()
}

func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("/", webhoookHandler)
	srv := &http.Server{
		Addr:    ":3682",
		Handler: cors.Default().Handler(mux),
	}
	log.Fatal(srv.ListenAndServe())
}
<commit_msg>remove pprof<commit_after>package main

import (
	"log"
	"net/http"

	"github.com/rs/cors"
	"gopkg.in/gographics/imagick.v3/imagick"
)

var (
	appconfig = appConfig{}
	dao       = dbDao{}
)

func init() {
	loadConfig("config/app.json", &appconfig)
	dao.Server = appconfig.DBServer
	dao.Database = appconfig.DBName
	dao.connect()
	imagick.Initialize()
	defer imagick.Terminate()
}

func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("/", webhoookHandler)
	srv := &http.Server{
		Addr:    ":3682",
		Handler: cors.Default().Handler(mux),
	}
	log.Fatal(srv.ListenAndServe())
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/facebookgo/grace/gracehttp"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
)

func main() {
	e := echo.New()

	e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
		Skipper: middleware.DefaultSkipper,
		Format:  "{\"time\":\"${time_rfc3339}\", \"remote_ip\":\"${remote_ip}\", \"host\":\"${host}\", \"method\":\"${method}\", \"uri\":\"${uri}\", \"status\":${status}, latency:${latency}, \"latency_human\":\"${latency_human}\", \"bytes_in\":${bytes_in}, \"bytes_out\":${bytes_out}}\n",
		Output:  os.Stdout,
	}))
	e.Use(middleware.Recover())

	dbh := DBHandler{}
	err := dbh.initDB()
	if err != nil {
		e.Logger.Panic(err)
	}
	e.Logger.Printf("DB handler initiated")

	e.File("/favicon.ico", "images/favicon.ico")

	e.GET("/indicators", dbh.getIndicators)
	e.GET("/health/:indicator/github.com/:owner/:repo", dbh.getHealth)
	e.Logger.Printf("%d routes created: %v", len(e.Routes()), e.Routes())

	e.Server.Addr = ":8080"

	e.Logger.Fatal(gracehttp.Serve(e.Server))
}
<commit_msg>Update routes<commit_after>package main

import (
	"os"

	"github.com/facebookgo/grace/gracehttp"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
)

func main() {
	e := echo.New()

	e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
		Skipper: middleware.DefaultSkipper,
		Format:  "{\"time\":\"${time_rfc3339_nano}\", \"remote_ip\":\"${remote_ip}\", \"host\":\"${host}\", \"method\":\"${method}\", \"uri\":\"${uri}\", \"status\":${status}, latency:${latency}, \"latency_human\":\"${latency_human}\", \"bytes_in\":${bytes_in}, \"bytes_out\":${bytes_out}}\n",
		Output:  os.Stdout,
	}))
	e.Use(middleware.Recover())

	dbh := DBHandler{}
	err := dbh.initDB()
	if err != nil {
		e.Logger.Panic(err)
	}
	e.Logger.Print("DB handler initiated")

	e.File("/favicon.ico", "images/favicon.ico")

	// Instrumentation
	e.GET("/_count", getGoroutinesCount)
	e.GET("/_stack", getStackTraceHandler)

	// Indicators
	e.GET("/indicators", dbh.getIndicators)
	e.GET("/indicators/:indicator", dbh.getIndicator)

	// Repository
	e.GET("/repo/:owner/:repo", dbh.getRepository)

	// Health
	e.GET("/repo/:owner/:repo/health/docs", dbh.getRepositoryHealthDocs)
	e.GET("/repo/:owner/:repo/health/response_times", dbh.getRepositoryHealthResponseTimes)

	// data, err := json.MarshalIndent(e.Routes(), "", "  ")
	// if err != nil {
	// 	e.Logger.Fatal(err)
	// }
	// ioutil.WriteFile("routes.json", data, 0644)
	// e.Logger.Print("routes.json file created")

	e.Logger.Printf("%d routes created", len(e.Routes()))

	e.Server.Addr = ":8080"

	e.Logger.Fatal(gracehttp.Serve(e.Server))
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io/ioutil"
	"os"

	"github.com/mithrandie/csvq/lib/action"
	"github.com/mithrandie/csvq/lib/cmd"
	"github.com/mithrandie/csvq/lib/query"

	"github.com/urfave/cli"
)

var version = "v0.7.5"

func main() {
	cli.AppHelpTemplate = appHHelpTemplate
	cli.CommandHelpTemplate = commandHelpTemplate

	app := cli.NewApp()

	app.Name = "csvq"
	app.Usage = "SQL like query language for csv"
	app.ArgsUsage = "[\"query\"|\"statements\"|argument]"
	app.Version = version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "delimiter, d",
			Usage: "field delimiter. Default is \",\" for csv files, \"\\t\" for tsv files.",
		},
		cli.StringFlag{
			Name:  "encoding, e",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "line-break, l",
			Value: "LF",
			Usage: "line break. one of: CRLF|LF|CR",
		},
		cli.StringFlag{
			Name:  "timezone, z",
			Value: "Local",
			Usage: "default timezone. \"Local\", \"UTC\" or a timezone name(e.g. \"America/Los_Angeles\")",
		},
		cli.StringFlag{
			Name:  "repository, r",
			Usage: "directory path where files are located",
		},
		cli.StringFlag{
			Name:  "source, s",
			Usage: "load query from `FILE`",
		},
		cli.StringFlag{
			Name:  "datetime-format, t",
			Usage: "set datetime format to parse strings",
		},
		cli.BoolFlag{
			Name:  "no-header, n",
			Usage: "import the first line as a record",
		},
		cli.BoolFlag{
			Name:  "without-null, a",
			Usage: "parse empty fields as empty strings",
		},
		cli.StringFlag{
			Name:  "write-encoding, E",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "out, o",
			Usage: "write output to `FILE`",
		},
		cli.StringFlag{
			Name:  "format, f",
			Usage: "output format. one of: CSV|TSV|JSON|TEXT",
		},
		cli.StringFlag{
			Name:  "write-delimiter, D",
			Usage: "field delimiter for CSV",
		},
		cli.BoolFlag{
			Name:  "without-header, N",
			Usage: "when the file format is specified as CSV or TSV, write without the header line",
		},
		cli.BoolFlag{
			Name:  "silent",
			Usage: "Does not output operation logs",
		},
		cli.IntFlag{
			Name:  "cpu, p",
			Usage: "hint for the number of cpu cores to be used. 1 - number of cpu cores",
		},
		cli.BoolFlag{
			Name:  "stats, x",
			Usage: "show execution time and memory statistics",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:      "fields",
			Usage:     "Show fields in a file",
			ArgsUsage: "CSV_FILE_PATH",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "fields")
					return cli.NewExitError("table is not specified", 1)
				}

				table := c.Args().First()

				err := action.ShowFields(table)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
		{
			Name:      "calc",
			Usage:     "Calculate a value from stdin",
			ArgsUsage: "\"expression\"",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "calc")
					return cli.NewExitError("expression is empty", 1)
				}

				expr := c.Args().First()
				err := action.Calc(expr)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
	}

	app.Before = func(c *cli.Context) error {
		return setFlags(c)
	}

	app.Action = func(c *cli.Context) error {
		queryString, err := readQuery(c)
		if err != nil {
			cli.ShowAppHelp(c)
			return cli.NewExitError(err.Error(), 1)
		}

		if err = action.Run(queryString, cmd.GetFlags().Source); err != nil {
			code := 1
			if apperr, ok := err.(query.AppError); ok {
				code = apperr.GetCode()
			} else if ex, ok := err.(*query.Exit); ok {
				code = ex.GetCode()
			}
			return cli.NewExitError(err.Error(), code)
		}

		return nil
	}

	app.Run(os.Args)
}

func readQuery(c *cli.Context) (string, error) {
	var queryString string

	flags := cmd.GetFlags()
	if 0 < len(flags.Source) {
		fp, err := os.Open(flags.Source)
		if err != nil {
			return queryString, err
		}
		defer fp.Close()

		buf, err := ioutil.ReadAll(fp)
		if err != nil {
			return queryString, err
		}
		queryString = string(buf)

	} else {
		if c.NArg() != 1 {
			return queryString, errors.New("query is empty")
		}
		queryString = c.Args().First()
	}

	return queryString, nil
}

func setFlags(c *cli.Context) error {
	if err := cmd.SetDelimiter(c.GlobalString("delimiter")); err != nil {
		return err
	}
	if err := cmd.SetEncoding(c.GlobalString("encoding")); err != nil {
		return err
	}
	if err := cmd.SetLineBreak(c.String("line-break")); err != nil {
		return err
	}
	if err := cmd.SetLocation(c.String("timezone")); err != nil {
		return err
	}
	if err := cmd.SetRepository(c.GlobalString("repository")); err != nil {
		return err
	}
	if err := cmd.SetSource(c.GlobalString("source")); err != nil {
		return err
	}
	cmd.SetDatetimeFormat(c.GlobalString("datetime-format"))
	cmd.SetNoHeader(c.GlobalBool("no-header"))
	cmd.SetWithoutNull(c.GlobalBool("without-null"))

	if err := cmd.SetWriteEncoding(c.GlobalString("write-encoding")); err != nil {
		return err
	}
	if err := cmd.SetOut(c.GlobalString("out")); err != nil {
		return err
	}
	if err := cmd.SetFormat(c.GlobalString("format")); err != nil {
		return err
	}
	if err := cmd.SetWriteDelimiter(c.GlobalString("write-delimiter")); err != nil {
		return err
	}
	cmd.SetWithoutHeader(c.GlobalBool("without-header"))

	cmd.SetSilent(c.GlobalBool("silent"))
	cmd.SetCPU(c.GlobalInt("cpu"))
	cmd.SetStats(c.GlobalBool("stats"))

	return nil
}
<commit_msg>Fix help message.<commit_after>package main

import (
	"errors"
	"io/ioutil"
	"os"

	"github.com/mithrandie/csvq/lib/action"
	"github.com/mithrandie/csvq/lib/cmd"
	"github.com/mithrandie/csvq/lib/query"

	"github.com/urfave/cli"
)

var version = "v0.7.5"

func main() {
	cli.AppHelpTemplate = appHHelpTemplate
	cli.CommandHelpTemplate = commandHelpTemplate

	app := cli.NewApp()

	app.Name = "csvq"
	app.Usage = "SQL like query language for csv"
	app.ArgsUsage = "[\"query\"|\"statements\"|argument]"
	app.Version = version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "delimiter, d",
			Usage: "field delimiter. Default is \",\" for csv files, \"\\t\" for tsv files.",
		},
		cli.StringFlag{
			Name:  "encoding, e",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "line-break, l",
			Value: "LF",
			Usage: "line break. one of: CRLF|LF|CR",
		},
		cli.StringFlag{
			Name:  "timezone, z",
			Value: "Local",
			Usage: "default timezone. \"Local\", \"UTC\" or a timezone name(e.g. \"America/Los_Angeles\")",
		},
		cli.StringFlag{
			Name:  "repository, r",
			Usage: "directory path where files are located",
		},
		cli.StringFlag{
			Name:  "source, s",
			Usage: "load query from `FILE`",
		},
		cli.StringFlag{
			Name:  "datetime-format, t",
			Usage: "set datetime format to parse strings",
		},
		cli.BoolFlag{
			Name:  "no-header, n",
			Usage: "import the first line as a record",
		},
		cli.BoolFlag{
			Name:  "without-null, a",
			Usage: "parse empty fields as empty strings",
		},
		cli.StringFlag{
			Name:  "write-encoding, E",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "out, o",
			Usage: "write output to `FILE`",
		},
		cli.StringFlag{
			Name:  "format, f",
			Usage: "output format. one of: CSV|TSV|JSON|TEXT",
		},
		cli.StringFlag{
			Name:  "write-delimiter, D",
			Usage: "field delimiter for CSV",
		},
		cli.BoolFlag{
			Name:  "without-header, N",
			Usage: "when the file format is specified as CSV or TSV, write without the header line",
		},
		cli.BoolFlag{
			Name:  "silent",
			Usage: "does not output operation logs",
		},
		cli.IntFlag{
			Name:  "cpu, p",
			Usage: "hint for the number of cpu cores to be used. 1 - number of cpu cores",
		},
		cli.BoolFlag{
			Name:  "stats, x",
			Usage: "show execution time and memory statistics",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:      "fields",
			Usage:     "Show fields in a file",
			ArgsUsage: "CSV_FILE_PATH",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "fields")
					return cli.NewExitError("table is not specified", 1)
				}

				table := c.Args().First()

				err := action.ShowFields(table)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
		{
			Name:      "calc",
			Usage:     "Calculate a value from stdin",
			ArgsUsage: "\"expression\"",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "calc")
					return cli.NewExitError("expression is empty", 1)
				}

				expr := c.Args().First()
				err := action.Calc(expr)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
	}

	app.Before = func(c *cli.Context) error {
		return setFlags(c)
	}

	app.Action = func(c *cli.Context) error {
		queryString, err := readQuery(c)
		if err != nil {
			cli.ShowAppHelp(c)
			return cli.NewExitError(err.Error(), 1)
		}

		if err = action.Run(queryString, cmd.GetFlags().Source); err != nil {
			code := 1
			if apperr, ok := err.(query.AppError); ok {
				code = apperr.GetCode()
			} else if ex, ok := err.(*query.Exit); ok {
				code = ex.GetCode()
			}
			return cli.NewExitError(err.Error(), code)
		}

		return nil
	}

	app.Run(os.Args)
}

func readQuery(c *cli.Context) (string, error) {
	var queryString string

	flags := cmd.GetFlags()
	if 0 < len(flags.Source) {
		fp, err := os.Open(flags.Source)
		if err != nil {
			return queryString, err
		}
		defer fp.Close()

		buf, err := ioutil.ReadAll(fp)
		if err != nil {
			return queryString, err
		}
		queryString = string(buf)

	} else {
		if c.NArg() != 1 {
			return queryString, errors.New("query is empty")
		}
		queryString = c.Args().First()
	}

	return queryString, nil
}

func setFlags(c *cli.Context) error {
	if err := cmd.SetDelimiter(c.GlobalString("delimiter")); err != nil {
		return err
	}
	if err := cmd.SetEncoding(c.GlobalString("encoding")); err != nil {
		return err
	}
	if err := cmd.SetLineBreak(c.String("line-break")); err != nil {
		return err
	}
	if err := cmd.SetLocation(c.String("timezone")); err != nil {
		return err
	}
	if err := cmd.SetRepository(c.GlobalString("repository")); err != nil {
		return err
	}
	if err := cmd.SetSource(c.GlobalString("source")); err != nil {
		return err
	}
	cmd.SetDatetimeFormat(c.GlobalString("datetime-format"))
	cmd.SetNoHeader(c.GlobalBool("no-header"))
	cmd.SetWithoutNull(c.GlobalBool("without-null"))

	if err := cmd.SetWriteEncoding(c.GlobalString("write-encoding")); err != nil {
		return err
	}
	if err := cmd.SetOut(c.GlobalString("out")); err != nil {
		return err
	}
	if err := cmd.SetFormat(c.GlobalString("format")); err != nil {
		return err
	}
	if err := cmd.SetWriteDelimiter(c.GlobalString("write-delimiter")); err != nil {
		return err
	}
	cmd.SetWithoutHeader(c.GlobalBool("without-header"))

	cmd.SetSilent(c.GlobalBool("silent"))
	cmd.SetCPU(c.GlobalInt("cpu"))
	cmd.SetStats(c.GlobalBool("stats"))

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"log"
	"os"
	"time"

	"github.com/ChimeraCoder/anaconda"
	_ "github.com/go-sql-driver/mysql"
	"github.com/shiwork/favpost/config"
	"github.com/shiwork/favpost/model"
	"github.com/shiwork/favpost/server"
	"github.com/shiwork/favpost/storage"
	"flag"
)

var confPath = os.Getenv("FAVPOST_CONFIG")

func main() {
	conf, err := config.Parse(confPath)
	if err != nil {
		log.Fatal(err)
	}

	db, err := sql.Open("mysql", conf.DbDsn)
	if err != nil {
		log.Fatalf("Db initialize error %v\n", err)
	}
	err = storage.Prepare(db)
	if err != nil {
		log.Fatalf("Db prepare error: %v\n", err)
	}

	flag.Set("bind", ":80")

	anaconda.SetConsumerKey(conf.Consumer.ConsumerKey)
	anaconda.SetConsumerSecret(conf.Consumer.ConsumerSecret)

	// このへんは後で
	//	var callbackURL = "http://localhost:9001/auth/callback"
	//	credentials, err := anaconda.AuthorizationURL(callbackURL)

	//var user_id = int64(90649479) // @shiwork
	userRepos := model.GetUserRepository(db)
	go func() {
		for {
			// 酷いけどとりあえず全ユーザーを取得して処理を回す
			users := &[]model.User{}
			users, err = userRepos.GetAll()
			// sleep 5min
			time.Sleep(5 * time.Minute)

			for _, user := range *users {
				//user := &model.User{}
				//user, err = userRepos.Get(user_id)
				api := anaconda.NewTwitterApi(user.AccessToken.Token, user.AccessToken.Secret)
				searchResult, _ := api.GetFavorites(nil)

				for _, tweet := range searchResult {
					if len(tweet.Entities.Media) > 0 {

						exists, _ := storage.Exists(db, tweet)
						if !exists {
							storage.Add(db, tweet)
							model.SlackShare{conf.WebHookURL}.Share(tweet)
						}
					}
				}
			}

			// sleep 5min
			time.Sleep(5 * time.Minute)
		}
	}()

	/*
		fmt.Println("run http")
		http.HandleFunc("/", server.SayhelloName)
		err = http.ListenAndServe(":9090", nil)
		if err != nil {
			fmt.Println("http Listen error")
			log.Fatal("ListenAndServe: ", err)
		}
		fmt.Println("end of main func.")
	*/
	server.Run(conf, db)
}
<commit_msg>Port変更<commit_after>package main

import (
	"database/sql"
	"log"
	"os"
	"time"

	"github.com/ChimeraCoder/anaconda"
	_ "github.com/go-sql-driver/mysql"
	"github.com/shiwork/favpost/config"
	"github.com/shiwork/favpost/model"
	"github.com/shiwork/favpost/server"
	"github.com/shiwork/favpost/storage"
	"flag"
)

var confPath = os.Getenv("FAVPOST_CONFIG")

func main() {
	conf, err := config.Parse(confPath)
	if err != nil {
		log.Fatal(err)
	}

	db, err := sql.Open("mysql", conf.DbDsn)
	if err != nil {
		log.Fatalf("Db initialize error %v\n", err)
	}
	err = storage.Prepare(db)
	if err != nil {
		log.Fatalf("Db prepare error: %v\n", err)
	}

	flag.Set("bind", ":8001")

	anaconda.SetConsumerKey(conf.Consumer.ConsumerKey)
	anaconda.SetConsumerSecret(conf.Consumer.ConsumerSecret)

	// このへんは後で
	//	var callbackURL = "http://localhost:9001/auth/callback"
	//	credentials, err := anaconda.AuthorizationURL(callbackURL)

	//var user_id = int64(90649479) // @shiwork
	userRepos := model.GetUserRepository(db)
	go func() {
		for {
			// 酷いけどとりあえず全ユーザーを取得して処理を回す
			users := &[]model.User{}
			users, err = userRepos.GetAll()
			// sleep 5min
			time.Sleep(5 * time.Minute)

			for _, user := range *users {
				//user := &model.User{}
				//user, err = userRepos.Get(user_id)
				api := anaconda.NewTwitterApi(user.AccessToken.Token, user.AccessToken.Secret)
				searchResult, _ := api.GetFavorites(nil)

				for _, tweet := range searchResult {
					if len(tweet.Entities.Media) > 0 {

						exists, _ := storage.Exists(db, tweet)
						if !exists {
							storage.Add(db, tweet)
							model.SlackShare{conf.WebHookURL}.Share(tweet)
						}
					}
				}
			}

			// sleep 5min
			time.Sleep(5 * time.Minute)
		}
	}()

	/*
		fmt.Println("run http")
		http.HandleFunc("/", server.SayhelloName)
		err = http.ListenAndServe(":9090", nil)
		if err != nil {
			fmt.Println("http Listen error")
			log.Fatal("ListenAndServe: ", err)
		}
		fmt.Println("end of main func.")
	*/
	server.Run(conf, db)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/komukomo/postslack/slack"
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"text/template"
)

func main() {
	configFilePath := os.Getenv("HOME") + "/.postslackrc"
	config := loadDefaultConfig(configFilePath)

	args := make(map[string]*string)
	args["channel"] = flag.String("ch", config.Channel, "channel name")
	args["botname"] = flag.String("name", config.Name, "bot name")
	args["icon"] = flag.String("icon", config.Icon, "bot icon. emoji or URL ")
	args["incomingURL"] = flag.String("url", config.Url, "incomingURL")
	args["attachmentsFile"] = flag.String("att", attachmentsFilePath, "attachment filepath")
	args["param"] = flag.String("param", "", "parameters")
	noStdin := flag.Bool("empty", false, "no stdin (for attachments post)")
	flag.Parse()

	if *args["incomingURL"] == "" {
		panic("no value for incoming-webhook URL")
	}

	output := ""
	if !*noStdin {
		output = getStdin()
	}

	if *args["attachmentsFile"] != "" {
		parameters := str2map(*args["param"], output)
		postAttachments(*args["incomingURL"], *args["attachmentsFile"], parameters)
	} else {
		simplePost(args, output)
	}
}

func postAttachments(incomingURL string, attachmentsFile string, parameters map[string]string) {
	var doc bytes.Buffer
	slackMessage := slack.SlackMessage{}
	tpl := template.Must(template.ParseFiles(attachmentsFile))
	tpl.Execute(&doc, parameters)
	json.Unmarshal(doc.Bytes(), &slackMessage)
	slack.PostSlack(incomingURL, slackMessage)
}

func simplePost(args map[string]*string, text string) {
	slack.PostSlack(*args["incomingURL"], slack.SlackMessage{
		text,
		*args["botname"],
		*args["channel"],
		*args["icon"],
		nil,
	})
}

func exists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func str2map(param string, str string) map[string]string {
	paramlist := strings.Split(param, "&")
	result := make(map[string]string)

	for _, val := range paramlist {
		a := strings.Split(val, "=")
		if len(a) == 2 {
			if a[1] == "__stdin" {
				result[a[0]] = strings.Replace(str, "\n", "\\n", -1)
			} else {
				result[a[0]] = a[1]
			}
		}
	}
	return result
}

func getStdin() (stdin string) {
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		stdin += scanner.Text() + "\n"
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading stdin:", err)
	}
	return stdin
}

func loadDefaultConfig(configFilePath string) slack.Config {
	config := loadEnvConfig()

	if exists(configFilePath) {
		file, err := ioutil.ReadFile(configFilePath)
		if err != nil {
			panic(err)
		}
		json.Unmarshal(file, &config)
	}
	return config
}

func loadEnvConfig() slack.Config{
	return slack.Config{
		os.Getenv("SLACK_URL"),
		os.Getenv("SLACK_CHANNEL"),
		os.Getenv("SLACK_ICON"),
		os.Getenv("SLACK_NAME"),
	}
}
<commit_msg>Set attachments post as a default mode<commit_after>package main

import (
	"github.com/komukomo/postslack/slack"
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"text/template"
)

func main() {
	configFilePath := os.Getenv("HOME") + "/.postslackrc"
	attachmentsFilePath := os.Getenv("HOME") + "/.at-postslackrc"
	config := loadDefaultConfig(configFilePath)

	args := make(map[string]*string)
	args["channel"] = flag.String("ch", config.Channel, "channel name")
	args["botname"] = flag.String("name", config.Name, "bot name")
	args["icon"] = flag.String("icon", config.Icon, "bot icon. emoji or URL ")
	args["incomingURL"] = flag.String("url", config.Url, "incomingURL")
	args["attachmentsFile"] = flag.String("att", attachmentsFilePath, "attachment filepath")
	args["param"] = flag.String("param", "", "parameters")
	noStdin := flag.Bool("empty", false, "no stdin (for attachments post)")
	noAttachments := flag.Bool("no-attachments", false, "no attachments")
	flag.Parse()

	if *args["incomingURL"] == "" {
		panic("no value for incoming-webhook URL")
	}

	output := ""
	if !*noStdin {
		output = getStdin()
	}

	if *noAttachments {
		simplePost(args, output)
	} else {
		if exists(*args["attachmentsFile"]) {
			parameters := str2map(*args["param"], output)
			postAttachments(*args["incomingURL"], *args["attachmentsFile"], parameters)
		} else {
			panic("no attachmentsFile")
		}
	}
}

func postAttachments(incomingURL string, attachmentsFile string, parameters map[string]string) {
	var doc bytes.Buffer
	slackMessage := slack.SlackMessage{}
	tpl := template.Must(template.ParseFiles(attachmentsFile))
	tpl.Execute(&doc, parameters)
	json.Unmarshal(doc.Bytes(), &slackMessage)
	slack.PostSlack(incomingURL, slackMessage)
}

func simplePost(args map[string]*string, text string) {
	slack.PostSlack(*args["incomingURL"], slack.SlackMessage{
		text,
		*args["botname"],
		*args["channel"],
		*args["icon"],
		nil,
	})
}

func exists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func str2map(param string, str string) map[string]string {
	paramlist := strings.Split(param, "&")
	result := make(map[string]string)

	result["Stdin"] = strings.Replace(str, "\n", "\\n", -1)
	for _, val := range paramlist {
		a := strings.Split(val, "=")
		if len(a) == 2 {
			if a[1] == "__stdin" {
				result[a[0]] = strings.Replace(str, "\n", "\\n", -1)
			} else {
				result[a[0]] = a[1]
			}
		}
	}
	return result
}

func getStdin() (stdin string) {
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		stdin += scanner.Text() + "\n"
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading stdin:", err)
	}
	return stdin
}

func loadDefaultConfig(configFilePath string) slack.Config {
	config := loadEnvConfig()

	if exists(configFilePath) {
		file, err := ioutil.ReadFile(configFilePath)
		if err != nil {
			panic(err)
		}
		json.Unmarshal(file, &config)
	}
	return config
}

func loadEnvConfig() slack.Config{
	return slack.Config{
		os.Getenv("SLACK_URL"),
		os.Getenv("SLACK_CHANNEL"),
		os.Getenv("SLACK_ICON"),
		os.Getenv("SLACK_NAME"),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/DrItanium/unicornhat"
	"log"
	"math/rand"
	"runtime"
	"time"
)

type Memory []byte
type Word uint32

const (
	NumCpus   = 64
	Kilo      = 1024
	Meg       = Kilo * Kilo
	MemSize   = 64 * Meg
	µcoreSize = MemSize / 64
)

type µcore struct {
	index   int
	memory  Memory
	result  chan *unicornhat.Pixel
	done    chan int
	r, g, b byte
}

const (
	OpDelay = iota
	OpAdd
	OpSub
	OpMul
	OpDiv
	OpSet
	OpRotate
	OpDrain
	OpFill
	OpDrainTo
	OpFillTo
)

func (this *µcore) Execute() {
	log.Printf("µcore %d: Walking through memory", this.index)
	saturationIncrease := func(val, compare byte) byte {
		if val < compare {
			return 255
		} else {
			return val
		}
	}
	saturationDecrease := func(val, compare byte) byte {
		if val > compare {
			return 0
		} else {
			return val
		}
	}
	for i := 0; i < len(this.memory); i += 4 {
		r, g, b := this.memory[i+red], this.memory[i+green], this.memory[i+blue]
		// check the control byte
		c := this.memory[i+control]
		switch c {
		case OpDelay:
			MillisecondDelay(time.Duration((r + g + b) % 128))
		case OpAdd:
			this.r = saturationIncrease(this.r+r, this.r)
			this.g = saturationIncrease(this.g+g, this.g)
			this.b = saturationIncrease(this.b+b, this.b)
		case OpSub:
			this.r = saturationDecrease(this.r-r, this.r)
			this.g = saturationDecrease(this.g-g, this.g)
			this.b = saturationDecrease(this.b-b, this.b)
		case OpMul:
			this.r *= r
			this.b *= b
			this.g *= g
		case OpSet:
			this.r = r
			this.g = g
			this.b = b
		case OpDiv:
			if r != 0 {
				this.r /= r
			}
			if g != 0 {
				this.g /= g
			}
			if b != 0 {
				this.b /= b
			}
		case OpRotate:
			cr, cg, cb := this.r, this.g, this.b
			this.r = cb
			this.g = cr
			this.b = cg
		case OpDrain:
			// drain the pixel out each generation
			for this.g > 0 || this.b > 0 || this.r > 0 {
				this.r = saturationDecrease(this.r-1, this.r)
				this.g = saturationDecrease(this.g-1, this.g)
				this.b = saturationDecrease(this.b-1, this.b)
				this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
			}
			pixel := unicornhat.NewPixel(this.r, this.g, this.b)
			this.result <- pixel
			this.result <- pixel
			// once finished set the pixel to the r,g,b values in the instruction
			this.r, this.g, this.b = r, g, b
			this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
		case OpDrainTo:
			// drain the pixel out each generation
			for this.g > g || this.b > b || this.r > r {
				if this.r > r {
					this.r = saturationDecrease(this.r-1, this.r)
				}
				if this.g > g {
					this.g = saturationDecrease(this.g-1, this.g)
				}
				if this.b > b {
					this.b = saturationDecrease(this.b-1, this.b)
				}
				this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
			}
			pixel := unicornhat.NewPixel(this.r, this.g, this.b)
			this.result <- pixel
			this.result <- pixel
			// once finished set the pixel to the r,g,b values in the instruction
			this.r, this.g, this.b = r, g, b
			this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
		case OpFill:
			for this.g < 255 || this.b < 255 || this.r < 255 {
				this.r = saturationIncrease(this.r+1, this.r)
				this.g = saturationIncrease(this.g+1, this.g)
				this.b = saturationIncrease(this.b+1, this.b)
				this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
			}
			pixel := unicornhat.NewPixel(this.r, this.g, this.b)
			this.result <- pixel
			this.result <- pixel
			// once finished set the pixel to the r,g,b values in the instruction
			this.r, this.g, this.b = r, g, b
			this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
		case OpFillTo:
			for this.g < g || this.b < b || this.r < r {
				if this.r < r {
					this.r = saturationIncrease(this.r+1, this.r)
				}
				if this.g < g {
					this.g = saturationIncrease(this.g+1, this.g)
				}
				if this.b < b {
					this.b = saturationIncrease(this.b+1, this.b)
				}
				this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
			}
			pixel := unicornhat.NewPixel(this.r, this.g, this.b)
			this.result <- pixel
			this.result <- pixel
			// once finished set the pixel to the r,g,b values in the instruction
			this.r, this.g, this.b = r, g, b
			this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
		default:
		}
		this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
	}
	log.Printf("µcore %d: done walking", this.index)
	this.done <- 0
	close(this.done)
}

func New(index int, memory Memory) *µcore {
	var c µcore
	c.memory = memory
	c.index = index
	c.done = make(chan int)
	c.result = make(chan *unicornhat.Pixel)
	return &c
}

const (
	red = iota
	green
	blue
	control
)

func main() {
	var cpus []*µcore
	var memory Memory
	runtime.GOMAXPROCS(runtime.NumCPU())
	defer unicornhat.Shutdown()
	unicornhat.SetBrightness(unicornhat.DefaultBrightness / 2)
	if err := unicornhat.Initialize(); err != nil {
		fmt.Println(err)
		return
	}
	unicornhat.ClearLEDBuffer()
	unicornhat.Show()
	cpus = make([]*µcore, NumCpus)
	memory = make(Memory, MemSize)
	log.Print("Randomizing memory")
	fn := func(v int) byte {
		return byte(v + rand.Int())
	}
	for i, j := 0, 0; i < len(memory); i, j = i+4, j+1 {
		memory[i+red] = fn(j)
		memory[i+green] = fn(j)
		memory[i+blue] = fn(j)
		memory[i+control] = fn(j)
	}
	log.Print("Done randomizing memory")
	upperHalf := memory
	for i := 0; i < 64; i++ {
		targetMem := upperHalf[:µcoreSize]
		cpus[i] = New(i, targetMem)
		go cpus[i].Execute()
		upperHalf = upperHalf[µcoreSize:]
	}

	count := 0
	for {
		if count >= 64 {
			break
		}
		for i := 0; i < 64; i++ {
			c := cpus[i]
			if c == nil {
				continue
			}
			select {
			case value := <-c.result:
				unicornhat.SetPixelColor(c.index, value.R, value.G, value.B)
			case <-c.done:
				count++
				cpus[i] = nil
			default:
			}
		}
		unicornhat.Show()
		MillisecondDelay(33)
	}
}
func MillisecondDelay(msec time.Duration) {
	time.Sleep(msec * time.Millisecond)
}
<commit_msg>Added commandline flags<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/DrItanium/unicornhat"
	"log"
	"math/rand"
	"runtime"
	"time"
)

var msecdelay = flag.Uint("msecdelay", 33, "Millisecond delay between unicornhat updates")
var debug = flag.Bool("debug", false, "Enable debug logging")

type Memory []byte
type Word uint32

const (
	NumCpus   = 64
	Kilo      = 1024
	Meg       = Kilo * Kilo
	MemSize   = 64 * Meg
	µcoreSize = MemSize / 64
)

type µcore struct {
	index   int
	memory  Memory
	result  chan *unicornhat.Pixel
	done    chan int
	r, g, b byte
}

const (
	OpDelay = iota
	OpAdd
	OpSub
	OpMul
	OpDiv
	OpSet
	OpRotate
	OpDrain
	OpFill
)

func (this *µcore) Execute() {
	logPrint(fmt.Sprintf("µcore %d: Walking through memory", this.index))
	saturationIncrease := func(val, compare byte) byte {
		if val < compare {
			return 255
		} else {
			return val
		}
	}
	saturationDecrease := func(val, compare byte) byte {
		if val > compare {
			return 0
		} else {
			return val
		}
	}
	for i := 0; i < len(this.memory); i += 4 {
		r, g, b := this.memory[i+red], this.memory[i+green], this.memory[i+blue]
		// check the control byte
		c := this.memory[i+control]
		switch c {
		case OpDelay:
			MillisecondDelay(time.Duration((r + g + b) % 128))
		case OpAdd:
			this.r = saturationIncrease(this.r+r, this.r)
			this.g = saturationIncrease(this.g+g, this.g)
			this.b = saturationIncrease(this.b+b, this.b)
		case OpSub:
			this.r = saturationDecrease(this.r-r, this.r)
			this.g = saturationDecrease(this.g-g, this.g)
			this.b = saturationDecrease(this.b-b, this.b)
		case OpMul:
			this.r *= r
			this.b *= b
			this.g *= g
		case OpSet:
			this.r = r
			this.g = g
			this.b = b
		case OpDiv:
			if r != 0 {
				this.r /= r
			}
			if g != 0 {
				this.g /= g
			}
			if b != 0 {
				this.b /= b
			}
		case OpRotate:
			cr, cg, cb := this.r, this.g, this.b
			this.r = cb
			this.g = cr
			this.b = cg
		case OpDrain:
			// drain the pixel out each generation
			for this.g > g || this.b > b || this.r > r {
				if this.r > r {
					this.r = saturationDecrease(this.r-1, this.r)
				}
				if this.g > g {
					this.g = saturationDecrease(this.g-1, this.g)
				}
				if this.b > b {
					this.b = saturationDecrease(this.b-1, this.b)
				}
				this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
			}
			pixel := unicornhat.NewPixel(this.r, this.g, this.b)
			this.result <- pixel
			this.result <- pixel
		case OpFill:
			for this.g < g || this.b < b || this.r < r {
				if this.r < r {
					this.r = saturationIncrease(this.r+1, this.r)
				}
				if this.g < g {
					this.g = saturationIncrease(this.g+1, this.g)
				}
				if this.b < b {
					this.b = saturationIncrease(this.b+1, this.b)
				}
				this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
			}
			pixel := unicornhat.NewPixel(this.r, this.g, this.b)
			this.result <- pixel
			this.result <- pixel
		default:
		}
		this.result <- unicornhat.NewPixel(this.r, this.g, this.b)
	}
	logPrint(fmt.Sprintf("µcore %d: done walking", this.index))
	this.done <- 0
	close(this.done)
}

func New(index int, memory Memory) *µcore {
	var c µcore
	c.memory = memory
	c.index = index
	c.done = make(chan int)
	c.result = make(chan *unicornhat.Pixel)
	return &c
}

const (
	red = iota
	green
	blue
	control
)

func logPrint(msg string) {
	if *debug {
		log.Print(msg)
	}
}
func main() {
	flag.Parse()
	delay := time.Duration(*msecdelay)
	var cpus []*µcore
	var memory Memory
	runtime.GOMAXPROCS(runtime.NumCPU())
	defer unicornhat.Shutdown()
	unicornhat.SetBrightness(unicornhat.DefaultBrightness / 2)
	if err := unicornhat.Initialize(); err != nil {
		fmt.Println(err)
		return
	}
	unicornhat.ClearLEDBuffer()
	unicornhat.Show()
	cpus = make([]*µcore, NumCpus)
	memory = make(Memory, MemSize)
	logPrint("Randomizing memory")
	fn := func(v int) byte {
		return byte(v + rand.Int())
	}
	for i, j := 0, 0; i < len(memory); i, j = i+4, j+1 {
		memory[i+red] = fn(j)
		memory[i+green] = fn(j)
		memory[i+blue] = fn(j)
		memory[i+control] = fn(j)
	}
	logPrint("Done randomizing memory")
	upperHalf := memory
	for i := 0; i < 64; i++ {
		targetMem := upperHalf[:µcoreSize]
		cpus[i] = New(i, targetMem)
		go cpus[i].Execute()
		upperHalf = upperHalf[µcoreSize:]
	}

	count := 0
	for {
		if count >= 64 {
			break
		}
		for i := 0; i < 64; i++ {
			c := cpus[i]
			if c == nil {
				continue
			}
			select {
			case value := <-c.result:
				unicornhat.SetPixelColor(c.index, value.R, value.G, value.B)
			case <-c.done:
				count++
				cpus[i] = nil
			default:
			}
		}
		unicornhat.Show()
		MillisecondDelay(delay)
	}
}
func MillisecondDelay(msec time.Duration) {
	time.Sleep(msec * time.Millisecond)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/orivej/e"
	"github.com/pkg/errors"
)

type ReadResult struct {
	n   int
	err error
	buf []byte
}

type Log func(side, data string)

const (
	bufSize               = 4000
	clientTLSMarker       = "<starttls"
	serverTLSMarker       = "<proceed"
	clientAuthMarker      = "<auth"
	clientAuthReplacement = "<auth>CENSORED</auth>"
)

var (
	flListenAddress = flag.String("listen", ":5222", "listen address")
	flServer        = flag.String("server", "<server:port>", "target server:port")
	flLogDir        = flag.String("log-dir", "", "directory for session logs")
	flCensor        = flag.Bool("censor", false, "censor credentials from the log")

	flKeyPath  = flag.String("key", "", "path to TLS certificate key")
	flCertPath = flag.String("cert", "", "path to TLS certificate")

	flReplaceLocal  = flag.String("replace-local", "", "")
	flReplaceRemote = flag.String("replace-remote", "", "")
)

func main() {
	flag.Parse()

	certificate, err := tls.X509KeyPair([]byte(localhostCert), []byte(localhostKey))
	e.Exit(err)
	if len(*flCertPath) > 0 {
		certificate, err = tls.LoadX509KeyPair(*flCertPath, *flKeyPath)
		e.Exit(errors.Wrap(err, "can not load TLS key pair"))
	}

	listener, err := net.Listen("tcp", *flListenAddress)
	e.Exit(err)

	for idx := nextIdx(); true; idx++ {
		idx := idx
		client, err := listener.Accept()
		e.Exit(err)

		var f *os.File
		if *flLogDir != "" {
			path := filepath.Join(*flLogDir, logName(idx))
			flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL
			f, err = os.OpenFile(path, flags, 0666)
			e.Exit(err)
		}
		pr := func(side, data string) {
			ts := time.Now().Format(time.RFC3339)
			fmt.Printf("%04d %s %s %s\n", idx, ts, side, data)
			if f != nil {
				fmt.Fprintf(f, "%s %s %s\n", ts, side, data)
			}
		}

		go func() {
			defer e.CloseOrPrint(client)
			if f != nil {
				defer e.CloseOrPrint(f)
			}
			err := serve(client, pr, certificate)
			pr("X", err.Error())
		}()
	}
}

func nextIdx() int {
	if *flLogDir == "" {
		return 1
	}

	files, err := ioutil.ReadDir(*flLogDir)
	e.Exit(err)

	idx := 1
	var n int
	for _, file := range files {
		_, err := fmt.Sscanf(file.Name(), "%d.log", &n)
		if err == nil && idx <= n {
			idx = n + 1
		}
	}
	return idx
}

func logName(idx int) string {
	return fmt.Sprintf("%04d.log", idx)
}

func serve(client net.Conn, pr Log, certificate tls.Certificate) error {
	server, err := net.Dial("tcp", *flServer)
	if err != nil {
		return errors.Wrap(err, "failed to connect to server")
	}
	defer e.CloseOrPrint(server)

	err = proxy(client, server, pr, true)
	if err != nil {
		return errors.Wrap(err, "session terminated")
	}

	tlsClient := tls.Server(client, &tls.Config{
		Certificates: []tls.Certificate{certificate},
	})
	tlsServer := tls.Client(server, &tls.Config{
		InsecureSkipVerify: true,
	})
	// pr("X", "establishing TLS")
	err = proxy(tlsClient, tlsServer, pr, false)
	return errors.Wrap(err, "TLS session terminated")
}

func proxy(client, server io.ReadWriter, pr Log, untilTLS bool) error {
	clientNext, clientResults := startReader(client)
	serverNext, serverResults := startReader(server)

	censor := *flCensor
	rLocal := []byte(*flReplaceLocal)
	rRemote := []byte(*flReplaceRemote)
	replace := len(rLocal) > 0 && len(rRemote) > 0

	for {
		select {
		case result := <-clientResults:
			err := result.err
			if err != nil {
				return errors.Wrap(err, "failed to read from client")
			}

			buf := result.buf[:result.n]
			if replace {
				buf = bytes.Replace(buf, rLocal, rRemote, -1)
			}
			if censor && bytes.Contains(buf, []byte(clientAuthMarker)) {
				censor = false
				pr("?", clientAuthReplacement)
			} else {
				pr("?", escape(string(buf)))
			}

			_, err = server.Write(buf)
			if err != nil {
				return errors.Wrap(err, "failed to write to server")
			}

			if untilTLS && bytes.Contains(buf, []byte(clientTLSMarker)) {
				// pr("X", "client requests TLS")
				clientNext <- false
				continue
			}

			clientNext <- true

		case result := <-serverResults:
			err := result.err
			if err != nil {
				return errors.Wrap(err, "failed to read from server")
			}

			buf := result.buf[:result.n]
			pr("!", escape(string(buf)))
			if replace {
				buf = bytes.Replace(buf, rRemote, rLocal, -1)
			}

			_, err = client.Write(buf)
			if err != nil {
				return errors.Wrap(err, "failed to write to client")
			}

			if untilTLS && bytes.Contains(buf, []byte(serverTLSMarker)) {
				// pr("X", "server approves TLS")
				serverNext <- false
				return nil
			}

			serverNext <- true
		}
	}
}

func startReader(reader io.Reader) (chan<- bool, chan ReadResult) {
	readNext := make(chan bool)
	buf := make([]byte, bufSize)
	results := make(chan ReadResult)
	go func() {
		for {
			n, err := reader.Read(buf)
			results <- ReadResult{n, err, buf}
			if !<-readNext {
				return
			}
		}
	}()
	return readNext, results
}

var escaper = strings.NewReplacer(
	"\\", "\\\\",
	"\n", "\\n",
	"\r", "\\r",
)

func escape(s string) string {
	return escaper.Replace(s)
}
<commit_msg>Add blackhole-after option (for testing)<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/orivej/e"
	"github.com/pkg/errors"
)

type ReadResult struct {
	n   int
	err error
	buf []byte
}

type Log func(side, data string)

const (
	bufSize               = 4000
	clientTLSMarker       = "<starttls"
	serverTLSMarker       = "<proceed"
	clientAuthMarker      = "<auth"
	clientAuthReplacement = "<auth>CENSORED</auth>"
)

var (
	flListenAddress = flag.String("listen", ":5222", "listen address")
	flServer        = flag.String("server", "<server:port>", "target server:port")
	flLogDir        = flag.String("log-dir", "", "directory for session logs")
	flCensor        = flag.Bool("censor", false, "censor credentials from the log")

	flBlackhole = flag.Duration("blackhole-after", 0, "stop relaying (for testing)")

	flKeyPath  = flag.String("key", "", "path to TLS certificate key")
	flCertPath = flag.String("cert", "", "path to TLS certificate")

	flReplaceLocal  = flag.String("replace-local", "", "")
	flReplaceRemote = flag.String("replace-remote", "", "")
)

func main() {
	flag.Parse()

	certificate, err := tls.X509KeyPair([]byte(localhostCert), []byte(localhostKey))
	e.Exit(err)
	if len(*flCertPath) > 0 {
		certificate, err = tls.LoadX509KeyPair(*flCertPath, *flKeyPath)
		e.Exit(errors.Wrap(err, "can not load TLS key pair"))
	}

	listener, err := net.Listen("tcp", *flListenAddress)
	e.Exit(err)

	for idx := nextIdx(); true; idx++ {
		idx := idx
		client, err := listener.Accept()
		e.Exit(err)

		var f *os.File
		if *flLogDir != "" {
			path := filepath.Join(*flLogDir, logName(idx))
			flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL
			f, err = os.OpenFile(path, flags, 0666)
			e.Exit(err)
		}
		pr := func(side, data string) {
			ts := time.Now().Format(time.RFC3339)
			fmt.Printf("%04d %s %s %s\n", idx, ts, side, data)
			if f != nil {
				fmt.Fprintf(f, "%s %s %s\n", ts, side, data)
			}
		}

		go func() {
			defer e.CloseOrPrint(client)
			if f != nil {
				defer e.CloseOrPrint(f)
			}
			err := serve(client, pr, certificate)
			pr("X", err.Error())
		}()
	}
}

func nextIdx() int {
	if *flLogDir == "" {
		return 1
	}

	files, err := ioutil.ReadDir(*flLogDir)
	e.Exit(err)

	idx := 1
	var n int
	for _, file := range files {
		_, err := fmt.Sscanf(file.Name(), "%d.log", &n)
		if err == nil && idx <= n {
			idx = n + 1
		}
	}
	return idx
}

func logName(idx int) string {
	return fmt.Sprintf("%04d.log", idx)
}

func serve(client net.Conn, pr Log, certificate tls.Certificate) error {
	server, err := net.Dial("tcp", *flServer)
	if err != nil {
		return errors.Wrap(err, "failed to connect to server")
	}
	defer e.CloseOrPrint(server)

	err = proxy(client, server, pr, true)
	if err != nil {
		return errors.Wrap(err, "session terminated")
	}

	tlsClient := tls.Server(client, &tls.Config{
		Certificates: []tls.Certificate{certificate},
	})
	tlsServer := tls.Client(server, &tls.Config{
		InsecureSkipVerify: true,
	})
	// pr("X", "establishing TLS")
	err = proxy(tlsClient, tlsServer, pr, false)
	return errors.Wrap(err, "TLS session terminated")
}

func proxy(client, server io.ReadWriter, pr Log, untilTLS bool) error {
	clientNext, clientResults := startReader(client)
	serverNext, serverResults := startReader(server)

	censor := *flCensor
	rLocal := []byte(*flReplaceLocal)
	rRemote := []byte(*flReplaceRemote)
	replace := len(rLocal) > 0 && len(rRemote) > 0

	relay := true
	checkDeadline := func() {}
	if *flBlackhole != 0 {
		deadline := time.Now().Add(*flBlackhole)
		checkDeadline = func() {
			if relay && time.Now().After(deadline) {
				relay = false
				pr("X", "blackhole")
			}
		}
	}

	for {
		select {
		case result := <-clientResults:
			err := result.err
			if err != nil {
				return errors.Wrap(err, "failed to read from client")
			}

			buf := result.buf[:result.n]
			if replace {
				buf = bytes.Replace(buf, rLocal, rRemote, -1)
			}
			checkDeadline()
			if censor && bytes.Contains(buf, []byte(clientAuthMarker)) {
				censor = false
				pr("?", clientAuthReplacement)
			} else {
				pr("?", escape(string(buf)))
			}

			if relay {
				_, err = server.Write(buf)
				if err != nil {
					return errors.Wrap(err, "failed to write to server")
				}
			}

			if untilTLS && bytes.Contains(buf, []byte(clientTLSMarker)) {
				// pr("X", "client requests TLS")
				clientNext <- false
				continue
			}

			clientNext <- true

		case result := <-serverResults:
			err := result.err
			if err != nil {
				return errors.Wrap(err, "failed to read from server")
			}

			buf := result.buf[:result.n]
			checkDeadline()
			pr("!", escape(string(buf)))
			if replace {
				buf = bytes.Replace(buf, rRemote, rLocal, -1)
			}

			if relay {
				_, err = client.Write(buf)
				if err != nil {
					return errors.Wrap(err, "failed to write to client")
				}
			}

			if untilTLS && bytes.Contains(buf, []byte(serverTLSMarker)) {
				// pr("X", "server approves TLS")
				serverNext <- false
				return nil
			}

			serverNext <- true
		}
	}
}

func startReader(reader io.Reader) (chan<- bool, chan ReadResult) {
	readNext := make(chan bool)
	buf := make([]byte, bufSize)
	results := make(chan ReadResult)
	go func() {
		for {
			n, err := reader.Read(buf)
			results <- ReadResult{n, err, buf}
			if !<-readNext {
				return
			}
		}
	}()
	return readNext, results
}

var escaper = strings.NewReplacer(
	"\\", "\\\\",
	"\n", "\\n",
	"\r", "\\r",
)

func escape(s string) string {
	return escaper.Replace(s)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/mpl/basicauth"
)

const (
	idstring = "http://golang.org/pkg/http/#ListenAndServe"
)

var (
	flagHost     = flag.String("host", "0.0.0.0:8080", "listening port and hostname")
	flagHelp     = flag.Bool("h", false, "show this help")
	flagUserpass = flag.String("userpass", "", "optional username:password protection")
	flagCommand  = flag.String("command", "", "The command to run")
)

var (
	rootdir, _ = os.Getwd()
	up         *basicauth.UserPass
)

func usage() {
	fmt.Fprintf(os.Stderr, "\t httprunner \n")
	flag.PrintDefaults()
	os.Exit(2)
}

func makeHandler(fn func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if e, ok := recover().(error); ok {
				http.Error(w, e.Error(), http.StatusInternalServerError)
				return
			}
		}()
		w.Header().Set("Server", idstring)
		if isAllowed(r) {
			fn(w, r)
		} else {
			basicauth.SendUnauthorized(w, r, "httprunner")
		}
	}
}

func isAllowed(r *http.Request) bool {
	if *flagUserpass == "" {
		return true
	}
	return up.IsAllowed(r)
}

func initUserPass() {
	if *flagUserpass == "" {
		return
	}
	var err error
	up, err = basicauth.New(*flagUserpass)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO(mpl): have a look at https://github.com/cespare/window

type limitWriter struct {
	deadline   time.Time
	limit      int
	sum        int
	buf        *bytes.Buffer
	discarding bool
}

func (lw limitWriter) Write(p []byte) (n int, err error) {
	if lw.discarding {
		return ioutil.Discard.Write(p)
	}
	n, err = lw.buf.Write(p)
	lw.sum += n
	if lw.sum > lw.limit {
		lw.discarding = true
	}
	return
}

func (lw limitWriter) Read(p []byte) (n int, err error) {
	return lw.buf.Read(p)
}

func handleCommand(w http.ResponseWriter, r *http.Request) {
	// TODO(mpl): be less lazy about the doubled spaces, and probably other things.
	args := strings.Fields(*flagCommand)
	cmd := exec.Command(args[0], args[1:]...)
	var buf, berr bytes.Buffer
	lw := limitWriter{
		limit: 1 << 20,
		buf:   &buf,
	}
	stdout := io.MultiWriter(os.Stdout, lw)
	cmd.Stdout = stdout
	cmd.Stderr = &berr
	if err := cmd.Start(); err != nil {
		log.Printf("%v failed to start: %v, %v", args[0], err, berr.String())
		return
	}
	go func() {
		if err := cmd.Wait(); err != nil {
			log.Printf("%v failed: %v, %v", args[0], err, berr.String())
		}
	}()
	var bufout bytes.Buffer
	sendResponse := func(b *bytes.Buffer) {
		var response io.Reader
		if b.Len() > 0 {
			response = b
		} else {
			response = strings.NewReader("Command started but no output yet.")
		}
		if _, err := io.Copy(w, response); err != nil {
			log.Printf("response copy error: %v", err)
		}
	}
	var seenData bool
	// TODO(mpl): test if we could relax both these times now that we're sending the header asap.
	maxIdle := 200 * time.Millisecond
	t := time.After(1 * time.Second)
	lastDataTime := time.Now()
	for {
		select {
		case <-t:
			sendResponse(&bufout)
			return
		default:
		}
		n, err := io.Copy(&bufout, lw)
		if err != nil {
			log.Printf("output copy error: %v", err)
			break
		}
		if n > 0 {
			if !seenData {
				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
				w.WriteHeader(http.StatusOK)
				seenData = true
			}
			lastDataTime = time.Now()
		} else {
			if lastDataTime.Add(maxIdle).Before(time.Now()) {
				log.Printf("no output for more than %v, wrapping up.", maxIdle)
				break
			}
		}
	}
	sendResponse(&bufout)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *flagHelp {
		usage()
	}
	nargs := flag.NArg()
	if nargs > 0 {
		usage()
	}
	if *flagCommand == "" {
		fmt.Printf("No command to run")
		usage()
	}

	initUserPass()

	http.Handle("/run", makeHandler(handleCommand))
	if err := http.ListenAndServeTLS(
		*flagHost,
		filepath.Join(os.Getenv("HOME"), "keys", "cert.pem"),
		filepath.Join(os.Getenv("HOME"), "keys", "key.pem"),
		nil); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>add TODOs<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/mpl/basicauth"
)

const (
	idstring = "http://golang.org/pkg/http/#ListenAndServe"
)

var (
	flagHost     = flag.String("host", "0.0.0.0:8080", "listening port and hostname")
	flagHelp     = flag.Bool("h", false, "show this help")
	flagUserpass = flag.String("userpass", "", "optional username:password protection")
	flagCommand  = flag.String("command", "", "The command to run")
)

var (
	rootdir, _ = os.Getwd()
	up         *basicauth.UserPass
)

func usage() {
	fmt.Fprintf(os.Stderr, "\t httprunner \n")
	flag.PrintDefaults()
	os.Exit(2)
}

func makeHandler(fn func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if e, ok := recover().(error); ok {
				http.Error(w, e.Error(), http.StatusInternalServerError)
				return
			}
		}()
		w.Header().Set("Server", idstring)
		if isAllowed(r) {
			fn(w, r)
		} else {
			basicauth.SendUnauthorized(w, r, "httprunner")
		}
	}
}

func isAllowed(r *http.Request) bool {
	if *flagUserpass == "" {
		return true
	}
	return up.IsAllowed(r)
}

func initUserPass() {
	if *flagUserpass == "" {
		return
	}
	var err error
	up, err = basicauth.New(*flagUserpass)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO(mpl): have a look at https://github.com/cespare/window

type limitWriter struct {
	deadline   time.Time
	limit      int
	sum        int
	buf        *bytes.Buffer
	discarding bool
}

func (lw limitWriter) Write(p []byte) (n int, err error) {
	if lw.discarding {
		return ioutil.Discard.Write(p)
	}
	n, err = lw.buf.Write(p)
	lw.sum += n
	if lw.sum > lw.limit {
		lw.discarding = true
	}
	return
}

func (lw limitWriter) Read(p []byte) (n int, err error) {
	return lw.buf.Read(p)
}

func handleCommand(w http.ResponseWriter, r *http.Request) {
	// TODO(mpl): be less lazy about the doubled spaces, and probably other things.
	args := strings.Fields(*flagCommand)
	cmd := exec.Command(args[0], args[1:]...)
	var buf, berr bytes.Buffer
	lw := limitWriter{
		limit: 1 << 20,
		buf:   &buf,
	}
	stdout := io.MultiWriter(os.Stdout, lw)
	cmd.Stdout = stdout
	cmd.Stderr = &berr
	if err := cmd.Start(); err != nil {
		log.Printf("%v failed to start: %v, %v", args[0], err, berr.String())
		return
	}
	go func() {
		if err := cmd.Wait(); err != nil {
			log.Printf("%v failed: %v, %v", args[0], err, berr.String())
		}
	}()
	var bufout bytes.Buffer
	sendResponse := func(b *bytes.Buffer) {
		var response io.Reader
		if b.Len() > 0 {
			response = b
		} else {
			response = strings.NewReader("Command started but no output yet.")
		}
		// TODO(mpl): there's some sort of race in the output?
		if _, err := io.Copy(w, response); err != nil {
			log.Printf("response copy error: %v", err)
		}
	}
	var seenData bool
	// TODO(mpl): test if we could relax both these times now that we're sending the header asap.
	maxIdle := 200 * time.Millisecond
	t := time.After(1 * time.Second)
	lastDataTime := time.Now()
	for {
		select {
		case <-t:
			sendResponse(&bufout)
			return
		default:
		}
		n, err := io.Copy(&bufout, lw)
		if err != nil {
			log.Printf("output copy error: %v", err)
			break
		}
		if n > 0 {
			if !seenData {
				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
				w.WriteHeader(http.StatusOK)
				seenData = true
			}
			lastDataTime = time.Now()
		} else {
			if lastDataTime.Add(maxIdle).Before(time.Now()) {
				log.Printf("no output for more than %v, wrapping up.", maxIdle)
				break
			}
		}
	}
	sendResponse(&bufout)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *flagHelp {
		usage()
	}
	nargs := flag.NArg()
	if nargs > 0 {
		usage()
	}
	if *flagCommand == "" {
		fmt.Printf("No command to run")
		usage()
	}

	initUserPass()

	// TODO(mpl): handler to make it die
	http.Handle("/run", makeHandler(handleCommand))
	if err := http.ListenAndServeTLS(
		*flagHost,
		filepath.Join(os.Getenv("HOME"), "keys", "cert.pem"),
		filepath.Join(os.Getenv("HOME"), "keys", "key.pem"),
		nil); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (

	"fmt"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	// "github.com/btcsuite/btcd/blockchain/indexers"
	// "github.com/btcsuite/btcd/database"
	// "github.com/btcsuite/btcd/limits"

	"context"
	"github.com/btcboost/copernicus/conf"
	"github.com/btcboost/copernicus/connmgr"
	"github.com/btcboost/copernicus/limits"
)

const (
	// blockDbNamePrefix is the prefix for the block database name.  The
	// database type is appended to this value to form the full block
	// database name.
	blockDbNamePrefix = "blocks"
)

// winServiceMain is only invoked on Windows.  It detects when btcd is running
// as a service and reacts accordingly.
var winServiceMain func() (bool, error)

// btcdMain is the real main function for btcd.  It is necessary to work around
// the fact that deferred functions do not run when os.Exit() is called.  The
// optional serverChan parameter is mainly used by the service code to be
// notified with the server once it is setup so it can gracefully stop it when
// requested from the service control manager.
func bchMain(ctx context.Context, serverChan chan<- *server) error {
	// Load configuration and parse command line.  This function also
	// initializes logging and configures it accordingly.
	cfg := conf.Cfg

	// Get a channel that will be closed when a shutdown signal has been
	// triggered either from an OS signal such as SIGINT (Ctrl+C) or from
	// another subsystem such as the RPC server.
	interrupt := interruptListener()

	// Return now if an interrupt signal was triggered.
	if interruptRequested(interrupt) {
		return nil
	}

	// Load the block database.
	// db, err := loadBlockDB()
	// if err != nil {
	// 	btcdLog.Errorf("%v", err)
	// 	return err
	// }
	// defer func() {
	// 	// Ensure the database is sync'd and closed on shutdown.
	// 	btcdLog.Infof("Gracefully shutting down the database...")
	// 	db.Close()
	// }()

	// Return now if an interrupt signal was triggered.
	// if interruptRequested(interrupt) {
	// 	return nil
	// }

	// Drop indexes and exit if requested.
	//
	// NOTE: The order is important here because dropping the tx index also
	// drops the address index since it relies on it.
	// if cfg.DropAddrIndex {
	// 	if err := indexers.DropAddrIndex(db, interrupt); err != nil {
	// 		btcdLog.Errorf("%v", err)
	// 		return err
	// 	}

	// 	return nil
	// }
	// if cfg.DropTxIndex {
	// 	if err := indexers.DropTxIndex(db, interrupt); err != nil {
	// 		btcdLog.Errorf("%v", err)
	// 		return err
	// 	}

	// 	return nil
	// }

	// Create server and start it.
	// server, err := newServer(cfg.P2PNet.ListenerAddrs, db, activeNetParams.Params,
	// 	interrupt)
	// if err != nil {
	// 	// TODO: this logging could do with some beautifying.
	// 	btcdLog.Errorf("Unable to start server on %v: %v",
	// 		cfg.Listeners, err)
	// 	return err
	// }
	// defer func() {
	// 	btcdLog.Infof("Gracefully shutting down the server...")
	// 	server.Stop()
	// 	server.WaitForShutdown()
	// 	srvrLog.Infof("Server shutdown complete")
	// }()
	// server.Start()
	// if serverChan != nil {
	// 	serverChan <- server
	// }

	// FIXME
	s := server{
		chainParams:          chainParams,
		addrManager:          amgr,
		newPeers:             make(chan *serverPeer, cfg.MaxPeers),
		donePeers:            make(chan *serverPeer, cfg.MaxPeers),
		banPeers:             make(chan *serverPeer, cfg.MaxPeers),
		query:                make(chan interface{}),
		relayInv:             make(chan relayMsg, cfg.MaxPeers),
		broadcast:            make(chan broadcastMsg, cfg.MaxPeers),
		quit:                 make(chan struct{}),
		modifyRebroadcastInv: make(chan interface{}),
		peerHeightsUpdate:    make(chan updatePeerHeightsMsg),
		nat:                  nat,
		db:                   db,
		timeSource:           blockchain.NewMedianTime(),
		services:             services,
		sigCache:             txscript.NewSigCache(cfg.SigCacheMaxSize),
		hashCache:            txscript.NewHashCache(cfg.SigCacheMaxSize),
	}

	amgr := addrmgr.New(cfg.DataDir, iplookup)

	cmgr, err := connmgr.New(&connmgr.Config{
		ListenAddr:     cfg.P2PNet.ListenAddrs,
		RetryDuration:  cfg.P2PNet.RetryDuration,
		RetgetOutbound: cfg.P2PNet.TargetOutbound,

		Dial: func(ctx context.Context, netaddr net.Addr) {
			return net.Dialer{}.DialContext(ctx, netaddr.Network(), netaddr.String())
		},
		OnAccept:      s.inboundPeerConnected,
		OnConnect:     s.outboundPeerConnected,
		GetNewAddress: addmgr.GetNewAddress(),
	})
	for _, addr := range cfg.PeersOnStart {
		netAddr, err := addrStringToNetAddr(addr)
		if err != nil {
			return err
		}
		go cmgr.Connect(ctx, netAddr, true)
	}

	// qiw: we must do Start() after connect PeersOnStart
	cmgr.Start(ctx)

	// Wait until the interrupt signal is received from an OS signal or
	// shutdown is requested through one of the subsystems such as the RPC
	// server.
	<-interrupt
	return nil
}
<<<<<<< Updated upstream
=======

// removeRegressionDB removes the existing regression test database if running
// in regression test mode and it already exists.
func removeRegressionDB(dbPath string) error {
	// Don't do anything if not in regression test mode.
	if !cfg.RegressionTest {
		return nil
	}

	// Remove the old regression test database if it already exists.
	fi, err := os.Stat(dbPath)
	if err == nil {
		btcdLog.Infof("Removing regression test database from '%s'", dbPath)
		if fi.IsDir() {
			err := os.RemoveAll(dbPath)
			if err != nil {
				return err
			}
		} else {
			err := os.Remove(dbPath)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// dbPath returns the path to the block database given a database type.
func blockDbPath(dbType string) string {
	// The database name is based on the database type.
	dbName := blockDbNamePrefix + "_" + dbType
	if dbType == "sqlite" {
		dbName = dbName + ".db"
	}
	dbPath := filepath.Join(cfg.DataDir, dbName)
	return dbPath
}

// warnMultipleDBs shows a warning if multiple block database types are detected.
// This is not a situation most users want.  It is handy for development however
// to support multiple side-by-side databases.
func warnMultipleDBs() {
	// This is intentionally not using the known db types which depend
	// on the database types compiled into the binary since we want to
	// detect legacy db types as well.
	dbTypes := []string{"ffldb", "leveldb", "sqlite"}
	duplicateDbPaths := make([]string, 0, len(dbTypes)-1)
	for _, dbType := range dbTypes {
		if dbType == cfg.DbType {
			continue
		}

		// Store db path as a duplicate db if it exists.
		dbPath := blockDbPath(dbType)
		if fileExists(dbPath) {
			duplicateDbPaths = append(duplicateDbPaths, dbPath)
		}
	}

	// Warn if there are extra databases.
	if len(duplicateDbPaths) > 0 {
		selectedDbPath := blockDbPath(cfg.DbType)
		btcdLog.Warnf("WARNING: There are multiple block chain databases "+
			"using different database types.\nYou probably don't "+
			"want to waste disk space by having more than one.\n"+
			"Your current database is located at [%v].\nThe "+
			"additional database is located at %v", selectedDbPath,
			duplicateDbPaths)
	}
}

// loadBlockDB loads (or creates when needed) the block database taking into
// account the selected database backend and returns a handle to it.  It also
// contains additional logic such warning the user if there are multiple
// databases which consume space on the file system and ensuring the regression
// test database is clean when in regression test mode.
func loadBlockDB() (database.DB, error) {
	// The memdb backend does not have a file path associated with it, so
	// handle it uniquely.  We also don't want to worry about the multiple
	// database type warnings when running with the memory database.
	if cfg.DbType == "memdb" {
		btcdLog.Infof("Creating block database in memory.")
		db, err := database.Create(cfg.DbType)
		if err != nil {
			return nil, err
		}
		return db, nil
	}

	warnMultipleDBs()

	// The database name is based on the database type.
	dbPath := blockDbPath(cfg.DbType)

	// The regression test is special in that it needs a clean database for
	// each run, so remove it now if it already exists.
	removeRegressionDB(dbPath)

	btcdLog.Infof("Loading block database from '%s'", dbPath)
	db, err := database.Open(cfg.DbType, dbPath, activeNetParams.Net)
	if err != nil {
		// Return the error if it's not because the database doesn't
		// exist.
		if dbErr, ok := err.(database.Error); !ok || dbErr.ErrorCode !=
			database.ErrDbDoesNotExist {

			return nil, err
		}

		// Create the db if it does not exist.
		err = os.MkdirAll(cfg.DataDir, 0700)
		if err != nil {
			return nil, err
		}
		db, err = database.Create(cfg.DbType, dbPath, activeNetParams.Net)
		if err != nil {
			return nil, err
		}
	}

	btcdLog.Info("Block database loaded")
	return db, nil
}

func main() {
	// Use all processor cores.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Block and transaction processing can cause bursty allocations.  This
	// limits the garbage collector from excessively overallocating during
	// bursts.  This value was arrived at with the help of profiling live
	// usage.
	debug.SetGCPercent(10)

	// Up some limits.
	if err := limits.SetLimits(); err != nil {
		fmt.Fprintf(os.Stderr, "failed to set limits: %v\n", err)
		os.Exit(1)
	}

	// Call serviceMain on Windows to handle running as a service.  When
	// the return isService flag is true, exit now since we ran as a
	// service.  Otherwise, just fall through to normal operation.
	if runtime.GOOS == "windows" {
		isService, err := winServiceMain()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if isService {
			os.Exit(0)
		}
	}

	// Work around defer not working after os.Exit()
	if err := bchMain(nil); err != nil {
		os.Exit(1)
	}
}
>>>>>>> Stashed changes
<commit_msg>fix conflict<commit_after>// Copyright (c) 2013-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (

	"fmt"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	// "github.com/btcsuite/btcd/blockchain/indexers"
	// "github.com/btcsuite/btcd/database"
	// "github.com/btcsuite/btcd/limits"

	"context"
	"github.com/btcboost/copernicus/conf"
	"github.com/btcboost/copernicus/connmgr"
	"github.com/btcboost/copernicus/limits"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/addrmgr"
)

const (
	// blockDbNamePrefix is the prefix for the block database name.  The
	// database type is appended to this value to form the full block
	// database name.
	blockDbNamePrefix = "blocks"
)

// winServiceMain is only invoked on Windows.  It detects when btcd is running
// as a service and reacts accordingly.
var winServiceMain func() (bool, error)

// btcdMain is the real main function for btcd.  It is necessary to work around
// the fact that deferred functions do not run when os.Exit() is called.  The
// optional serverChan parameter is mainly used by the service code to be
// notified with the server once it is setup so it can gracefully stop it when
// requested from the service control manager.
func bchMain(ctx context.Context, serverChan chan<- *server) error {
	// Load configuration and parse command line.  This function also
	// initializes logging and configures it accordingly.
	cfg := conf.Cfg

	// Get a channel that will be closed when a shutdown signal has been
	// triggered either from an OS signal such as SIGINT (Ctrl+C) or from
	// another subsystem such as the RPC server.
	interrupt := interruptListener()

	// Return now if an interrupt signal was triggered.
	if interruptRequested(interrupt) {
		return nil
	}

	// Load the block database.
	// db, err := loadBlockDB()
	// if err != nil {
	// 	btcdLog.Errorf("%v", err)
	// 	return err
	// }
	// defer func() {
	// 	// Ensure the database is sync'd and closed on shutdown.
	// 	btcdLog.Infof("Gracefully shutting down the database...")
	// 	db.Close()
	// }()

	// Return now if an interrupt signal was triggered.
	// if interruptRequested(interrupt) {
	// 	return nil
	// }

	// Drop indexes and exit if requested.
	//
	// NOTE: The order is important here because dropping the tx index also
	// drops the address index since it relies on it.
	// if cfg.DropAddrIndex {
	// 	if err := indexers.DropAddrIndex(db, interrupt); err != nil {
	// 		btcdLog.Errorf("%v", err)
	// 		return err
	// 	}

	// 	return nil
	// }
	// if cfg.DropTxIndex {
	// 	if err := indexers.DropTxIndex(db, interrupt); err != nil {
	// 		btcdLog.Errorf("%v", err)
	// 		return err
	// 	}

	// 	return nil
	// }

	// Create server and start it.
	// server, err := newServer(cfg.P2PNet.ListenerAddrs, db, activeNetParams.Params,
	// 	interrupt)
	// if err != nil {
	// 	// TODO: this logging could do with some beautifying.
	// 	btcdLog.Errorf("Unable to start server on %v: %v",
	// 		cfg.Listeners, err)
	// 	return err
	// }
	// defer func() {
	// 	btcdLog.Infof("Gracefully shutting down the server...")
	// 	server.Stop()
	// 	server.WaitForShutdown()
	// 	srvrLog.Infof("Server shutdown complete")
	// }()
	// server.Start()
	// if serverChan != nil {
	// 	serverChan <- server
	// }

	// FIXME
	s := server{
		chainParams:          chainParams,
		addrManager:          amgr,
		newPeers:             make(chan *serverPeer, cfg.MaxPeers),
		donePeers:            make(chan *serverPeer, cfg.MaxPeers),
		banPeers:             make(chan *serverPeer, cfg.MaxPeers),
		query:                make(chan interface{}),
		relayInv:             make(chan relayMsg, cfg.MaxPeers),
		broadcast:            make(chan broadcastMsg, cfg.MaxPeers),
		quit:                 make(chan struct{}),
		modifyRebroadcastInv: make(chan interface{}),
		peerHeightsUpdate:    make(chan updatePeerHeightsMsg),
		nat:                  nat,
		db:                   db,
		timeSource:           blockchain.NewMedianTime(),
		services:             services,
		sigCache:             txscript.NewSigCache(cfg.SigCacheMaxSize),
		hashCache:            txscript.NewHashCache(cfg.SigCacheMaxSize),
	}

	amgr := addrmgr.New(cfg.DataDir, iplookup)

	cmgr, err := connmgr.New(&connmgr.Config{
		ListenAddr:     cfg.P2PNet.ListenAddrs,
		RetryDuration:  cfg.P2PNet.RetryDuration,
		RetgetOutbound: cfg.P2PNet.TargetOutbound,

		Dial: func(ctx context.Context, netaddr net.Addr) {
			return net.Dialer{}.DialContext(ctx, netaddr.Network(), netaddr.String())
		},
		OnAccept:      s.inboundPeerConnected,
		OnConnect:     s.outboundPeerConnected,
		GetNewAddress: addmgr.GetNewAddress(),
	})
	for _, addr := range cfg.PeersOnStart {
		netAddr, err := addrStringToNetAddr(addr)
		if err != nil {
			return err
		}
		go cmgr.Connect(ctx, netAddr, true)
	}

	// qiw: we must do Start() after connect PeersOnStart
	cmgr.Start(ctx)

	// Wait until the interrupt signal is received from an OS signal or
	// shutdown is requested through one of the subsystems such as the RPC
	// server.
	<-interrupt
	return nil
}

// removeRegressionDB removes the existing regression test database if running
// in regression test mode and it already exists.
func removeRegressionDB(dbPath string) error {
	// Don't do anything if not in regression test mode.
	if !cfg.RegressionTest {
		return nil
	}

	// Remove the old regression test database if it already exists.
	fi, err := os.Stat(dbPath)
	if err == nil {
		btcdLog.Infof("Removing regression test database from '%s'", dbPath)
		if fi.IsDir() {
			err := os.RemoveAll(dbPath)
			if err != nil {
				return err
			}
		} else {
			err := os.Remove(dbPath)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// dbPath returns the path to the block database given a database type.
func blockDbPath(dbType string) string {
	// The database name is based on the database type.
	dbName := blockDbNamePrefix + "_" + dbType
	if dbType == "sqlite" {
		dbName = dbName + ".db"
	}
	dbPath := filepath.Join(cfg.DataDir, dbName)
	return dbPath
}

// warnMultipleDBs shows a warning if multiple block database types are detected.
// This is not a situation most users want.  It is handy for development however
// to support multiple side-by-side databases.
func warnMultipleDBs() {
	// This is intentionally not using the known db types which depend
	// on the database types compiled into the binary since we want to
	// detect legacy db types as well.
	dbTypes := []string{"ffldb", "leveldb", "sqlite"}
	duplicateDbPaths := make([]string, 0, len(dbTypes)-1)
	for _, dbType := range dbTypes {
		if dbType == cfg.DbType {
			continue
		}

		// Store db path as a duplicate db if it exists.
		dbPath := blockDbPath(dbType)
		if fileExists(dbPath) {
			duplicateDbPaths = append(duplicateDbPaths, dbPath)
		}
	}

	// Warn if there are extra databases.
	if len(duplicateDbPaths) > 0 {
		selectedDbPath := blockDbPath(cfg.DbType)
		btcdLog.Warnf("WARNING: There are multiple block chain databases "+
			"using different database types.\nYou probably don't "+
			"want to waste disk space by having more than one.\n"+
			"Your current database is located at [%v].\nThe "+
			"additional database is located at %v", selectedDbPath,
			duplicateDbPaths)
	}
}

// loadBlockDB loads (or creates when needed) the block database taking into
// account the selected database backend and returns a handle to it.  It also
// contains additional logic such warning the user if there are multiple
// databases which consume space on the file system and ensuring the regression
// test database is clean when in regression test mode.
func loadBlockDB() (database.DB, error) {
	// The memdb backend does not have a file path associated with it, so
	// handle it uniquely.  We also don't want to worry about the multiple
	// database type warnings when running with the memory database.
	if cfg.DbType == "memdb" {
		btcdLog.Infof("Creating block database in memory.")
		db, err := database.Create(cfg.DbType)
		if err != nil {
			return nil, err
		}
		return db, nil
	}

	warnMultipleDBs()

	// The database name is based on the database type.
	dbPath := blockDbPath(cfg.DbType)

	// The regression test is special in that it needs a clean database for
	// each run, so remove it now if it already exists.
	removeRegressionDB(dbPath)

	btcdLog.Infof("Loading block database from '%s'", dbPath)
	db, err := database.Open(cfg.DbType, dbPath, activeNetParams.Net)
	if err != nil {
		// Return the error if it's not because the database doesn't
		// exist.
		if dbErr, ok := err.(database.Error); !ok || dbErr.ErrorCode !=
			database.ErrDbDoesNotExist {

			return nil, err
		}

		// Create the db if it does not exist.
		err = os.MkdirAll(cfg.DataDir, 0700)
		if err != nil {
			return nil, err
		}
		db, err = database.Create(cfg.DbType, dbPath, activeNetParams.Net)
		if err != nil {
			return nil, err
		}
	}

	btcdLog.Info("Block database loaded")
	return db, nil
}

func main() {
	// Use all processor cores.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Block and transaction processing can cause bursty allocations.  This
	// limits the garbage collector from excessively overallocating during
	// bursts.  This value was arrived at with the help of profiling live
	// usage.
	debug.SetGCPercent(10)

	// Up some limits.
	if err := limits.SetLimits(); err != nil {
		fmt.Fprintf(os.Stderr, "failed to set limits: %v\n", err)
		os.Exit(1)
	}

	// Call serviceMain on Windows to handle running as a service.  When
	// the return isService flag is true, exit now since we ran as a
	// service.  Otherwise, just fall through to normal operation.
	if runtime.GOOS == "windows" {
		isService, err := winServiceMain()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if isService {
			os.Exit(0)
		}
	}

	// Work around defer not working after os.Exit()
	if err := bchMain(nil); err != nil {
		os.Exit(1)
	}
}
<|endoftext|>