<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/elazarl/goproxy"
	"github.com/golang/glog"
	"io"
	"io/ioutil"
	"net/http"
	"sync"
	"time"
)

type EtchProxy struct {
	goproxy.ProxyHttpServer
	Cache *Cache
}

type EtchContextData struct {
	CachedContent *bytes.Buffer
}

func ReqMethodIs(method string) goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		return req.Method == method
	}
}

func StatusCodeIs(code int) goproxy.RespCondition {
	return goproxy.RespConditionFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) bool {
		if resp == nil {
			return false
		}
		return resp.StatusCode == code
	})
}

type RequestMutex struct {
	sync.Mutex
	resChans map[string][]chan *http.Response
}

var requestMutex = &RequestMutex{resChans: make(map[string][]chan *http.Response)}

func (proxy *EtchProxy) GuardRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
	requestMutex.Lock()
	chans, ok := requestMutex.resChans[req.URL.String()]

	if ok {
		ch := make(chan *http.Response)
		requestMutex.resChans[req.URL.String()] = append(chans, ch)
		requestMutex.Unlock()

		glog.V(3).Infof("[%s] Waiting for ongoing request", req.URL)

		res := <-ch

		glog.V(3).Infof("[%s] Response got from chan: %s", req.URL, res)

		if res != nil {
			return req, res
		}
	} else {
		requestMutex.resChans[req.URL.String()] = make([]chan *http.Response, 0)
		requestMutex.Unlock()
	}

	return req, nil
}

func (proxy *EtchProxy) MakeRangedRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
	cache := proxy.Cache

	if entry := cache.GetEntry(req.URL); entry.FileInfo != nil {
		glog.Infof("%s: found cache entry", req.URL)

		content, err := entry.Content()
		if err != nil {
			glog.Errorf("OnRequest: retrieving cache content: %s", err)
			return req, nil
		}

		// TODO mutex
		cachedContent := bytes.NewBuffer(content)
		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", cachedContent.Len()-1))
		req.Header.Add("If-Modified-Since", entry.FileInfo.ModTime().Format(time.RFC850))

		_, resp, err := proxy.Tr.DetailedRoundTrip(req)
		if err != nil {
			glog.Errorf("OnRequest: executing request: %s", err)
			return req, nil
		}

		if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
			glog.Infof("[%s] Got 416: attempting re-fetch", req.URL)

			// clear cache
			req.Header.Del("Range")
			req.Header.Del("If-Modified-Since")

			_, _resp, err := proxy.Tr.DetailedRoundTrip(req)

			if err != nil {
				glog.Errorf("OnRequest: re-fetch: %s", err)
				return req, nil
			}

			resp = _resp
			cachedContent = new(bytes.Buffer)
		}

		ctx.UserData = &EtchContextData{cachedContent}

		return req, resp
	}

	return req, nil
}

func (proxy *EtchProxy) RestoreCache(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	glog.V(3).Infof("[%s] Headers: %s", ctx.Req.URL, resp.Header)

	if ctx.UserData == nil {
		return resp
	}

	userData := ctx.UserData.(*EtchContextData)

	switch resp.StatusCode {
	case http.StatusOK:
		// nop

	case http.StatusPartialContent:
		buf := userData.CachedContent

		// TODO check Content-Range value
		contentRange := resp.Header.Get("Content-Range")
		glog.V(2).Infof("Content-Range: %s", contentRange)

		// 1 バイトだけキャッシュと重複してるはずなので
		// その部分を整合性チェックに使う
		responseBody := bufio.NewReader(resp.Body)
		firstByte, err := responseBody.ReadByte()

		if err != nil {
			glog.Errorf("[%s] Reading response: %s", ctx.Req.URL, err)
			return goproxy.NewResponse(
				ctx.Req, goproxy.ContentTypeText, http.StatusInternalServerError, fmt.Sprintf("Reading response: %s", err))
		}

		if buf.Bytes()[buf.Len()-1] != firstByte {
			// TODO re-attempt
			glog.Errorf("[%s] Cache mismatch", ctx.Req.URL)
			return resp
		}

		io.Copy(buf, responseBody)

		resp.StatusCode = http.StatusOK
		resp.Header.Del("Content-Range")
		resp.Body = ioutil.NopCloser(buf)

	case http.StatusNotModified:
		resp.StatusCode = http.StatusOK
		resp.Body = ioutil.NopCloser(userData.CachedContent)

	case http.StatusNonAuthoritativeInfo:
		resp.StatusCode = http.StatusGone

	default:
		glog.Errorf("[%s] Unhandled status code: %d", ctx.Req.URL, resp.StatusCode)
	}

	return resp
}

func (proxy *EtchProxy) StoreCache(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	cache := proxy.Cache

	lastModified := time.Now()
	lastModifiedString := resp.Header.Get("Last-Modified")
	if lastModifiedString != "" {
		var err error
		lastModified, err = http.ParseTime(lastModifiedString)
		if err != nil {
			glog.Errorf(`[%s]: Parsing Last-Modified header "%s": %s`, ctx.Req.URL, lastModifiedString, err)
		}
	}

	cacheEntry := cache.GetEntry(ctx.Req.URL)
	if cacheEntry.FileInfo == nil || cacheEntry.FileInfo.ModTime().Before(lastModified) {
		glog.Infof("[%s] update cache: %s", ctx.Req.URL, cacheEntry.FilePath)

		cacheWriter, err := cacheEntry.GetWriter()
		if err != nil {
			glog.Errorf("[%s]: cacheEntry.GetWriter: %s", ctx.Req.URL, err)
			return resp
		}

		buf := new(bytes.Buffer)
		io.Copy(io.MultiWriter(buf, cacheWriter), resp.Body)
		cacheWriter.Close()

		cacheEntry.SetMtime(lastModified)

		resp.Body = ioutil.NopCloser(buf)
	} else {
		glog.V(2).Infof("[%s] Response is not fresher than cache: %s <= %s", ctx.Req.URL, lastModified, cacheEntry.FileInfo.ModTime())
	}

	return resp
}

func (proxy *EtchProxy) UnguardRequest(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	requestMutex.Lock()
	defer requestMutex.Unlock()
	if chans := requestMutex.resChans[ctx.Req.URL.String()]; chans != nil {
		for _, ch := range chans {
			ch <- resp
		}
		delete(requestMutex.resChans, ctx.Req.URL.String())
	}

	return resp
}

func main() {
	cacheDir := flag.String("cache-dir", "cache", "cache directory")
	proxyPort := flag.Int("port", 8080, "proxy port")

	flag.Parse()

	cache := &Cache{*cacheDir}
	proxy := &EtchProxy{*goproxy.NewProxyHttpServer(), cache}

	proxy.OnRequest(ReqMethodIs("GET")).DoFunc(proxy.GuardRequest)
	proxy.OnRequest(ReqMethodIs("GET")).DoFunc(proxy.MakeRangedRequest)
	proxy.OnResponse(ReqMethodIs("GET")).DoFunc(proxy.RestoreCache)
	proxy.OnResponse(goproxy.ContentTypeIs("text/plain"), ReqMethodIs("GET"), StatusCodeIs(200)).DoFunc(proxy.StoreCache)
	proxy.OnResponse().DoFunc(proxy.UnguardRequest)

	proxy.Verbose = true

	glog.Errorf("%s", http.ListenAndServe(fmt.Sprintf(":%d", *proxyPort), proxy))
}
<commit_msg>re-attempt on あぼーん<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/elazarl/goproxy"
	"github.com/golang/glog"
	"io"
	"io/ioutil"
	"net/http"
	"sync"
	"time"
)

type EtchProxy struct {
	goproxy.ProxyHttpServer
	Cache *Cache
}

type EtchContextData struct {
	CachedContent *bytes.Buffer
}

func ReqMethodIs(method string) goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		return req.Method == method
	}
}

func StatusCodeIs(code int) goproxy.RespCondition {
	return goproxy.RespConditionFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) bool {
		if resp == nil {
			return false
		}
		return resp.StatusCode == code
	})
}

type RequestMutex struct {
	sync.Mutex
	resChans map[string][]chan *http.Response
}

var requestMutex = &RequestMutex{resChans: make(map[string][]chan *http.Response)}

func (proxy *EtchProxy) GuardRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
	requestMutex.Lock()
	chans, ok := requestMutex.resChans[req.URL.String()]

	if ok {
		ch := make(chan *http.Response)
		requestMutex.resChans[req.URL.String()] = append(chans, ch)
		requestMutex.Unlock()

		glog.V(3).Infof("[%s] Waiting for ongoing request", req.URL)

		res := <-ch

		glog.V(3).Infof("[%s] Response got from chan: %s", req.URL, res)

		if res != nil {
			return req, res
		}
	} else {
		requestMutex.resChans[req.URL.String()] = make([]chan *http.Response, 0)
		requestMutex.Unlock()
	}

	return req, nil
}

func (proxy *EtchProxy) MakeRangedRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
	cache := proxy.Cache

	if entry := cache.GetEntry(req.URL); entry.FileInfo != nil {
		glog.Infof("%s: found cache entry", req.URL)

		content, err := entry.Content()
		if err != nil {
			glog.Errorf("OnRequest: retrieving cache content: %s", err)
			return req, nil
		}

		// TODO mutex
		cachedContent := bytes.NewBuffer(content)
		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", cachedContent.Len()-1))
		req.Header.Add("If-Modified-Since", entry.FileInfo.ModTime().Format(time.RFC850))

		_, resp, err := proxy.Tr.DetailedRoundTrip(req)
		if err != nil {
			glog.Errorf("OnRequest: executing request: %s", err)
			return req, nil
		}

		if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
			glog.Infof("[%s] Got 416: attempting re-fetch", req.URL)

			// clear cache
			req.Header.Del("Range")
			req.Header.Del("If-Modified-Since")

			_, _resp, err := proxy.Tr.DetailedRoundTrip(req)

			if err != nil {
				glog.Errorf("OnRequest: re-fetch: %s", err)
				return req, nil
			}

			resp = _resp
			cachedContent = new(bytes.Buffer)
		}

		ctx.UserData = &EtchContextData{cachedContent}

		return req, resp
	}

	return req, nil
}

func (proxy *EtchProxy) RestoreCache(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	glog.V(3).Infof("[%s] Headers: %s", ctx.Req.URL, resp.Header)

	if ctx.UserData == nil {
		return resp
	}

	userData := ctx.UserData.(*EtchContextData)

	switch resp.StatusCode {
	case http.StatusOK:
		// nop

	case http.StatusPartialContent:
		buf := userData.CachedContent

		// TODO check Content-Range value
		contentRange := resp.Header.Get("Content-Range")
		glog.V(2).Infof("Content-Range: %s", contentRange)

		// 1 バイトだけキャッシュと重複してるはずなので
		// その部分を整合性チェックに使う
		responseBody := bufio.NewReader(resp.Body)
		firstByte, err := responseBody.ReadByte()

		if err != nil {
			glog.Errorf("[%s] Reading response: %s", ctx.Req.URL, err)
			return goproxy.NewResponse(
				ctx.Req, goproxy.ContentTypeText, http.StatusInternalServerError, fmt.Sprintf("Reading response: %s", err))
		}

		if buf.Bytes()[buf.Len()-1] != firstByte {
			glog.V(2).Infof("[%s] Cache mismatch", ctx.Req.URL)

			glog.V(2).Infof("[%s] Attempting re-fetch", ctx.Req.URL)

			ctx.Req.Header.Del("Range")
			ctx.Req.Header.Del("If-Modified-Since")
			ctx.UserData = nil

			_, _resp, err := proxy.Tr.DetailedRoundTrip(ctx.Req)
			if _resp == nil || err != nil {
				glog.Errorf("[%s] Re-fetch failed: %s", ctx.Req.URL, err)
				return resp
			}

			return _resp
		}

		io.Copy(buf, responseBody)

		resp.StatusCode = http.StatusOK
		resp.Header.Del("Content-Range")
		resp.Body = ioutil.NopCloser(buf)

	case http.StatusNotModified:
		resp.StatusCode = http.StatusOK
		resp.Body = ioutil.NopCloser(userData.CachedContent)

	case http.StatusNonAuthoritativeInfo:
		resp.StatusCode = http.StatusGone

	default:
		glog.Errorf("[%s] Unhandled status code: %d", ctx.Req.URL, resp.StatusCode)
	}

	return resp
}

func (proxy *EtchProxy) StoreCache(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	cache := proxy.Cache

	lastModified := time.Now()
	lastModifiedString := resp.Header.Get("Last-Modified")
	if lastModifiedString != "" {
		var err error
		lastModified, err = http.ParseTime(lastModifiedString)
		if err != nil {
			glog.Errorf(`[%s]: Parsing Last-Modified header "%s": %s`, ctx.Req.URL, lastModifiedString, err)
		}
	}

	cacheEntry := cache.GetEntry(ctx.Req.URL)
	if cacheEntry.FileInfo == nil || cacheEntry.FileInfo.ModTime().Before(lastModified) {
		glog.Infof("[%s] update cache: %s", ctx.Req.URL, cacheEntry.FilePath)

		cacheWriter, err := cacheEntry.GetWriter()
		if err != nil {
			glog.Errorf("[%s]: cacheEntry.GetWriter: %s", ctx.Req.URL, err)
			return resp
		}

		buf := new(bytes.Buffer)
		io.Copy(io.MultiWriter(buf, cacheWriter), resp.Body)
		cacheWriter.Close()

		cacheEntry.SetMtime(lastModified)

		resp.Body = ioutil.NopCloser(buf)
	} else {
		glog.V(2).Infof("[%s] Response is not fresher than cache: %s <= %s", ctx.Req.URL, lastModified, cacheEntry.FileInfo.ModTime())
	}

	return resp
}

func (proxy *EtchProxy) UnguardRequest(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	requestMutex.Lock()
	defer requestMutex.Unlock()
	if chans := requestMutex.resChans[ctx.Req.URL.String()]; chans != nil {
		for _, ch := range chans {
			ch <- resp
		}
		delete(requestMutex.resChans, ctx.Req.URL.String())
	}

	return resp
}

func main() {
	cacheDir := flag.String("cache-dir", "cache", "cache directory")
	proxyPort := flag.Int("port", 8080, "proxy port")

	flag.Parse()

	cache := &Cache{*cacheDir}
	proxy := &EtchProxy{*goproxy.NewProxyHttpServer(), cache}

	proxy.OnRequest(ReqMethodIs("GET")).DoFunc(proxy.GuardRequest)
	proxy.OnRequest(ReqMethodIs("GET")).DoFunc(proxy.MakeRangedRequest)
	proxy.OnResponse(ReqMethodIs("GET")).DoFunc(proxy.RestoreCache)
	proxy.OnResponse(goproxy.ContentTypeIs("text/plain"), ReqMethodIs("GET"), StatusCodeIs(200)).DoFunc(proxy.StoreCache)
	proxy.OnResponse().DoFunc(proxy.UnguardRequest)

	proxy.Verbose = true

	glog.Errorf("%s", http.ListenAndServe(fmt.Sprintf(":%d", *proxyPort), proxy))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net"
	"time"

	"github.com/dustin/go-coap"
	"github.com/golang/glog"

	"github.com/couchbaselabs/consolio/types"
)

type wireEvent struct {
	DB   string
	Type string
	Data *json.RawMessage
}

var NotFound = errors.New("not found")

var evListen = flag.String("evin", ":8675", "Event input binding")

func evInfoHandler(l *net.UDPConn, a *net.UDPAddr, m *coap.Message) *coap.Message {
	if db == nil {
		glog.Warning("db isn't ready")
		return nil
	}

	// Verify we got something sensible
	cType, ok := m.Option(coap.ContentType).(uint32)
	if !ok || coap.MediaType(cType) != coap.AppJSON {
		glog.Warningf("Got invalid content type on request: %v",
			coap.MediaType(cType))
		return nil
	}

	we := wireEvent{}
	err := json.Unmarshal(m.Payload, &we)
	if err != nil {
		glog.Warningf("Error unmarshaling %s: %v", m.Payload, err)
		return nil
	}

	if we.DB == db.Name {
		glog.Infof("Skipping update for my own bucket: %s", m.Payload)
		return nil
	}

	err = db.Update("db-"+we.DB, 0, func(current []byte) ([]byte, error) {
		if len(current) == 0 {
			return nil, NotFound
		}

		item := consolio.Item{}
		err := json.Unmarshal(current, &item)
		if err != nil {
			return nil, err
		}

		switch we.Type {
		case "state":
			s := struct{ State string }{}
			err := json.Unmarshal([]byte(*we.Data), &s)
			if err != nil {
				return nil, err
			}
			item.LastChange = time.Now().UTC()
			item.State = s.State
			glog.Infof("Got state change for %q: %s", we.DB, *we.Data)
		case "stats":
			item.LastStat = time.Now().UTC()
			glog.Infof("Got stats for %q: %s", we.DB, *we.Data)
			item.Stats = we.Data
		default:
			return nil, fmt.Errorf("Unknown event type: %q", we.Type)
		}

		return json.Marshal(&item)
	})

	if err != nil {
		glog.Warningf("Error updating %q as %v: %s: %v",
			we.DB, we.Type, *we.Data, err)
		return nil
	}

	var rv *coap.Message
	if m.IsConfirmable() {
		rv = &coap.Message{
			Type:      coap.Acknowledgement,
			Code:      coap.Created,
			MessageID: m.MessageID,
		}
	}
	return rv
}

func eventListener() {
	if *evListen == "" {
		return
	}

	handler := coap.NewServeMux()
	handler.HandleFunc("/dbinfo", evInfoHandler)

	glog.Fatal(coap.ListenAndServe("udp", *evListen, handler))
}
<commit_msg>Update CoAP<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net"
	"time"

	"github.com/dustin/go-coap"
	"github.com/golang/glog"

	"github.com/couchbaselabs/consolio/types"
)

type wireEvent struct {
	DB   string
	Type string
	Data *json.RawMessage
}

var NotFound = errors.New("not found")

var evListen = flag.String("evin", ":8675", "Event input binding")

func evInfoHandler(l *net.UDPConn, a *net.UDPAddr, m *coap.Message) *coap.Message {
	if db == nil {
		glog.Warning("db isn't ready")
		return nil
	}

	// Verify we got something sensible
	cType, ok := m.Option(coap.ContentFormat).(uint32)
	if !ok || coap.MediaType(cType) != coap.AppJSON {
		glog.Warningf("Got invalid content type on request: %v",
			coap.MediaType(cType))
		return nil
	}

	we := wireEvent{}
	err := json.Unmarshal(m.Payload, &we)
	if err != nil {
		glog.Warningf("Error unmarshaling %s: %v", m.Payload, err)
		return nil
	}

	if we.DB == db.Name {
		glog.Infof("Skipping update for my own bucket: %s", m.Payload)
		return nil
	}

	err = db.Update("db-"+we.DB, 0, func(current []byte) ([]byte, error) {
		if len(current) == 0 {
			return nil, NotFound
		}

		item := consolio.Item{}
		err := json.Unmarshal(current, &item)
		if err != nil {
			return nil, err
		}

		switch we.Type {
		case "state":
			s := struct{ State string }{}
			err := json.Unmarshal([]byte(*we.Data), &s)
			if err != nil {
				return nil, err
			}
			item.LastChange = time.Now().UTC()
			item.State = s.State
			glog.Infof("Got state change for %q: %s", we.DB, *we.Data)
		case "stats":
			item.LastStat = time.Now().UTC()
			glog.Infof("Got stats for %q: %s", we.DB, *we.Data)
			item.Stats = we.Data
		default:
			return nil, fmt.Errorf("Unknown event type: %q", we.Type)
		}

		return json.Marshal(&item)
	})

	if err != nil {
		glog.Warningf("Error updating %q as %v: %s: %v",
			we.DB, we.Type, *we.Data, err)
		return nil
	}

	var rv *coap.Message
	if m.IsConfirmable() {
		rv = &coap.Message{
			Type:      coap.Acknowledgement,
			Code:      coap.Created,
			MessageID: m.MessageID,
		}
	}
	return rv
}

func eventListener() {
	if *evListen == "" {
		return
	}

	handler := coap.NewServeMux()
	handler.HandleFunc("/dbinfo", evInfoHandler)

	glog.Fatal(coap.ListenAndServe("udp", *evListen, handler))
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"strings"
	"syscall"
)

func execLocalCommand(command string, args []string) (err error) {
	log.Printf("COMMAND: '%s %s'\n", command, strings.Join(args, " "))

	// the exec syscall expects the command name to be in argv
	argv := make([]string, len(args)+1)
	argv[0] = command
	for i := range args {
		argv[i+1] = args[i]
	}

	return syscall.Exec(command, argv, os.Environ())
}

func runRemoteCommand(command []string) (err error) {
	return nil
}

// vim: ts=4 sw=4 noet tw=120 softtabstop=4
<commit_msg>remove dead code<commit_after>package main

import (
	"os"
	"syscall"
)

func execLocalCommand(command string, args []string) (err error) {
	// the exec syscall expects the command name to be in argv
	argv := make([]string, len(args)+1)
	argv[0] = command
	for i := range args {
		argv[i+1] = args[i]
	}

	return syscall.Exec(command, argv, os.Environ())
}

// vim: ts=4 sw=4 noet tw=120 softtabstop=4
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/exec"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

var cmdExec = &Command{
	Exec:        runExec,
	UsageLine:   "exec [OPTIONS] SERVER COMMAND", // FIXME: add [ARGS...] support
	Description: "Run a command on a running server",
	Help:        "Run a command on a running server.",
}

func init() {
	// FIXME: -h
	cmdExec.Flag.BoolVar(&execW, []string{"w", "-wait"}, false, "")
}

// Flags
var execW bool // -w flag

func NewSshExecCmd(ipAddress string, allocateTTY bool) []string {
	execCmd := []string{}

	if os.Getenv("DEBUG") != "1" {
		execCmd = append(execCmd, "-q")
	}

	if os.Getenv("exec_secure") != "1" {
		execCmd = append(execCmd, "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no")
	}

	execCmd = append(execCmd, "-l", "root", ipAddress)

	if allocateTTY {
		execCmd = append(execCmd, "-t")
	}

	return execCmd
}

func WaitForServerState(api *ScalewayAPI, serverId string, targetState string) (*ScalewayServer, error) {
	var server *ScalewayServer
	var err error

	for {
		server, err = api.GetServer(serverId)
		if err != nil {
			return nil, err
		}
		if server.State == targetState {
			break
		}
		time.Sleep(2)
	}

	return server, nil
}

func runExec(cmd *Command, args []string) {
	if len(args) < 2 {
		log.Fatalf("usage: scw %s", cmd.UsageLine)
	}
	command := args[1]

	serverId := cmd.GetServer(args[0])

	var server *ScalewayServer
	var err error

	if execW {
		// --wait
		server, err = WaitForServerState(cmd.API, serverId, "running")
		if err != nil {
			log.Fatalf("Failed to wait for server to be ready, %v", err)
		}
	} else {
		// no --wait
		server, err := cmd.API.GetServer(serverId)
		if err != nil {
			log.Fatalf("Failed to get server information for %s: %s", server.Identifier, err)
		}
	}

	execCmd := append(NewSshExecCmd(server.PublicAddress.IP, true), "--", command)

	log.Debugf("Executing: ssh %s", strings.Join(execCmd, " "))
	spawn := exec.Command("ssh", execCmd...)
	spawn.Stdout = os.Stdout
	spawn.Stdin = os.Stdin
	spawn.Stderr = os.Stderr
	err = spawn.Run()
	if err != nil {
		os.Exit(1)
	}
}
<commit_msg>Waiting for ssh port to be open (Fixes #24)<commit_after>package main

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

var cmdExec = &Command{
	Exec:        runExec,
	UsageLine:   "exec [OPTIONS] SERVER COMMAND", // FIXME: add [ARGS...] support
	Description: "Run a command on a running server",
	Help:        "Run a command on a running server.",
}

func init() {
	// FIXME: -h
	cmdExec.Flag.BoolVar(&execW, []string{"w", "-wait"}, false, "")
}

// Flags
var execW bool // -w flag

func NewSshExecCmd(ipAddress string, allocateTTY bool) []string {
	execCmd := []string{}

	if os.Getenv("DEBUG") != "1" {
		execCmd = append(execCmd, "-q")
	}

	if os.Getenv("exec_secure") != "1" {
		execCmd = append(execCmd, "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no")
	}

	execCmd = append(execCmd, "-l", "root", ipAddress)

	if allocateTTY {
		execCmd = append(execCmd, "-t")
	}

	return execCmd
}

func WaitForServerState(api *ScalewayAPI, serverId string, targetState string) (*ScalewayServer, error) {
	var server *ScalewayServer
	var err error

	for {
		server, err = api.GetServer(serverId)
		if err != nil {
			return nil, err
		}
		if server.State == targetState {
			break
		}
		time.Sleep(1 * time.Second)
	}

	return server, nil
}

func WaitForTcpPortOpen(dest string) error {
	for {
		conn, err := net.Dial("tcp", dest)
		if err == nil {
			defer conn.Close()
			break
		}
		time.Sleep(1 * time.Second)
	}
	return nil
}

func runExec(cmd *Command, args []string) {
	if len(args) < 2 {
		log.Fatalf("usage: scw %s", cmd.UsageLine)
	}
	command := args[1]

	serverId := cmd.GetServer(args[0])

	var server *ScalewayServer
	var err error

	if execW {
		// --wait
		server, err = WaitForServerState(cmd.API, serverId, "running")
		if err != nil {
			log.Fatalf("Failed to wait for server to be ready, %v", err)
		}

		dest := fmt.Sprintf("%s:22", server.PublicAddress.IP)
		err = WaitForTcpPortOpen(dest)
		if err != nil {
			log.Fatalf("Failed to get an open SSH port: %v", err)
		}
	} else {
		// no --wait
		server, err := cmd.API.GetServer(serverId)
		if err != nil {
			log.Fatalf("Failed to get server information for %s: %s", server.Identifier, err)
		}
	}

	execCmd := append(NewSshExecCmd(server.PublicAddress.IP, true), "--", command)

	log.Debugf("Executing: ssh %s", strings.Join(execCmd, " "))
	spawn := exec.Command("ssh", execCmd...)
	spawn.Stdout = os.Stdout
	spawn.Stdin = os.Stdin
	spawn.Stderr = os.Stderr
	err = spawn.Run()
	if err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 - 2022 The excelize Authors. All rights reserved. Use of
// this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//
// Package excelize providing a set of functions that allow you to write to and
// read from XLAM / XLSM / XLSX / XLTM / XLTX files. Supports reading and
// writing spreadsheet documents generated by Microsoft Excel™ 2007 and later.
// Supports complex components by high compatibility, and provided streaming
// API for generating or reading data from a worksheet with huge amounts of
// data. This library needs Go version 1.15 or later.

package excelize

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"io"
	"os"
	"path/filepath"
	"sync"
)

// NewFile provides a function to create new file by default template. For
// example:
//
//    f := NewFile()
//
func NewFile() *File {
	f := newFile()
	f.Pkg.Store("_rels/.rels", []byte(xml.Header+templateRels))
	f.Pkg.Store(defaultXMLPathDocPropsApp, []byte(xml.Header+templateDocpropsApp))
	f.Pkg.Store(defaultXMLPathDocPropsCore, []byte(xml.Header+templateDocpropsCore))
	f.Pkg.Store("xl/_rels/workbook.xml.rels", []byte(xml.Header+templateWorkbookRels))
	f.Pkg.Store("xl/theme/theme1.xml", []byte(xml.Header+templateTheme))
	f.Pkg.Store("xl/worksheets/sheet1.xml", []byte(xml.Header+templateSheet))
	f.Pkg.Store(defaultXMLPathStyles, []byte(xml.Header+templateStyles))
	f.Pkg.Store(defaultXMLPathWorkbook, []byte(xml.Header+templateWorkbook))
	f.Pkg.Store(defaultXMLPathContentTypes, []byte(xml.Header+templateContentTypes))
	f.SheetCount = 1
	f.CalcChain = f.calcChainReader()
	f.Comments = make(map[string]*xlsxComments)
	f.ContentTypes = f.contentTypesReader()
	f.Drawings = sync.Map{}
	f.Styles = f.stylesReader()
	f.DecodeVMLDrawing = make(map[string]*decodeVmlDrawing)
	f.VMLDrawing = make(map[string]*vmlDrawing)
	f.WorkBook = f.workbookReader()
	f.Relationships = sync.Map{}
	f.Relationships.Store("xl/_rels/workbook.xml.rels", f.relsReader("xl/_rels/workbook.xml.rels"))
	f.sheetMap["Sheet1"] = "xl/worksheets/sheet1.xml"
	ws, _ := f.workSheetReader("Sheet1")
	f.Sheet.Store("xl/worksheets/sheet1.xml", ws)
	f.Theme = f.themeReader()
	return f
}

// Save provides a function to override the spreadsheet with origin path.
func (f *File) Save() error {
	if f.Path == "" {
		return ErrSave
	}
	if f.options != nil {
		return f.SaveAs(f.Path, *f.options)
	}
	return f.SaveAs(f.Path)
}

// SaveAs provides a function to create or update to a spreadsheet at the
// provided path.
func (f *File) SaveAs(name string, opt ...Options) error {
	if len(name) > MaxFilePathLength {
		return ErrMaxFilePathLength
	}
	f.Path = name
	contentType, ok := map[string]string{
		".xlam": ContentTypeAddinMacro,
		".xlsm": ContentTypeMacro,
		".xlsx": ContentTypeSheetML,
		".xltm": ContentTypeTemplateMacro,
		".xltx": ContentTypeTemplate,
	}[filepath.Ext(f.Path)]
	if !ok {
		return ErrWorkbookFileFormat
	}
	f.setContentTypePartProjectExtensions(contentType)
	file, err := os.OpenFile(filepath.Clean(name), os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.ModePerm)
	if err != nil {
		return err
	}
	defer file.Close()
	f.options = nil
	for i := range opt {
		f.options = &opt[i]
	}
	return f.Write(file)
}

// Close closes and cleanup the open temporary file for the spreadsheet.
func (f *File) Close() error {
	var err error
	if f.sharedStringTemp != nil {
		if err := f.sharedStringTemp.Close(); err != nil {
			return err
		}
	}
	f.tempFiles.Range(func(k, v interface{}) bool {
		if err = os.Remove(v.(string)); err != nil {
			return false
		}
		return true
	})
	return err
}

// Write provides a function to write to an io.Writer.
func (f *File) Write(w io.Writer) error {
	_, err := f.WriteTo(w)
	return err
}

// WriteTo implements io.WriterTo to write the file.
func (f *File) WriteTo(w io.Writer) (int64, error) {
	if f.options != nil && f.options.Password != "" {
		buf, err := f.WriteToBuffer()
		if err != nil {
			return 0, err
		}
		return buf.WriteTo(w)
	}
	if err := f.writeDirectToWriter(w); err != nil {
		return 0, err
	}
	return 0, nil
}

// WriteToBuffer provides a function to get bytes.Buffer from the saved file,
// and it allocates space in memory. Be careful when the file size is large.
func (f *File) WriteToBuffer() (*bytes.Buffer, error) {
	buf := new(bytes.Buffer)
	zw := zip.NewWriter(buf)

	if err := f.writeToZip(zw); err != nil {
		return buf, zw.Close()
	}

	if f.options != nil && f.options.Password != "" {
		if err := zw.Close(); err != nil {
			return buf, err
		}
		b, err := Encrypt(buf.Bytes(), f.options)
		if err != nil {
			return buf, err
		}
		buf.Reset()
		buf.Write(b)
		return buf, nil
	}
	return buf, zw.Close()
}

// writeDirectToWriter provides a function to write to io.Writer.
func (f *File) writeDirectToWriter(w io.Writer) error {
	zw := zip.NewWriter(w)
	if err := f.writeToZip(zw); err != nil {
		_ = zw.Close()
		return err
	}
	return zw.Close()
}

// writeToZip provides a function to write to zip.Writer
func (f *File) writeToZip(zw *zip.Writer) error {
	f.calcChainWriter()
	f.commentsWriter()
	f.contentTypesWriter()
	f.drawingsWriter()
	f.vmlDrawingWriter()
	f.workBookWriter()
	f.workSheetWriter()
	f.relsWriter()
	f.sharedStringsLoader()
	f.sharedStringsWriter()
	f.styleSheetWriter()

	for path, stream := range f.streams {
		fi, err := zw.Create(path)
		if err != nil {
			return err
		}
		var from io.Reader
		from, err = stream.rawData.Reader()
		if err != nil {
			_ = stream.rawData.Close()
			return err
		}
		_, err = io.Copy(fi, from)
		if err != nil {
			return err
		}
		_ = stream.rawData.Close()
	}
	var err error
	f.Pkg.Range(func(path, content interface{}) bool {
		if err != nil {
			return false
		}
		if _, ok := f.streams[path.(string)]; ok {
			return true
		}
		var fi io.Writer
		fi, err = zw.Create(path.(string))
		if err != nil {
			return false
		}
		_, err = fi.Write(content.([]byte))
		return true
	})
	f.tempFiles.Range(func(path, content interface{}) bool {
		if _, ok := f.Pkg.Load(path); ok {
			return true
		}
		var fi io.Writer
		fi, err = zw.Create(path.(string))
		if err != nil {
			return false
		}
		_, err = fi.Write(f.readBytes(path.(string)))
		return true
	})
	return err
}
<commit_msg>Optimizing line breaks for comments (#1281)<commit_after>// Copyright 2016 - 2022 The excelize Authors. All rights reserved. Use of
// this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//
// Package excelize providing a set of functions that allow you to write to and
// read from XLAM / XLSM / XLSX / XLTM / XLTX files. Supports reading and
// writing spreadsheet documents generated by Microsoft Excel™ 2007 and later.
// Supports complex components by high compatibility, and provided streaming
// API for generating or reading data from a worksheet with huge amounts of
// data. This library needs Go version 1.15 or later.

package excelize

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"io"
	"os"
	"path/filepath"
	"sync"
)

// NewFile provides a function to create new file by default template.
// For example:
//
//    f := NewFile()
//
func NewFile() *File {
	f := newFile()
	f.Pkg.Store("_rels/.rels", []byte(xml.Header+templateRels))
	f.Pkg.Store(defaultXMLPathDocPropsApp, []byte(xml.Header+templateDocpropsApp))
	f.Pkg.Store(defaultXMLPathDocPropsCore, []byte(xml.Header+templateDocpropsCore))
	f.Pkg.Store("xl/_rels/workbook.xml.rels", []byte(xml.Header+templateWorkbookRels))
	f.Pkg.Store("xl/theme/theme1.xml", []byte(xml.Header+templateTheme))
	f.Pkg.Store("xl/worksheets/sheet1.xml", []byte(xml.Header+templateSheet))
	f.Pkg.Store(defaultXMLPathStyles, []byte(xml.Header+templateStyles))
	f.Pkg.Store(defaultXMLPathWorkbook, []byte(xml.Header+templateWorkbook))
	f.Pkg.Store(defaultXMLPathContentTypes, []byte(xml.Header+templateContentTypes))
	f.SheetCount = 1
	f.CalcChain = f.calcChainReader()
	f.Comments = make(map[string]*xlsxComments)
	f.ContentTypes = f.contentTypesReader()
	f.Drawings = sync.Map{}
	f.Styles = f.stylesReader()
	f.DecodeVMLDrawing = make(map[string]*decodeVmlDrawing)
	f.VMLDrawing = make(map[string]*vmlDrawing)
	f.WorkBook = f.workbookReader()
	f.Relationships = sync.Map{}
	f.Relationships.Store("xl/_rels/workbook.xml.rels", f.relsReader("xl/_rels/workbook.xml.rels"))
	f.sheetMap["Sheet1"] = "xl/worksheets/sheet1.xml"
	ws, _ := f.workSheetReader("Sheet1")
	f.Sheet.Store("xl/worksheets/sheet1.xml", ws)
	f.Theme = f.themeReader()
	return f
}

// Save provides a function to override the spreadsheet with origin path.
func (f *File) Save() error {
	if f.Path == "" {
		return ErrSave
	}
	if f.options != nil {
		return f.SaveAs(f.Path, *f.options)
	}
	return f.SaveAs(f.Path)
}

// SaveAs provides a function to create or update to a spreadsheet at the
// provided path.
func (f *File) SaveAs(name string, opt ...Options) error {
	if len(name) > MaxFilePathLength {
		return ErrMaxFilePathLength
	}
	f.Path = name
	contentType, ok := map[string]string{
		".xlam": ContentTypeAddinMacro,
		".xlsm": ContentTypeMacro,
		".xlsx": ContentTypeSheetML,
		".xltm": ContentTypeTemplateMacro,
		".xltx": ContentTypeTemplate,
	}[filepath.Ext(f.Path)]
	if !ok {
		return ErrWorkbookFileFormat
	}
	f.setContentTypePartProjectExtensions(contentType)
	file, err := os.OpenFile(filepath.Clean(name), os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.ModePerm)
	if err != nil {
		return err
	}
	defer file.Close()
	f.options = nil
	for i := range opt {
		f.options = &opt[i]
	}
	return f.Write(file)
}

// Close closes and cleanup the open temporary file for the spreadsheet.
func (f *File) Close() error {
	var err error
	if f.sharedStringTemp != nil {
		if err := f.sharedStringTemp.Close(); err != nil {
			return err
		}
	}
	f.tempFiles.Range(func(k, v interface{}) bool {
		if err = os.Remove(v.(string)); err != nil {
			return false
		}
		return true
	})
	return err
}

// Write provides a function to write to an io.Writer.
func (f *File) Write(w io.Writer) error {
	_, err := f.WriteTo(w)
	return err
}

// WriteTo implements io.WriterTo to write the file.
func (f *File) WriteTo(w io.Writer) (int64, error) {
	if f.options != nil && f.options.Password != "" {
		buf, err := f.WriteToBuffer()
		if err != nil {
			return 0, err
		}
		return buf.WriteTo(w)
	}
	if err := f.writeDirectToWriter(w); err != nil {
		return 0, err
	}
	return 0, nil
}

// WriteToBuffer provides a function to get bytes.Buffer from the saved file,
// and it allocates space in memory. Be careful when the file size is large.
func (f *File) WriteToBuffer() (*bytes.Buffer, error) {
	buf := new(bytes.Buffer)
	zw := zip.NewWriter(buf)

	if err := f.writeToZip(zw); err != nil {
		return buf, zw.Close()
	}

	if f.options != nil && f.options.Password != "" {
		if err := zw.Close(); err != nil {
			return buf, err
		}
		b, err := Encrypt(buf.Bytes(), f.options)
		if err != nil {
			return buf, err
		}
		buf.Reset()
		buf.Write(b)
		return buf, nil
	}
	return buf, zw.Close()
}

// writeDirectToWriter provides a function to write to io.Writer.
func (f *File) writeDirectToWriter(w io.Writer) error {
	zw := zip.NewWriter(w)
	if err := f.writeToZip(zw); err != nil {
		_ = zw.Close()
		return err
	}
	return zw.Close()
}

// writeToZip provides a function to write to zip.Writer
func (f *File) writeToZip(zw *zip.Writer) error {
	f.calcChainWriter()
	f.commentsWriter()
	f.contentTypesWriter()
	f.drawingsWriter()
	f.vmlDrawingWriter()
	f.workBookWriter()
	f.workSheetWriter()
	f.relsWriter()
	f.sharedStringsLoader()
	f.sharedStringsWriter()
	f.styleSheetWriter()

	for path, stream := range f.streams {
		fi, err := zw.Create(path)
		if err != nil {
			return err
		}
		var from io.Reader
		from, err = stream.rawData.Reader()
		if err != nil {
			_ = stream.rawData.Close()
			return err
		}
		_, err = io.Copy(fi, from)
		if err != nil {
			return err
		}
		_ = stream.rawData.Close()
	}
	var err error
	f.Pkg.Range(func(path, content interface{}) bool {
		if err != nil {
			return false
		}
		if _, ok := f.streams[path.(string)]; ok {
			return true
		}
		var fi io.Writer
		fi, err = zw.Create(path.(string))
		if err != nil {
			return false
		}
		_, err = fi.Write(content.([]byte))
		return true
	})
	f.tempFiles.Range(func(path, content interface{}) bool {
		if _, ok := f.Pkg.Load(path); ok {
			return true
		}
		var fi io.Writer
		fi, err = zw.Create(path.(string))
		if err != nil {
			return false
		}
		_, err = fi.Write(f.readBytes(path.(string)))
		return true
	})
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"path"
	"runtime"
	"time"

	. "github.com/fuzzy/gocolor"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/urfave/cli"
)

var (
	Filled  int64
	Overlap int64
)

func ProgressIndicator(c int64, t int64) {
	fmt.Printf("%s %d of %d (%6.02f%%) runtime: %dsec\r",
		String("Info").Green().Bold(),
		c,
		t,
		(float64(c)/float64(t))*100.00,
		time.Now().Unix()-StartTime)
}

func fillWorker(d chan map[string]string) {
	for {
		msg := <-d
		if msg["SRC"] == "__EXIT__" && msg["DST"] == "__EXIT__" {
			return
		} else {
			if isFile(msg["SRC"]) && isFile(msg["DST"]) {
				if !backfillFile(msg["SRC"], msg["DST"]) {
					Error.Printf("S:%s D:%s - Backfill operation failed.",
						msg["SRC"],
						msg["DST"])
					os.Exit(1)
				} else {
					Filled++
					ProgressIndicator(Filled, Overlap)
				}
			}
		}
	}
}

func backfillFile(s string, d string) bool {
	// Open our filehandles
	sDb, sErr := whisper.Open(s)
	dDb, dErr := whisper.Open(d)
	if !chkErr(sErr) || !chkErr(dErr) {
		os.Exit(1)
	}
	// Defer their closings
	defer sDb.Close()
	defer dDb.Close()

	// Now for a series of checks, first to ensure that both
	// files have the same number of archives in them.
	if sDb.Header.Metadata.ArchiveCount != dDb.Header.Metadata.ArchiveCount {
		Error.Println("The files have a mismatched set of archives.")
		return false
	}

	// Now we'll start processing the archives, checking as we go to see if they
	// are matched. That way we at least fill in what we can, possibly....
	for i, a := range sDb.Header.Archives {
		// The offset
		if a.Offset == dDb.Header.Archives[i].Offset {
			// and the number of points
			if a.Points == dDb.Header.Archives[i].Points {
				// and finally the interval
				if a.SecondsPerPoint == dDb.Header.Archives[i].SecondsPerPoint {
					// ok, now let's get rolling through the archives
					sp, se := sDb.DumpArchive(i)
					if se != nil {
						Error.Println(se.Error())
						os.Exit(1)
					}
					dp, de := dDb.DumpArchive(i)
					if de != nil {
						Error.Println(de.Error())
						os.Exit(1)
					}
					for idx := 0; idx < len(sp); idx++ {
						if sp[idx].Timestamp != 0 && sp[idx].Value != 0 {
							if dp[idx].Timestamp == 0 || dp[idx].Value == 0 {
								dp[idx].Timestamp = sp[idx].Timestamp
								dp[idx].Value = sp[idx].Value
								dDb.Update(dp[idx])
							}
						}
					}
				}
			}
		}
	}
	return true
}

func Filler(c *cli.Context) error {
	if len(c.Args()) == 2 {
		args := c.Args()
		if isDir(args[0]) && isDir(args[1]) {
			Filled = 0

			// First things first, let's spawn our pool of workers.
			dataCh := make(chan map[string]string, 1)
			for i := 0; i < c.Int("j"); i++ {
				go fillWorker(dataCh)
			}

			// Let's get this dir walking in there then shall we?
			srcFiles := listFiles(args[0])
			dstFiles := listFiles(args[1])

			// Now let's find all our overlap
			overlap := []string{}
			for a := 0; a < len(srcFiles); a++ {
				for b := 0; b < len(dstFiles); b++ {
					if srcFiles[a] == dstFiles[b] {
						overlap = append(overlap, srcFiles[a])
					}
				}
			}

			// And display some stats
			Info.Printf("srcDir: %d files, dstDir: %d files, %d overlap.",
				len(srcFiles),
				len(dstFiles),
				len(overlap))
			Overlap = int64(len(overlap))

			// Now we can push in all our data, and let our workers do their
			// lovely little thing. Ahhhhh concurrency.
			StartTime = time.Now().Unix()
			for i := 0; i < len(overlap); i++ {
				dataCh <- map[string]string{
					"SRC": fmt.Sprintf("%s/%s", args[0], overlap[i]),
					"DST": fmt.Sprintf("%s/%s", args[1], overlap[i]),
				}
			}

			// And while they're off doing that, here we will just sit and watch
			// the return channel and count things. Once we have all our
			// backfill operations accounted for, we can reap all of our workers
			// and carry on.
			for Filled < int64(len(overlap)) {
				time.Sleep(1 * time.Second)
			}
			// And finally let's reap all our children
			for idx := 0; idx < c.Int("j"); idx++ {
				dataCh <- map[string]string{
					"SRC": "__EXIT__",
					"DST": "__EXIT__",
				}
			}

			fmt.Println("")
			Info.Println((Overlap / (time.Now().Unix() - StartTime)),
				"whisper files processed per second.")
		} else {
			if !backfillFile(args[0], args[1]) {
				e := fmt.Sprintf("Error while backfilling.")
				Error.Println(e)
				return errors.New(e)
			}
		}
	} else {
		var e string
		e = fmt.Sprintf("Wrong number of paramters given.")
		Error.Println(e)
		Error.Printf("Try '%s help fill' for more information",
			path.Base(os.Args[0]))
		return errors.New(e)
	}
	return nil
}

func init() {
	Commands = append(Commands, cli.Command{
		Name:        "fill",
		Aliases:     []string{"f"},
		Usage:       "Backfill datapoints in the dst from the src",
		Description: "Backfill datapoints in the dst from the src",
		ArgsUsage:   "<src(File|Dir)> <dst(File|Dir)>",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "j",
				Usage: "Number of workers (for directory recursion)",
				Value: runtime.GOMAXPROCS(0),
			},
			cli.BoolFlag{Name: "c", Usage: "Prevent colors from being used"},
		},
		SkipFlagParsing: false,
		HideHelp:        false,
		Hidden:          false,
		Action:          Filler,
	})
}
<commit_msg>more informative progress output<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"path"
	"runtime"
	"time"

	. "github.com/fuzzy/gocolor"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/urfave/cli"
)

var (
	Filled  int64
	Overlap int64
)

func ProgressIndicator(c int64, t int64) {
	rtime := (time.Now().Unix() - StartTime)
	fmt.Printf("%s %d of %d (%6.02f%%) runtime: %dsec @ %d/sec\r",
		String("Info").Green().Bold(),
		c,
		t,
		(float64(c)/float64(t))*100.00,
		rtime,
		(Filled / rtime))
}

func fillWorker(d chan map[string]string) {
	for {
		msg := <-d
		if msg["SRC"] == "__EXIT__" && msg["DST"] == "__EXIT__" {
			return
		} else {
			if isFile(msg["SRC"]) && isFile(msg["DST"]) {
				if !backfillFile(msg["SRC"], msg["DST"]) {
					Error.Printf("S:%s D:%s - Backfill operation failed.",
						msg["SRC"],
						msg["DST"])
					os.Exit(1)
				} else {
					Filled++
					ProgressIndicator(Filled, Overlap)
				}
			}
		}
	}
}

func backfillFile(s string, d string) bool {
	// Open our filehandles
	sDb, sErr := whisper.Open(s)
	dDb, dErr := whisper.Open(d)
	if !chkErr(sErr) || !chkErr(dErr) {
		os.Exit(1)
	}
	// Defer their closings
	defer sDb.Close()
	defer dDb.Close()

	// Now for a series of checks, first to ensure that both
	// files have the same number of archives in them.
	if sDb.Header.Metadata.ArchiveCount != dDb.Header.Metadata.ArchiveCount {
		Error.Println("The files have a mismatched set of archives.")
		return false
	}

	// Now we'll start processing the archives, checking as we go to see if they
	// are matched. That way we at least fill in what we can, possibly....
	for i, a := range sDb.Header.Archives {
		// The offset
		if a.Offset == dDb.Header.Archives[i].Offset {
			// and the number of points
			if a.Points == dDb.Header.Archives[i].Points {
				// and finally the interval
				if a.SecondsPerPoint == dDb.Header.Archives[i].SecondsPerPoint {
					// ok, now let's get rolling through the archives
					sp, se := sDb.DumpArchive(i)
					if se != nil {
						Error.Println(se.Error())
						os.Exit(1)
					}
					dp, de := dDb.DumpArchive(i)
					if de != nil {
						Error.Println(de.Error())
						os.Exit(1)
					}
					for idx := 0; idx < len(sp); idx++ {
						if sp[idx].Timestamp != 0 && sp[idx].Value != 0 {
							if dp[idx].Timestamp == 0 || dp[idx].Value == 0 {
								dp[idx].Timestamp = sp[idx].Timestamp
								dp[idx].Value = sp[idx].Value
								dDb.Update(dp[idx])
							}
						}
					}
				}
			}
		}
	}
	return true
}

func Filler(c *cli.Context) error {
	if len(c.Args()) == 2 {
		args := c.Args()
		if isDir(args[0]) && isDir(args[1]) {
			Filled = 0

			// First things first, let's spawn our pool of workers.
			dataCh := make(chan map[string]string, 1)
			for i := 0; i < c.Int("j"); i++ {
				go fillWorker(dataCh)
			}

			// Let's get this dir walking in there then shall we?
			srcFiles := listFiles(args[0])
			dstFiles := listFiles(args[1])

			// Now let's find all our overlap
			overlap := []string{}
			for a := 0; a < len(srcFiles); a++ {
				for b := 0; b < len(dstFiles); b++ {
					if srcFiles[a] == dstFiles[b] {
						overlap = append(overlap, srcFiles[a])
					}
				}
			}

			// And display some stats
			Info.Printf("srcDir: %d files, dstDir: %d files, %d overlap.",
				len(srcFiles),
				len(dstFiles),
				len(overlap))
			Overlap = int64(len(overlap))

			// Now we can push in all our data, and let our workers do their
			// lovely little thing. Ahhhhh concurrency.
			StartTime = time.Now().Unix()
			for i := 0; i < len(overlap); i++ {
				dataCh <- map[string]string{
					"SRC": fmt.Sprintf("%s/%s", args[0], overlap[i]),
					"DST": fmt.Sprintf("%s/%s", args[1], overlap[i]),
				}
			}

			// And while they're off doing that, here we will just sit and watch
			// the return channel and count things. Once we have all our
			// backfill operations accounted for, we can reap all of our workers
			// and carry on.
			for Filled < int64(len(overlap)) {
				time.Sleep(1 * time.Second)
			}
			// And finally let's reap all our children
			for idx := 0; idx < c.Int("j"); idx++ {
				dataCh <- map[string]string{
					"SRC": "__EXIT__",
					"DST": "__EXIT__",
				}
			}

			fmt.Println("")
			Info.Println((Overlap / (time.Now().Unix() - StartTime)),
				"whisper files processed per second.")
		} else {
			if !backfillFile(args[0], args[1]) {
				e := fmt.Sprintf("Error while backfilling.")
				Error.Println(e)
				return errors.New(e)
			}
		}
	} else {
		var e string
		e = fmt.Sprintf("Wrong number of paramters given.")
		Error.Println(e)
		Error.Printf("Try '%s help fill' for more information",
			path.Base(os.Args[0]))
		return errors.New(e)
	}
	return nil
}

func init() {
	Commands = append(Commands, cli.Command{
		Name:        "fill",
		Aliases:     []string{"f"},
		Usage:       "Backfill datapoints in the dst from the src",
		Description: "Backfill datapoints in the dst from the src",
		ArgsUsage:   "<src(File|Dir)> <dst(File|Dir)>",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "j",
				Usage: "Number of workers (for directory recursion)",
				Value: runtime.GOMAXPROCS(0),
			},
			cli.BoolFlag{Name: "c", Usage: "Prevent colors from being used"},
		},
		SkipFlagParsing: false,
		HideHelp:        false,
		Hidden:          false,
		Action:          Filler,
	})
}
<|endoftext|>


<commit_before>package cli

import (
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"
)

// This flag enables bash-completion for all commands and subcommands
var BashCompletionFlag = BoolFlag{
	Name: "generate-bash-completion",
}

// This flag prints the version for the application
var VersionFlag = BoolFlag{
	Name:  "version, v",
	Usage: "print the version",
}

// This flag prints the help for all commands and subcommands
// Set to the zero value (BoolFlag{}) to disable flag -- keeps subcommand
// unless HideHelp is set to true)
var HelpFlag = BoolFlag{
	Name:  "help, h",
	Usage: "show help",
}

// Flag is a common interface related to parsing flags in cli.
// For more advanced flag parsing techniques, it is recomended that
// this interface be implemented.
type Flag interface {
	fmt.Stringer
	// Apply Flag settings to the given flag set
	Apply(*flag.FlagSet)
	getName() string
}

func flagSet(name string, flags []Flag) *flag.FlagSet {
	set := flag.NewFlagSet(name, flag.ContinueOnError)

	for _, f := range flags {
		f.Apply(set)
	}
	return set
}

func eachName(longName string, fn func(string)) {
	parts := strings.Split(longName, ",")
	for _, name := range parts {
		name = strings.Trim(name, " ")
		fn(name)
	}
}

// Generic is a generic parseable type identified by a specific flag
type Generic interface {
	Set(value string) error
	String() string
}

// GenericFlag is the flag type for types implementing Generic
type GenericFlag struct {
	Name   string
	Value  Generic
	Usage  string
	EnvVar string
}

// String returns the string representation of the generic flag to display the
// help text to the user (uses the String() method of the generic flag to show
// the value)
func (f GenericFlag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s%s \"%v\"\t%v", prefixFor(f.Name), f.Name, f.Value, f.Usage))
}

// Apply takes the flagset and calls Set on the generic flag with the value
// provided by the user for parsing by the flag
func (f GenericFlag) Apply(set *flag.FlagSet) {
	val := f.Value
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				val.Set(envVal)
				break
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Var(f.Value, name, f.Usage)
	})
}

func (f GenericFlag) getName() string {
	return f.Name
}

type StringSlice []string

func (f *StringSlice) Set(value string) error {
	*f = append(*f, value)
	return nil
}

func (f *StringSlice) String() string {
	return fmt.Sprintf("%s", *f)
}

func (f *StringSlice) Value() []string {
	return *f
}

type StringSliceFlag struct {
	Name   string
	Value  *StringSlice
	Usage  string
	EnvVar string
}

func (f StringSliceFlag) String() string {
	firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
	pref := prefixFor(firstName)
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
}

func (f StringSliceFlag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				newVal := &StringSlice{}
				for _, s := range strings.Split(envVal, ",") {
					s = strings.TrimSpace(s)
					newVal.Set(s)
				}
				f.Value = newVal
				break
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Var(f.Value, name, f.Usage)
	})
}

func (f StringSliceFlag) getName() string {
	return f.Name
}

type IntSlice []int

func (f *IntSlice) Set(value string) error {

	tmp, err := strconv.Atoi(value)
	if err != nil {
		return err
	} else {
		*f = append(*f, tmp)
	}
	return nil
}

func (f *IntSlice) String() string {
	return fmt.Sprintf("%d", *f)
}

func (f *IntSlice) Value() []int {
	return *f
}

type IntSliceFlag struct {
	Name   string
	Value  *IntSlice
	Usage  string
	EnvVar string
}

func (f IntSliceFlag) String() string {
	firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
	pref := prefixFor(firstName)
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
}

func (f IntSliceFlag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				newVal := &IntSlice{}
				for _, s := range strings.Split(envVal, ",") {
					s = strings.TrimSpace(s)
					err := newVal.Set(s)
					if err != nil {
						fmt.Fprintf(os.Stderr, err.Error())
					}
				}
				f.Value = newVal
				break
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Var(f.Value, name, f.Usage)
	})
}

func (f IntSliceFlag) getName() string {
	return f.Name
}

type BoolFlag struct {
	Name   string
	Usage  string
	EnvVar string
}

func (f BoolFlag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage))
}

func (f BoolFlag) Apply(set *flag.FlagSet) {
	val := false
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				envValBool, err := strconv.ParseBool(envVal)
				if err == nil {
					val = envValBool
				}
				break
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Bool(name, val, f.Usage)
	})
}

func (f BoolFlag) getName() string {
	return f.Name
}

type BoolTFlag struct {
	Name   string
	Usage  string
	EnvVar string
}

func (f BoolTFlag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage))
}

func (f BoolTFlag) Apply(set *flag.FlagSet) {
	val := true
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				envValBool, err := strconv.ParseBool(envVal)
				if err == nil {
					val = envValBool
					break
				}
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Bool(name, val, f.Usage)
	})
}

func (f BoolTFlag) getName() string {
	return f.Name
}

type StringFlag struct {
	Name   string
	Value  string
	Usage  string
	EnvVar string
}

func (f StringFlag) String() string {
	var fmtString string
	fmtString = "%s %v\t%v"

	if len(f.Value) > 0 {
		fmtString = "%s \"%v\"\t%v"
	} else {
		fmtString = "%s %v\t%v"
	}

	return withEnvHint(f.EnvVar, fmt.Sprintf(fmtString, prefixedNames(f.Name), f.Value, f.Usage))
}

func (f StringFlag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				f.Value = envVal
				break
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.String(name, f.Value, f.Usage)
	})
}

func (f StringFlag) getName() string {
	return f.Name
}

type IntFlag struct {
	Name   string
	Value  int
	Usage  string
	EnvVar string
}

func (f IntFlag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}

func (f IntFlag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				envValInt, err := strconv.ParseInt(envVal, 10, 64)
				if err == nil {
					f.Value = int(envValInt)
					break
				}
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Int(name, f.Value, f.Usage)
	})
}

func (f IntFlag) getName() string {
	return f.Name
}

type DurationFlag struct {
	Name   string
	Value  time.Duration
	Usage  string
	EnvVar string
}

func (f DurationFlag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}

func (f DurationFlag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				envValDuration, err := time.ParseDuration(envVal)
				if err == nil {
					f.Value = envValDuration
					break
				}
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Duration(name, f.Value, f.Usage)
	})
}

func (f DurationFlag) getName() string {
	return f.Name
}

type Float64Flag struct {
	Name   string
	Value  float64
	Usage  string
	EnvVar string
}

func (f Float64Flag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}

func (f Float64Flag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				envValFloat, err := strconv.ParseFloat(envVal, 10)
				if err == nil {
					f.Value = float64(envValFloat)
				}
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Float64(name, f.Value, f.Usage)
	})
}

func (f Float64Flag) getName() string {
	return f.Name
}

func prefixFor(name string) (prefix string) {
	if len(name) == 1 {
		prefix = "-"
	} else {
		prefix = "--"
	}

	return
}

func prefixedNames(fullName string) (prefixed string) {
	parts := strings.Split(fullName, ",")
	for i, name := range parts {
		name = strings.Trim(name, " ")
		prefixed += prefixFor(name) + name
		if i < len(parts)-1 {
			prefixed += ", "
		}
	}
	return
}

func withEnvHint(envVar, str string) string {
	envText := ""
	if envVar != "" {
		envText = fmt.Sprintf(" [$%s]", strings.Join(strings.Split(envVar, ","), ", $"))
	}
	return str + envText
}
<commit_msg>Use 0 as the base when parsing ints<commit_after>package cli

import (
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"
)

// This flag enables bash-completion for all commands and subcommands
var BashCompletionFlag = BoolFlag{
	Name: "generate-bash-completion",
}

// This flag prints the version for the application
var VersionFlag = BoolFlag{
	Name:  "version, v",
	Usage: "print the version",
}

// This flag prints the help for all commands and subcommands
// Set to the zero value (BoolFlag{}) to disable flag -- keeps subcommand
// unless HideHelp is set to true)
var HelpFlag = BoolFlag{
	Name:  "help, h",
	Usage: "show help",
}

// Flag is a common interface related to parsing flags in cli.
// For more advanced flag parsing techniques, it is recomended that
// this interface be implemented.
type Flag interface {
	fmt.Stringer
	// Apply Flag settings to the given flag set
	Apply(*flag.FlagSet)
	getName() string
}

func flagSet(name string, flags []Flag) *flag.FlagSet {
	set := flag.NewFlagSet(name, flag.ContinueOnError)

	for _, f := range flags {
		f.Apply(set)
	}
	return set
}

func eachName(longName string, fn func(string)) {
	parts := strings.Split(longName, ",")
	for _, name := range parts {
		name = strings.Trim(name, " ")
		fn(name)
	}
}

// Generic is a generic parseable type identified by a specific flag
type Generic interface {
	Set(value string) error
	String() string
}

// GenericFlag is the flag type for types implementing Generic
type GenericFlag struct {
	Name   string
	Value  Generic
	Usage  string
	EnvVar string
}

// String returns the string representation of the generic flag to display the
// help text to the user (uses the String() method of the generic flag to show
// the value)
func (f GenericFlag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s%s \"%v\"\t%v", prefixFor(f.Name), f.Name, f.Value, f.Usage))
}

// Apply takes the flagset and calls Set on the generic flag with the value
// provided by the user for parsing by the flag
func (f GenericFlag) Apply(set *flag.FlagSet) {
	val := f.Value
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				val.Set(envVal)
				break
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Var(f.Value, name, f.Usage)
	})
}

func (f GenericFlag) getName() string {
	return f.Name
}

type StringSlice []string

func (f *StringSlice) Set(value string) error {
	*f = append(*f, value)
	return nil
}

func (f *StringSlice) String() string {
	return fmt.Sprintf("%s", *f)
}

func (f *StringSlice) Value() []string {
	return *f
}

type StringSliceFlag struct {
	Name   string
	Value  *StringSlice
	Usage  string
	EnvVar string
}

func (f StringSliceFlag) String() string {
	firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
	pref := prefixFor(firstName)
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
}

func (f StringSliceFlag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				newVal := &StringSlice{}
				for _, s := range strings.Split(envVal, ",") {
					s = strings.TrimSpace(s)
					newVal.Set(s)
				}
				f.Value = newVal
				break
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Var(f.Value, name, f.Usage)
	})
}

func (f StringSliceFlag) getName() string {
	return f.Name
}

type IntSlice []int

func (f *IntSlice) Set(value string) error {

	tmp, err := strconv.Atoi(value)
	if err != nil {
		return err
	} else {
		*f = append(*f, tmp)
	}
	return nil
}

func (f *IntSlice) String() string {
	return fmt.Sprintf("%d", *f)
}

func (f *IntSlice) Value() []int {
	return *f
}

type IntSliceFlag struct {
	Name   string
	Value  *IntSlice
	Usage  string
	EnvVar string
}

func (f IntSliceFlag) String() string {
	firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
	pref := prefixFor(firstName)
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
}

func (f IntSliceFlag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				newVal := &IntSlice{}
				for _, s := range strings.Split(envVal, ",") {
					s = strings.TrimSpace(s)
					err := newVal.Set(s)
					if err != nil {
						fmt.Fprintf(os.Stderr, err.Error())
					}
				}
				f.Value = newVal
				break
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Var(f.Value, name, f.Usage)
	})
}

func (f IntSliceFlag) getName() string {
	return f.Name
}

type BoolFlag struct {
	Name   string
	Usage  string
	EnvVar string
}

func (f BoolFlag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage))
}

func (f BoolFlag) Apply(set *flag.FlagSet) {
	val := false
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				envValBool, err := strconv.ParseBool(envVal)
				if err == nil {
					val = envValBool
				}
				break
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Bool(name, val, f.Usage)
	})
}

func (f BoolFlag) getName() string {
	return f.Name
}

type BoolTFlag struct {
	Name   string
	Usage  string
	EnvVar string
}

func (f BoolTFlag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage))
}

func (f BoolTFlag) Apply(set *flag.FlagSet) {
	val := true
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				envValBool, err := strconv.ParseBool(envVal)
				if err == nil {
					val = envValBool
					break
				}
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Bool(name, val, f.Usage)
	})
}

func (f BoolTFlag) getName() string {
	return f.Name
}

type StringFlag struct {
	Name   string
	Value  string
	Usage  string
	EnvVar string
}

func (f StringFlag) String() string {
	var fmtString string
	fmtString = "%s %v\t%v"

	if len(f.Value) > 0 {
		fmtString = "%s \"%v\"\t%v"
	} else {
		fmtString = "%s %v\t%v"
	}

	return withEnvHint(f.EnvVar, fmt.Sprintf(fmtString, prefixedNames(f.Name), f.Value, f.Usage))
}

func (f StringFlag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				f.Value = envVal
				break
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.String(name, f.Value, f.Usage)
	})
}

func (f StringFlag) getName() string {
	return f.Name
}

type IntFlag struct {
	Name   string
	Value  int
	Usage  string
	EnvVar string
}

func (f IntFlag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}

func (f IntFlag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				envValInt, err := strconv.ParseInt(envVal, 0, 64)
				if err == nil {
					f.Value = int(envValInt)
					break
				}
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Int(name, f.Value, f.Usage)
	})
}

func (f IntFlag) getName() string {
	return f.Name
}

type DurationFlag struct {
	Name   string
	Value  time.Duration
	Usage  string
	EnvVar string
}

func (f DurationFlag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}

func (f DurationFlag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				envValDuration, err := time.ParseDuration(envVal)
				if err == nil {
					f.Value = envValDuration
					break
				}
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Duration(name, f.Value, f.Usage)
	})
}

func (f DurationFlag) getName() string {
	return f.Name
}

type Float64Flag struct {
	Name   string
	Value  float64
	Usage  string
	EnvVar string
}

func (f Float64Flag) String() string {
	return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}

func (f Float64Flag) Apply(set *flag.FlagSet) {
	if f.EnvVar != "" {
		for _, envVar := range strings.Split(f.EnvVar, ",") {
			envVar = strings.TrimSpace(envVar)
			if envVal := os.Getenv(envVar); envVal != "" {
				envValFloat, err := strconv.ParseFloat(envVal, 10)
				if err == nil {
					f.Value = float64(envValFloat)
				}
			}
		}
	}

	eachName(f.Name, func(name string) {
		set.Float64(name, f.Value, f.Usage)
	})
}

func (f Float64Flag) getName() string {
	return f.Name
}

func prefixFor(name string) (prefix string) {
	if len(name) == 1 {
		prefix = "-"
	} else {
		prefix = "--"
	}

	return
}

func prefixedNames(fullName string) (prefixed string) {
	parts := strings.Split(fullName, ",")
	for i, name := range parts {
		name = strings.Trim(name, " ")
		prefixed += prefixFor(name) + name
		if i < len(parts)-1 {
			prefixed += ", "
		}
	}
	return
}

func withEnvHint(envVar, str string) string {
	envText := ""
	if envVar != "" {
		envText = fmt.Sprintf(" [$%s]", strings.Join(strings.Split(envVar, ","), ", $"))
	}
	return str + envText
}
<|endoftext|>


<commit_before>package goptions

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

// Flag represents a single flag of a FlagSet.
type Flag struct {
	Short        string
	Long         string
	MutexGroups  []string
	Description  string
	Obligatory   bool
	WasSpecified bool
	value        reflect.Value
}

// Return the name of the flag preceding the right amount of dashes.
// The long name is preferred. If no name has been specified, "<unspecified>"
// will be returned.
func (f *Flag) Name() string {
	if len(f.Long) > 0 {
		return "--" + f.Long
	}
	if len(f.Short) > 0 {
		return "-" + f.Short
	}
	return "<unspecified>"
}

// NeedsExtraValue returns true if the flag expects a separate value.
func (f *Flag) NeedsExtraValue() bool {
	// Explicit over implicit
	if f.value.Type() == reflect.TypeOf(new(bool)).Elem() {
		return false
	}
	if _, ok := f.value.Interface().(Help); ok {
		return false
	}
	return true
}

// IsMulti returns true if the flag can be specified multiple times.
func (f *Flag) IsMulti() bool {
	if f.value.Kind() == reflect.Slice {
		return true
	}
	return false
}

func isShort(arg string) bool {
	return strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "--")
}

func isLong(arg string) bool {
	return strings.HasPrefix(arg, "--")
}

func (f *Flag) Handles(arg string) bool {
	return (isShort(arg) && arg[1:2] == f.Short) ||
		(isLong(arg) && arg[2:] == f.Long)

}

func (f *Flag) Parse(args []string) ([]string, error) {
	param, value := args[0], ""
	if f.NeedsExtraValue() &&
		(len(args) < 2 || (isShort(param) && len(param) > 2)) {
		return args, fmt.Errorf("Flag %s needs an argument", f.Name())
	}
	if f.WasSpecified && !f.IsMulti() {
		return args, fmt.Errorf("Flag %s can only be specified once", f.Name())
	}
	if isShort(param) && len(param) > 2 {
		// Short flag cluster
		args[0] = "-" + param[2:]
	} else if f.NeedsExtraValue() {
		value = args[1]
		args = args[2:]
	} else {
		args = args[1:]
	}
	f.WasSpecified = true
	return args, f.setValue(value)
}

type valueParser func(v reflect.Value, val string) error

var (
	parserMap = map[reflect.Type]valueParser{
		reflect.TypeOf(new(bool)).Elem():   boolValueParser,
		reflect.TypeOf(new(string)).Elem(): stringValueParser,
		reflect.TypeOf(new(int)).Elem():    intValueParser,
		reflect.TypeOf(new(Help)).Elem():   helpValueParser,
	}
)

func (f *Flag) setValue(s string) (err error) {
	defer func() {
		if x := recover(); x != nil {
			err = x.(error)
			return
		}
	}()
	if _, ok := f.value.Interface().(Marshaler); ok {
		newval := reflect.New(f.value.Type()).Elem()
		if newval.Kind() == reflect.Ptr {
			newptrval := reflect.New(f.value.Type().Elem())
			newval.Set(newptrval)
		}
		err := newval.Interface().(Marshaler).MarshalGoption(s)
		f.value.Set(newval)
		return err
	}
	if parser, ok := parserMap[f.value.Type()]; ok {
		return parser(f.value, s)
	} else {
		return fmt.Errorf("Unsupported flag type: %s", f.value.Type().Name())
	}
	panic("Invalid execution path")
}

func boolValueParser(v reflect.Value, val string) error {
	v.Set(reflect.ValueOf(true))
	return nil
}

func stringValueParser(v reflect.Value, val string) error {
	v.Set(reflect.ValueOf(val))
	return nil
}

func intValueParser(v reflect.Value, val string) error {
	intval, err := strconv.ParseInt(val, 10, 64)
	if err != nil {
		return err
	}
	v.Set(reflect.ValueOf(int(intval)))
	return nil
}

func helpValueParser(v reflect.Value, val string) error {
	return ErrHelpRequest
}
<commit_msg>Implement array support<commit_after>package goptions

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

// Flag represents a single flag of a FlagSet.
type Flag struct {
	Short        string
	Long         string
	MutexGroups  []string
	Description  string
	Obligatory   bool
	WasSpecified bool
	value        reflect.Value
}

// Return the name of the flag preceding the right amount of dashes.
// The long name is preferred. If no name has been specified, "<unspecified>"
// will be returned.
func (f *Flag) Name() string {
	if len(f.Long) > 0 {
		return "--" + f.Long
	}
	if len(f.Short) > 0 {
		return "-" + f.Short
	}
	return "<unspecified>"
}

// NeedsExtraValue returns true if the flag expects a separate value.
func (f *Flag) NeedsExtraValue() bool {
	// Explicit over implicit
	if f.value.Type() == reflect.TypeOf(new(bool)).Elem() {
		return false
	}
	if _, ok := f.value.Interface().(Help); ok {
		return false
	}
	return true
}

// IsMulti returns true if the flag can be specified multiple times.
func (f *Flag) IsMulti() bool {
	if f.value.Kind() == reflect.Slice {
		return true
	}
	return false
}

func isShort(arg string) bool {
	return strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "--")
}

func isLong(arg string) bool {
	return strings.HasPrefix(arg, "--")
}

func (f *Flag) Handles(arg string) bool {
	return (isShort(arg) && arg[1:2] == f.Short) ||
		(isLong(arg) && arg[2:] == f.Long)

}

func (f *Flag) Parse(args []string) ([]string, error) {
	param, value := args[0], ""
	if f.NeedsExtraValue() &&
		(len(args) < 2 || (isShort(param) && len(param) > 2)) {
		return args, fmt.Errorf("Flag %s needs an argument", f.Name())
	}
	if f.WasSpecified && !f.IsMulti() {
		return args, fmt.Errorf("Flag %s can only be specified once", f.Name())
	}
	if isShort(param) && len(param) > 2 {
		// Short flag cluster
		args[0] = "-" + param[2:]
	} else if f.NeedsExtraValue() {
		value = args[1]
		args = args[2:]
	} else {
		args = args[1:]
	}
	f.WasSpecified = true
	return args, f.setValue(value)
}

type valueParser func(val string) (reflect.Value, error)

var (
	parserMap = map[reflect.Type]valueParser{
		reflect.TypeOf(new(bool)).Elem():   boolValueParser,
		reflect.TypeOf(new(string)).Elem(): stringValueParser,
		reflect.TypeOf(new(int)).Elem():    intValueParser,
		reflect.TypeOf(new(Help)).Elem():   helpValueParser,
	}
)

func (f *Flag) setValue(s string) (err error) {
	defer func() {
		if x := recover(); x != nil {
			err = x.(error)
			return
		}
	}()
	if _, ok := f.value.Interface().(Marshaler); ok {
		newval := reflect.New(f.value.Type()).Elem()
		if newval.Kind() == reflect.Ptr {
			newptrval := reflect.New(f.value.Type().Elem())
			newval.Set(newptrval)
		}
		err := newval.Interface().(Marshaler).MarshalGoption(s)
		f.value.Set(newval)
		return err
	}
	vtype := f.value.Type()
	if f.value.Kind() == reflect.Slice {
		vtype = f.value.Type().Elem()
	}
	if parser, ok := parserMap[vtype]; ok {
		val, err := parser(s)
		if err != nil {
			return err
		}
		if f.value.Kind() == reflect.Slice {
			f.value.Set(reflect.Append(f.value, val))
		} else {
			f.value.Set(val)
		}
		return nil
	} else {
		return fmt.Errorf("Unsupported flag type: %s", f.value.Type().Name())
	}
	panic("Invalid execution path")
}

func boolValueParser(val string) (reflect.Value, error) {
	return reflect.ValueOf(true), nil
}

func stringValueParser(val string) (reflect.Value, error) {
	return reflect.ValueOf(val), nil
}

func intValueParser(val string) (reflect.Value, error) {
	intval, err := strconv.ParseInt(val, 10, 64)
	return reflect.ValueOf(int(intval)), err
}

func helpValueParser(val string) (reflect.Value, error) {
	return reflect.Value{}, ErrHelpRequest
}
<|endoftext|>


<commit_before>package httphandlers

import (
	"encoding/json"
	"github.com/Financial-Times/service-status-go/buildinfo"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"net/http"
)

//BuildInfoHandlerFunc is a HandlerFunc that returns a JSON representation of the build-info.
func BuildInfoHandlerFunc(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	if err := json.NewEncoder(w).Encode(buildinfo.GetBuildInfo()); err != nil {
		panic(err)
	}
}

//AppendBuildInfoHandler appends the BuildInfoHandler to a gorilla/mux Router.
//It enforces to respond to HTTP GETs only.
func AppendBuildInfoHandler(r *mux.Router) {
	r.Path("/build-info").Handler(handlers.MethodHandler{"GET": http.HandlerFunc(BuildInfoHandlerFunc)})
}
<commit_msg>Created a generic http/Handlder for build-info<commit_after>package httphandlers

import (
	"encoding/json"
	"github.com/Financial-Times/service-status-go/buildinfo"
	"net/http"
)

type FtHandler func(http.ResponseWriter, *http.Request)

//BuildInfoHandlerFunc is a HandlerFunc that returns a JSON representation of the build-info.
func BuildInfoHandlerFunc(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	if err := json.NewEncoder(w).Encode(buildinfo.GetBuildInfo()); err != nil {
		panic(err)
	}
}

func (f FtHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		f(w, r)
	} else {
		w.Header().Set("Allow", "GET")
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
	}
}
<|endoftext|>


<commit_before>// ratt operates on a Debian .changes file of a just-built package, identifies
// all reverse-build-dependencies and rebuilds them with the .debs from the
// .changes file.
//
// The intended use-case is, for example, to package a new snapshot of a Go
// library and verify that the new version does not break any other Go
// libraries/binaries.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"pault.ag/go/debian/control"
	"pault.ag/go/debian/version"
)

var (
	logDir = flag.String("log_dir",
		"buildlogs",
		"Directory in which sbuild(1) logs for all reverse-build-dependencies are stored")

	dryRun = flag.Bool("dry_run",
		false,
		"Print sbuild command lines, but do not actually build the reverse-build-dependencies")

	sbuildDist = flag.String("sbuild_dist",
		"",
		"sbuild --dist= value (e.g. \"sid\"). Defaults to the Distribution: entry from the specified .changes file")

	dist = flag.String("dist",
		"",
		"Distribution to look up reverse-build-dependencies from. Defaults to the Distribution: entry from the specified .changes file")

	listsPrefixRe = regexp.MustCompile(`/([^/]*_dists_.*)_InRelease$`)
)

func dependsOn(src control.SourceIndex, binaries map[string]bool) bool {
	buildDepends := src.GetBuildDepends()
	for _, possibility := range buildDepends.GetAllPossibilities() {
		if binaries[possibility.Name] {
			return true
		}
	}
	return false
}

func addReverseBuildDeps(sourcesPath string, binaries map[string]bool, rebuild map[string][]version.Version) error {
	log.Printf("Loading sources index %q\n", sourcesPath)
	catFile := exec.Command("/usr/lib/apt/apt-helper",
		"cat-file",
		sourcesPath)
	var s *bufio.Reader
	if lines, err := catFile.Output(); err == nil {
		s = bufio.NewReader(strings.NewReader(string(lines)))
	} else {
		// Fallback for older versions of apt-get. See
		// <20160111171230.GA17291@debian.org> for context.
		o, err := os.Open(sourcesPath)
		if err != nil {
			return err
		}
		defer o.Close()
		s = bufio.NewReader(o)
	}
	idx, err := control.ParseSourceIndex(s)
	if err != nil && err != io.EOF {
		return err
	}

	for _, src := range idx {
		if dependsOn(src, binaries) {
			rebuild[src.Package] = append(rebuild[src.Package], src.Version)
		}
	}

	return nil
}

func main() {
	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatalf("Usage: %s [options] <path-to-changes-file>\n", os.Args[0])
	}

	changesPath := flag.Arg(0)
	log.Printf("Loading changes file %q\n", changesPath)

	c, err := os.Open(changesPath)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()
	changes, err := control.ParseChanges(bufio.NewReader(c), changesPath)
	if err != nil && err != io.EOF {
		log.Fatal(err)
	}

	binaries := make(map[string]bool)
	var debs []string
	for _, file := range changes.Files {
		if filepath.Ext(file.Filename) == ".deb" {
			debs = append(debs, file.Filename)
		}
	}
	for _, binary := range changes.Binaries {
		binaries[binary] = true
	}

	log.Printf(" - %d binary packages: %s\n", len(changes.Binaries), strings.Join(changes.Binaries, " "))
	log.Printf(" - corresponding .debs (will be injected when building):\n")
	for _, deb := range debs {
		log.Printf("    %s\n", deb)
	}

	if strings.TrimSpace(*dist) == "" {
		*dist = changes.Distribution
		// Rewrite unstable to sid, which apt-get indextargets (below) requires.
		if *dist == "unstable" {
			*dist = "sid"
		}
		log.Printf("Setting -dist=%s (from .changes file)\n", *dist)
	}

	var sourcesPaths []string
	var packagesPaths []string
	indexTargets := exec.Command("apt-get",
		"indextargets",
		"--format",
		"$(FILENAME)",
		"Codename: " + *dist,
		"ShortDesc: Sources")
	if lines, err := indexTargets.Output(); err == nil {
		for _, line := range strings.Split(string(lines), "\n") {
			trimmed := strings.TrimSpace(line)
			if trimmed != "" {
				sourcesPaths = append(sourcesPaths, line)
			}
		}
		binaryIndexTargets := exec.Command(
			"apt-get",
			"indextargets",
			"--format",
			"$(FILENAME)",
			"Codename: " + *dist,
			"ShortDesc: Packages")
		lines, err = binaryIndexTargets.Output()
		if err != nil {
			log.Fatal("Could not get packages files using %+v: %v", binaryIndexTargets.Args, err)
		}
		for _, line := range strings.Split(string(lines), "\n") {
			trimmed := strings.TrimSpace(line)
			if trimmed != "" {
				packagesPaths = append(packagesPaths, line)
			}
		}
	} else {
		// Fallback for older versions of apt-get. See
		// https://bugs.debian.org/801594 for context.
		releaseMatches, err := filepath.Glob("/var/lib/apt/lists/*_InRelease")
		if err != nil {
			log.Fatal(err)
		}
		for _, releasepath := range releaseMatches {
			r, err := os.Open(releasepath)
			if err != nil {
				log.Fatal(err)
			}
			defer r.Close()
			var inRelease struct {
				Suite string
			}
			if err := control.Unmarshal(&inRelease, bufio.NewReader(r)); err != nil {
				log.Fatal(err)
			}

			listsPrefix := listsPrefixRe.FindStringSubmatch(releasepath)
			if len(listsPrefix) != 2 {
				log.Fatalf("release file path %q does not match regexp %q\n", releasepath, listsPrefixRe)
			}
			sourceMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Sources", listsPrefix[1]))
			if err != nil {
				log.Fatal(err)
			}
			sourcesPaths = append(sourcesPaths, sourceMatches...)
			packagesMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Packages", listsPrefix[1]))
			if err != nil {
				log.Fatal(err)
			}
			packagesPaths = append(packagesPaths, packagesMatches...)
		}
	}

	if len(sourcesPaths) == 0 {
		log.Fatal("Could not find InRelease file for " + *dist + " . Are you missing " + *dist + " in your /etc/apt/sources.list?")
	}

	rebuild := make(map[string][]version.Version)

	archCmd := exec.Command(
		"dpkg-architecture",
		"--query=DEB_BUILD_ARCH")
	archOut, err := archCmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	arch := strings.TrimSpace(string(archOut))

	// TODO: Cache this output based on the .changes file. dose-ceve takes quite a while.
	ceve := exec.Command(
		"dose-ceve",
		"--deb-native-arch="+arch,
		"-T", "debsrc",
		"-r", strings.Join(changes.Binaries, ","),
		"-G", "pkg")
	for _, packagesPath := range packagesPaths {
		ceve.Args = append(ceve.Args, "deb://"+packagesPath)
	}
	for _, sourcesPath := range sourcesPaths {
		ceve.Args = append(ceve.Args, "debsrc://"+sourcesPath)
	}

	log.Printf("Figuring out reverse build dependencies using dose-ceve(1). This might take a while\n")
	if out, err := ceve.Output(); err == nil {
		var doseCeves []struct {
			Package string
			Version version.Version
		}
		r := bufio.NewReader(strings.NewReader(string(out)))
		if err := control.Unmarshal(&doseCeves, r); err != nil {
			log.Fatal(err)
		}
		for _, doseCeve := range doseCeves {
			rebuild[doseCeve.Package] = append(rebuild[doseCeve.Package], doseCeve.Version)
		}
	} else {
		log.Printf("dose-ceve(1) failed (%v), falling back to interpreting Sources directly\n", err)
		for _, sourcesPath := range sourcesPaths {
			if err := addReverseBuildDeps(sourcesPath, binaries, rebuild); err != nil {
				log.Fatal(err)
			}
		}
	}

	// TODO: add -recursive flag to also cover dependencies which are not DIRECT dependencies. use http://godoc.org/pault.ag/go/debian/control#OrderDSCForBuild (topsort) to build dependencies in the right order (saving CPU time).

	// TODO: what’s a good integration method for doing this in more setups, e.g. on a cloud provider or something? mapreri from #debian-qa says jenkins.debian.net is suitable.

	if strings.TrimSpace(*sbuildDist) == "" {
		*sbuildDist = changes.Distribution
		log.Printf("Setting -sbuild_dist=%s (from .changes file)\n", *sbuildDist)
	}

	cnt := 1
	buildresults := make(map[string]bool)
	for src, versions := range rebuild {
		sort.Sort(sort.Reverse(version.Slice(versions)))
		newest := versions[0]
		target := fmt.Sprintf("%s_%s", src, newest)
		// TODO: discard resulting package immediately?
		args := []string{
			"--arch-all",
			"--dist=" + *sbuildDist,
			"--nolog",
			target,
		}
		for _, filename := range debs {
			args = append(args, fmt.Sprintf("--extra-package=%s", filename))
		}
		cmd := exec.Command("sbuild", args...)
		if err := os.MkdirAll(*logDir, 0755); err != nil {
			log.Fatal(err)
		}
		log.Printf("Building package %d of %d: %s (commandline: %v)\n", cnt, len(rebuild), target, cmd.Args)
		if *dryRun {
			continue
		}
		buildlog, err := os.Create(filepath.Join(*logDir, target))
		if err != nil {
			log.Fatal(err)
		}
		defer buildlog.Close()
		cmd.Stdout = buildlog
		cmd.Stderr = buildlog
		if err := cmd.Run(); err != nil {
			log.Printf("building %s failed: %v\n", target, err)
			buildresults[target] = false
		} else {
			buildresults[target] = true
		}
	}

	log.Printf("Build results:\n")
	// Print all successful builds first (not as interesting), then failed ones.
	for target, result := range buildresults {
		if !result {
			continue
		}
		log.Printf("PASSED: %s\n", target)
	}

	for target, result := range buildresults {
		if result {
			continue
		}
		log.Printf("FAILED: %s (see %s)\n", target, filepath.Join(*logDir, target))
	}
}
<commit_msg>refactor reverse build dep gathering for better error messages<commit_after>// ratt operates on a Debian .changes file of a just-built package, identifies
// all reverse-build-dependencies and rebuilds them with the .debs from the
// .changes file.
//
// The intended use-case is, for example, to package a new snapshot of a Go
// library and verify that the new version does not break any other Go
// libraries/binaries.
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"pault.ag/go/debian/control"
	"pault.ag/go/debian/version"
)

var (
	logDir = flag.String("log_dir",
		"buildlogs",
		"Directory in which sbuild(1) logs for all reverse-build-dependencies are stored")

	dryRun = flag.Bool("dry_run",
		false,
		"Print sbuild command lines, but do not actually build the reverse-build-dependencies")

	sbuildDist = flag.String("sbuild_dist",
		"",
		"sbuild --dist= value (e.g. \"sid\"). Defaults to the Distribution: entry from the specified .changes file")

	dist = flag.String("dist",
		"",
		"Distribution to look up reverse-build-dependencies from. Defaults to the Distribution: entry from the specified .changes file")

	listsPrefixRe = regexp.MustCompile(`/([^/]*_dists_.*)_InRelease$`)
)

func dependsOn(src control.SourceIndex, binaries map[string]bool) bool {
	buildDepends := src.GetBuildDepends()
	for _, possibility := range buildDepends.GetAllPossibilities() {
		if binaries[possibility.Name] {
			return true
		}
	}
	return false
}

func addReverseBuildDeps(sourcesPath string, binaries map[string]bool, rebuild map[string][]version.Version) error {
	log.Printf("Loading sources index %q\n", sourcesPath)
	catFile := exec.Command("/usr/lib/apt/apt-helper",
		"cat-file",
		sourcesPath)
	var s *bufio.Reader
	if lines, err := catFile.Output(); err == nil {
		s = bufio.NewReader(bytes.NewReader(lines))
	} else {
		// Fallback for older versions of apt-get. See
		// <20160111171230.GA17291@debian.org> for context.
		o, err := os.Open(sourcesPath)
		if err != nil {
			return err
		}
		defer o.Close()
		s = bufio.NewReader(o)
	}
	idx, err := control.ParseSourceIndex(s)
	if err != nil && err != io.EOF {
		return err
	}

	for _, src := range idx {
		if dependsOn(src, binaries) {
			rebuild[src.Package] = append(rebuild[src.Package], src.Version)
		}
	}

	return nil
}

func fallback(sourcesPaths []string, binaries []string) (map[string][]version.Version, error) {
	bins := make(map[string]bool)
	for _, bin := range binaries {
		bins[bin] = true
	}

	rebuild := make(map[string][]version.Version)
	for _, sourcesPath := range sourcesPaths {
		if err := addReverseBuildDeps(sourcesPath, bins, rebuild); err != nil {
			return nil, err
		}
	}
	return rebuild, nil
}

func reverseBuildDeps(packagesPaths, sourcesPaths []string, binaries []string) (map[string][]version.Version, error) {
	if _, err := exec.LookPath("dose-ceve"); err != nil {
		log.Printf("dose-ceve(1) not found. Please install the dose-extra package for more accurate results. Falling back to interpreting Sources directly")
		return fallback(sourcesPaths, binaries)
	}

	archOut, err := exec.Command("dpkg-architecture", "--query=DEB_BUILD_ARCH").Output()
	if err != nil {
		log.Fatal(err)
	}
	arch := strings.TrimSpace(string(archOut))

	// TODO: Cache this output based on the .changes file. dose-ceve takes quite a while.
	ceve := exec.Command(
		"dose-ceve",
		"--deb-native-arch="+arch,
		"-T", "debsrc",
		"-r", strings.Join(binaries, ","),
		"-G", "pkg")
	for _, packagesPath := range packagesPaths {
		ceve.Args = append(ceve.Args, "deb://"+packagesPath)
	}
	for _, sourcesPath := range sourcesPaths {
		ceve.Args = append(ceve.Args, "debsrc://"+sourcesPath)
	}
	ceve.Stderr = os.Stderr

	log.Printf("Figuring out reverse build dependencies using dose-ceve(1). This might take a while")
	out, err := ceve.Output()
	if err != nil {
		log.Printf("dose-ceve(1) failed (%v), falling back to interpreting Sources directly", err)
		return fallback(sourcesPaths, binaries)
	}
	var doseCeves []struct {
		Package string
		Version version.Version
	}
	r := bufio.NewReader(bytes.NewReader(out))
	if err := control.Unmarshal(&doseCeves, r); err != nil {
		return nil, err
	}
	rebuild := make(map[string][]version.Version)
	for _, doseCeve := range doseCeves {
		rebuild[doseCeve.Package] = append(rebuild[doseCeve.Package], doseCeve.Version)
	}
	return rebuild, nil
}

func main() {
	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatalf("Usage: %s [options] <path-to-changes-file>\n", os.Args[0])
	}

	changesPath := flag.Arg(0)
	log.Printf("Loading changes file %q\n", changesPath)

	c, err := os.Open(changesPath)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()
	changes, err := control.ParseChanges(bufio.NewReader(c), changesPath)
	if err != nil && err != io.EOF {
		log.Fatal(err)
	}

	var debs []string
	for _, file := range changes.Files {
		if filepath.Ext(file.Filename) == ".deb" {
			debs = append(debs, file.Filename)
		}
	}

	log.Printf(" - %d binary packages: %s\n", len(changes.Binaries), strings.Join(changes.Binaries, " "))
	log.Printf(" - corresponding .debs (will be injected when building):\n")
	for _, deb := range debs {
		log.Printf("    %s\n", deb)
	}

	if strings.TrimSpace(*dist) == "" {
		*dist = changes.Distribution
		// Rewrite unstable to sid, which apt-get indextargets (below) requires.
		if *dist == "unstable" {
			*dist = "sid"
		}
		log.Printf("Setting -dist=%s (from .changes file)\n", *dist)
	}

	var sourcesPaths []string
	var packagesPaths []string
	indexTargets := exec.Command("apt-get",
		"indextargets",
		"--format",
		"$(FILENAME)",
		"Codename: "+*dist,
		"ShortDesc: Sources")
	if lines, err := indexTargets.Output(); err == nil {
		for _, line := range strings.Split(string(lines), "\n") {
			trimmed := strings.TrimSpace(line)
			if trimmed != "" {
				sourcesPaths = append(sourcesPaths, line)
			}
		}
		binaryIndexTargets := exec.Command(
			"apt-get",
			"indextargets",
			"--format",
			"$(FILENAME)",
			"Codename: "+*dist,
			"ShortDesc: Packages")
		lines, err = binaryIndexTargets.Output()
		if err != nil {
			log.Fatal("Could not get packages files using %+v: %v", binaryIndexTargets.Args, err)
		}
		for _, line := range strings.Split(string(lines), "\n") {
			trimmed := strings.TrimSpace(line)
			if trimmed != "" {
				packagesPaths = append(packagesPaths, line)
			}
		}
	} else {
		// Fallback for older versions of apt-get. See
		// https://bugs.debian.org/801594 for context.
		releaseMatches, err := filepath.Glob("/var/lib/apt/lists/*_InRelease")
		if err != nil {
			log.Fatal(err)
		}
		for _, releasepath := range releaseMatches {
			r, err := os.Open(releasepath)
			if err != nil {
				log.Fatal(err)
			}
			defer r.Close()
			var inRelease struct {
				Suite string
			}
			if err := control.Unmarshal(&inRelease, bufio.NewReader(r)); err != nil {
				log.Fatal(err)
			}

			listsPrefix := listsPrefixRe.FindStringSubmatch(releasepath)
			if len(listsPrefix) != 2 {
				log.Fatalf("release file path %q does not match regexp %q\n", releasepath, listsPrefixRe)
			}
			sourceMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Sources", listsPrefix[1]))
			if err != nil {
				log.Fatal(err)
			}
			sourcesPaths = append(sourcesPaths, sourceMatches...)
			packagesMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Packages", listsPrefix[1]))
			if err != nil {
				log.Fatal(err)
			}
			packagesPaths = append(packagesPaths, packagesMatches...)
		}
	}

	if len(sourcesPaths) == 0 {
		log.Fatal("Could not find InRelease file for " + *dist + " . Are you missing " + *dist + " in your /etc/apt/sources.list?")
	}

	rebuild, err := reverseBuildDeps(packagesPaths, sourcesPaths, changes.Binaries)
	if err != nil {
		log.Fatal(err)
	}

	// TODO: add -recursive flag to also cover dependencies which are not DIRECT dependencies. use http://godoc.org/pault.ag/go/debian/control#OrderDSCForBuild (topsort) to build dependencies in the right order (saving CPU time).

	// TODO: what’s a good integration method for doing this in more setups, e.g. on a cloud provider or something? mapreri from #debian-qa says jenkins.debian.net is suitable.

	if strings.TrimSpace(*sbuildDist) == "" {
		*sbuildDist = changes.Distribution
		log.Printf("Setting -sbuild_dist=%s (from .changes file)\n", *sbuildDist)
	}

	cnt := 1
	buildresults := make(map[string]bool)
	for src, versions := range rebuild {
		sort.Sort(sort.Reverse(version.Slice(versions)))
		newest := versions[0]
		target := fmt.Sprintf("%s_%s", src, newest)
		// TODO: discard resulting package immediately?
		args := []string{
			"--arch-all",
			"--dist=" + *sbuildDist,
			"--nolog",
			target,
		}
		for _, filename := range debs {
			args = append(args, fmt.Sprintf("--extra-package=%s", filename))
		}
		cmd := exec.Command("sbuild", args...)
		if err := os.MkdirAll(*logDir, 0755); err != nil {
			log.Fatal(err)
		}
		log.Printf("Building package %d of %d: %s (commandline: %v)\n", cnt, len(rebuild), target, cmd.Args)
		if *dryRun {
			continue
		}
		buildlog, err := os.Create(filepath.Join(*logDir, target))
		if err != nil {
			log.Fatal(err)
		}
		defer buildlog.Close()
		cmd.Stdout = buildlog
		cmd.Stderr = buildlog
		if err := cmd.Run(); err != nil {
			log.Printf("building %s failed: %v\n", target, err)
			buildresults[target] = false
		} else {
			buildresults[target] = true
		}
	}

	log.Printf("Build results:\n")
	// Print all successful builds first (not as interesting), then failed ones.
	for target, result := range buildresults {
		if !result {
			continue
		}
		log.Printf("PASSED: %s\n", target)
	}

	for target, result := range buildresults {
		if result {
			continue
		}
		log.Printf("FAILED: %s (see %s)\n", target, filepath.Join(*logDir, target))
	}
}
<|endoftext|>


<commit_before>package negroni

import (
	"log"
	"net/http"
	"os"
	"time"
)

// ALogger interface
type ALogger interface {
	Println(v ...interface{})
	Printf(format string, v ...interface{})
}

// Logger is a middleware handler that logs the request as it goes in and the response as it goes out.
type Logger struct {
	// ALogger implements just enough log.Logger interface to be compatible with other implementations
	ALogger
}

// NewLogger returns a new Logger instance
func NewLogger() *Logger {
	return &Logger{log.New(os.Stdout, "[negroni] ", 0)}
}

func (l *Logger) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	start := time.Now()

	next(rw, r)

	res := rw.(ResponseWriter)
	l.Printf("%v | %v | \t %v | %v | %v %v", start.Format(time.RFC3339), res.Status(), time.Since(start), r.Host, r.Method, r.URL.Path)
}
<commit_msg>Use a template to define the logging output.<commit_after>package negroni

import (
	"bytes"

	"log"
	"net/http"
	"os"
	"sync"
	"text/template"
	"time"
)

// Log is the structure
// passed to the template.
type Log struct {
	StartTime string
	Status    int
	Duration  time.Duration
	Hostname  string
	Method    string
	Path      string
}

// DefaultFormat is the format
// logged used by the default Logger instance.
var DefaultFormat = "{{.StartTime}} | {{.Status}} | \t {{.Duration}} | {{.Hostname}} | {{.Method}} {{.Path}} \n"

// DefaultDateFormat is the
// format used for date by the
// default Logger instance.
var DefaultDateFormat = time.RFC3339

// ALogger interface
type ALogger interface {
	Println(v ...interface{})
	Printf(format string, v ...interface{})
}

// Logger is a middleware handler that logs the request as it goes in and the response as it goes out.
type Logger struct {
	// ALogger implements just enough log.Logger interface to be compatible with other implementations
	ALogger
	dateFormat string
	template   *template.Template
	buffer     *bytes.Buffer
	mu         sync.Mutex
}

// NewLogger returns a new Logger instance
func NewLogger() *Logger {
	logger := &Logger{ALogger: log.New(os.Stdout, "[negroni] ", 0), dateFormat: DefaultDateFormat, buffer: bytes.NewBufferString("")}
	logger.SetFormat(DefaultFormat)
	return logger
}

func (l *Logger) SetFormat(format string) {
	l.template = template.Must(template.New("negroni_parser").Parse(format))
}

func (l *Logger) SetDateFormat(format string) {
	l.dateFormat = format
}

func (l *Logger) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	start := time.Now()

	next(rw, r)

	res := rw.(ResponseWriter)
	log := Log{start.Format(l.dateFormat), res.Status(), time.Since(start), r.Host, r.Method, r.URL.Path}

	l.mu.Lock()
	defer l.mu.Unlock()
	l.template.Execute(l.buffer, log)
	l.Printf(l.buffer.String())
	l.buffer.Reset()
}
<|endoftext|>