<commit_before>// ATTENTION - FILE MANUAL FIXED AFTER CGO.
// Fixed line: Tv		_Ctype_struct_timeval -> Tv		UtTv
// Created by cgo -godefs, MANUAL FIXED
// cgo -godefs types_linux.go

package host

const (
	sizeofPtr      = 0x4
	sizeofShort    = 0x2
	sizeofInt      = 0x4
	sizeofLong     = 0x4
	sizeofLongLong = 0x8
)

type (
	_C_short     int16
	_C_int       int32
	_C_long      int32
	_C_long_long int64
)

type utmp struct {
	Type      int16
	Pad_cgo_0 [2]byte
	Pid       int32
	Line      [32]int8
	ID        [4]int8
	User      [32]int8
	Host      [256]int8
	Exit      exit_status
	Session   int32
	Tv        UtTv
	Addr_v6   [4]int32
	X__unused [20]int8
}
type exit_status struct {
	Termination int16
	Exit        int16
}
type UtTv struct {
	Sec  int32
	Usec int32
}
<commit_msg>[host]linux: forgot to add sizeOfUtmp.<commit_after>// ATTENTION - FILE MANUAL FIXED AFTER CGO.
// Fixed line: Tv		_Ctype_struct_timeval -> Tv		UtTv
// Created by cgo -godefs, MANUAL FIXED
// cgo -godefs types_linux.go

package host

const (
	sizeofPtr      = 0x4
	sizeofShort    = 0x2
	sizeofInt      = 0x4
	sizeofLong     = 0x4
	sizeofLongLong = 0x8
	sizeOfUtmp     = 0x180
)

type (
	_C_short     int16
	_C_int       int32
	_C_long      int32
	_C_long_long int64
)

type utmp struct {
	Type      int16
	Pad_cgo_0 [2]byte
	Pid       int32
	Line      [32]int8
	ID        [4]int8
	User      [32]int8
	Host      [256]int8
	Exit      exit_status
	Session   int32
	Tv        UtTv
	Addr_v6   [4]int32
	X__unused [20]int8
}
type exit_status struct {
	Termination int16
	Exit        int16
}
type UtTv struct {
	Sec  int32
	Usec int32
}
<|endoftext|>


<commit_before>package hostsfile

import (
	"encoding/csv"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"unicode/utf8"
)

const entryTemplate string = "\n%s\t%s\t# HOSTSUP %s"
const entryTag string = "HOSTSUP"

type Hostsfile struct {
	Filename string
	File     *os.File
}

func handleError(err error) {
	panic(err)
}

func NewHostsfile(filename string, ro ...bool) (*Hostsfile, error) {
	var f *os.File
	var err error

	if len(ro) > 0 && ro[0] == true {
		f, err = os.OpenFile(filename, os.O_RDONLY, 0666)

	} else {
		f, err = os.OpenFile(filename, os.O_RDWR, 0666)
	}

	if err != nil {
		return nil, err
	}

	return &Hostsfile{filename, f}, nil
}

func (h *Hostsfile) Close() error {
	return h.File.Close()
}

func (h *Hostsfile) AddEntry(host *Host) {
	defer h.File.Seek(0, 0)

	// Go the end of the file to append the new host entry.
	h.File.Seek(0, 2)

	entry := fmt.Sprintf(entryTemplate, host.IP, host.Hostname, host.Id)

	if _, err := h.File.WriteString(entry); err != nil {
		handleError(err)
	}
}

func (h *Hostsfile) RemoveEntry(host *Host) {
	defer h.File.Seek(0, 0)

	f, err := ioutil.ReadAll(h.File)

	if err != nil {
		handleError(err)
	}

	lines := strings.Split(string(f), "\n")
	updatedLines := []string{}

	for _, line := range lines {
		if !strings.Contains(line, host.Id) {
			updatedLines = append(updatedLines, line)
		}
	}

	output := strings.Join(updatedLines, "\n")

	err = ioutil.WriteFile(h.Filename, []byte(output), 0666)

	if err != nil {
		handleError(err)
	}
}

func (h *Hostsfile) FindEntry(hostname string) *Host {
	entries := h.ListEntries()

	for _, entry := range entries {
		if entry.Hostname == hostname {
			return entry
		}
	}

	return nil

}

func (h *Hostsfile) ListEntries() []*Host {
	defer h.File.Seek(0, 0)

	const ipIndex = 0
	const hostnameIndex = 1
	const idIndex = 2

	reader := csv.NewReader(h.File)
	reader.Comma, _ = utf8.DecodeRuneInString("\t")
	reader.Comment, _ = utf8.DecodeRuneInString("#")
	reader.FieldsPerRecord = -1

	lines, _ := reader.ReadAll()

	hosts := make([]*Host, 0)

	for _, line := range lines {
		// Verify that the line contains the entryTag. Hostsup entries will
		// always have 3 columns.
		if len(line) >= 3 && strings.Contains(line[idIndex], entryTag) {
			// TODO: See if we can unpack the list to create the Host
			// host.NewHost(line...)
			host := NewHost(line[1], line[0])
			hosts = append(hosts, host)
		}
	}

	return hosts
}

func (h *Hostsfile) Clean() []*Host {
	entries := h.ListEntries()

	for _, entry := range entries {
		h.RemoveEntry(entry)
	}

	return entries
}
<commit_msg>Add comments.<commit_after>package hostsfile

import (
	"encoding/csv"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"unicode/utf8"
)

const entryTemplate string = "\n%s\t%s\t# HOSTSUP %s"
const entryTag string = "HOSTSUP"

type Hostsfile struct {
	Filename string
	File     *os.File
}

func handleError(err error) {
	panic(err)
}

func NewHostsfile(filename string, ro ...bool) (*Hostsfile, error) {
	var f *os.File
	var err error

	// Determine if the hosts file should opened as read only or not.
	if len(ro) > 0 && ro[0] == true {
		f, err = os.OpenFile(filename, os.O_RDONLY, 0666)

	} else {
		f, err = os.OpenFile(filename, os.O_RDWR, 0666)
	}

	if err != nil {
		return nil, err
	}

	return &Hostsfile{filename, f}, nil
}

// Close the hostsfile.
func (h *Hostsfile) Close() error {
	return h.File.Close()
}

func (h *Hostsfile) AddEntry(host *Host) {
	defer h.File.Seek(0, 0)

	// Go the end of the file to append the new host entry.
	h.File.Seek(0, 2)

	entry := fmt.Sprintf(entryTemplate, host.IP, host.Hostname, host.Id)

	if _, err := h.File.WriteString(entry); err != nil {
		handleError(err)
	}
}

func (h *Hostsfile) RemoveEntry(host *Host) {
	defer h.File.Seek(0, 0)

	f, err := ioutil.ReadAll(h.File)

	if err != nil {
		handleError(err)
	}

	lines := strings.Split(string(f), "\n")
	updatedLines := []string{}

	for _, line := range lines {
		if !strings.Contains(line, host.Id) {
			updatedLines = append(updatedLines, line)
		}
	}

	output := strings.Join(updatedLines, "\n")

	err = ioutil.WriteFile(h.Filename, []byte(output), 0666)

	if err != nil {
		handleError(err)
	}
}

func (h *Hostsfile) FindEntry(hostname string) *Host {
	entries := h.ListEntries()

	for _, entry := range entries {
		if entry.Hostname == hostname {
			return entry
		}
	}

	return nil

}

func (h *Hostsfile) ListEntries() []*Host {
	defer h.File.Seek(0, 0)

	const ipIndex = 0
	const hostnameIndex = 1
	const idIndex = 2

	reader := csv.NewReader(h.File)
	reader.Comma, _ = utf8.DecodeRuneInString("\t")
	reader.Comment, _ = utf8.DecodeRuneInString("#")
	reader.FieldsPerRecord = -1

	lines, _ := reader.ReadAll()

	hosts := make([]*Host, 0)

	for _, line := range lines {
		// Verify that the line contains the entryTag. Hostsup entries will
		// always have 3 columns.
		if len(line) >= 3 && strings.Contains(line[idIndex], entryTag) {
			// TODO: See if we can unpack the list to create the Host
			// host.NewHost(line...)
			host := NewHost(line[1], line[0])
			hosts = append(hosts, host)
		}
	}

	return hosts
}

func (h *Hostsfile) Clean() []*Host {
	entries := h.ListEntries()

	for _, entry := range entries {
		h.RemoveEntry(entry)
	}

	return entries
}
<|endoftext|>


<commit_before>package dalga

import (
	"encoding/json"
	"fmt"
	"time"
)

// Job is the record stored in jobs table.
// Primary key is (Path, Body).
type Job struct {
	// Path is where the job is going to be POSTed when it's time came.
	Path string
	// Body of POST request.
	Body string
	// Interval is the duration between each publish to the endpoint.
	// If interval is 0 job will be deleted after first publish.
	Interval time.Duration
	// NextRun is the next run time of the job, stored in UTC.
	NextRun time.Time
}

func newJob(description, routingKey string, interval uint32, oneOff bool) *Job {
	j := Job{
		Body:     description,
		Path:     routingKey,
		Interval: time.Duration(interval) * time.Second,
	}
	j.setNewNextRun()
	if oneOff {
		j.Interval = 0
	}
	return &j
}

// String implements Stringer interface. Returns the job in human-readable form.
func (j *Job) String() string {
	return fmt.Sprintf("Job{%q, %q, %d, %s}", j.Body, j.Path, j.Interval/time.Second, j.NextRun.String()[:23])
}

// Remaining returns the duration left to the job's next run time.
func (j *Job) Remaining() time.Duration {
	return j.NextRun.Sub(time.Now().UTC())
}

// setNewNextRun calculates the new run time according to current time and sets it on the job.
func (j *Job) setNewNextRun() {
	j.NextRun = time.Now().UTC().Add(j.Interval)
}

func (j *Job) MarshalJSON() ([]byte, error) {
	return json.Marshal(struct {
		Job      string        `json:"job"`
		Path     string        `json:"routing_key"`
		Interval time.Duration `json:"interval"`
		NextRun  string        `json:"next_run"`
	}{
		Job:      j.Body,
		Path:     j.Path,
		Interval: j.Interval / time.Second,
		NextRun:  j.NextRun.Format(time.RFC3339),
	})
}
<commit_msg>fix json output<commit_after>package dalga

import (
	"encoding/json"
	"fmt"
	"time"
)

// Job is the record stored in jobs table.
// Primary key is (Path, Body).
type Job struct {
	// Path is where the job is going to be POSTed when it's time came.
	Path string
	// Body of POST request.
	Body string
	// Interval is the duration between each publish to the endpoint.
	// If interval is 0 job will be deleted after first publish.
	Interval time.Duration
	// NextRun is the next run time of the job, stored in UTC.
	NextRun time.Time
}

func newJob(description, routingKey string, interval uint32, oneOff bool) *Job {
	j := Job{
		Body:     description,
		Path:     routingKey,
		Interval: time.Duration(interval) * time.Second,
	}
	j.setNewNextRun()
	if oneOff {
		j.Interval = 0
	}
	return &j
}

// String implements Stringer interface. Returns the job in human-readable form.
func (j *Job) String() string {
	return fmt.Sprintf("Job{%q, %q, %d, %s}", j.Body, j.Path, j.Interval/time.Second, j.NextRun.String()[:23])
}

// Remaining returns the duration left to the job's next run time.
func (j *Job) Remaining() time.Duration {
	return j.NextRun.Sub(time.Now().UTC())
}

// setNewNextRun calculates the new run time according to current time and sets it on the job.
func (j *Job) setNewNextRun() {
	j.NextRun = time.Now().UTC().Add(j.Interval)
}

func (j *Job) MarshalJSON() ([]byte, error) {
	return json.Marshal(struct {
		Path     string        `json:"routing_key"`
		Body     string        `json:"body"`
		Interval time.Duration `json:"interval"`
		NextRun  string        `json:"next_run"`
	}{
		Path:     j.Path,
		Body:     j.Body,
		Interval: j.Interval / time.Second,
		NextRun:  j.NextRun.Format(time.RFC3339),
	})
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	Namespace   = "dashboard"
	ConsulAddr  = "127.0.0.1:8500"
	Version     string
	ExtAssetDir string
	Nodes       []Node
	mutex       sync.RWMutex
)

type KVPair struct {
	Key         string
	CreateIndex int64
	ModifyIndex int64
	LockIndex   int64
	Flags       int64
	Value       []byte
}

type Status int64

const (
	Success Status = iota
	Warning
	Danger
	Info
)

func (s Status) MarshalText() ([]byte, error) {
	if s <= Danger {
		return []byte(strings.ToLower(s.String())), nil
	} else {
		return []byte(strconv.FormatInt(int64(s), 10)), nil
	}
}

type Item struct {
	Category  string `json:"category"`
	Node      string `json:"node"`
	Address   string `json:"address"`
	Timestamp string `json:"timestamp"`
	Status    Status `json:"status"`
	Key       string `json:"key"`
	Data      string `json:"data"`
}

func (kv *KVPair) NewItem() Item {
	item := Item{
		Data:      string(kv.Value),
		Timestamp: time.Unix(kv.Flags/1000, 0).Format("2006-01-02 15:04:05 -0700"),
	}
	item.Status = Status(kv.Flags % 1000)

	// kv.Key : {namespace}/{category}/{node}/{key}
	path := strings.Split(kv.Key, "/")
	item.Category = path[1]
	if len(path) >= 3 {
		item.Node = path[2]
	}
	if len(path) >= 4 {
		item.Key = path[3]
	}
	return item
}

type Node struct {
	Node    string
	Address string
}

type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

func (w gzipResponseWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

func makeGzipHandler(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			fn(w, r)
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		fn(gzr, r)
	}
}

func main() {
	var (
		port        int
		showVersion bool
		trigger     string
	)
	flag.StringVar(&Namespace, "namespace", Namespace, "Consul kv top level key name. (/v1/kv/{namespace}/...)")
	flag.IntVar(&port, "port", 3000, "http listen port")
	flag.StringVar(&ExtAssetDir, "asset", "", "Serve files located in /assets from local directory. If not specified, use built-in asset.")
	flag.BoolVar(&showVersion, "v", false, "show vesion")
	flag.BoolVar(&showVersion, "version", false, "show vesion")
	flag.StringVar(&trigger, "trigger", "", "trigger command")
	flag.Parse()

	if showVersion {
		fmt.Println("consul-kv-dashboard: version:", Version)
		return
	}

	mux := http.NewServeMux()
	mux.HandleFunc("/", makeGzipHandler(indexPage))
	mux.HandleFunc("/api/", makeGzipHandler(kvApiProxy))

	if ExtAssetDir != "" {
		mux.Handle("/assets/",
			http.StripPrefix("/assets/", http.FileServer(http.Dir(ExtAssetDir))))
	} else {
		mux.Handle("/assets/",
			http.FileServer(NewAssetFileSystem("/assets/")))
	}
	http.Handle("/", mux)

	log.Println("listen port:", port)
	log.Println("asset directory:", ExtAssetDir)
	log.Println("namespace:", Namespace)
	if trigger != "" {
		log.Println("trigger:", trigger)
		go watchForTrigger(trigger)
	}
	go updateNodeList()

	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(port), nil))
}

func indexPage(w http.ResponseWriter, r *http.Request) {
	var (
		data []byte
		err  error
	)
	if ExtAssetDir == "" {
		data, err = Asset("index.html")
	} else {
		var f *os.File
		f, err = os.Open(ExtAssetDir + "/index.html")
		data, err = ioutil.ReadAll(f)
	}
	if err != nil {
		log.Println(err)
	}
	fmt.Fprint(w, string(data))
}

func kvApiProxy(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	path := strings.TrimLeft(r.URL.Path, "/api")
	resp, _, err := callConsulAPI(
		"/v1/kv/" + Namespace + "/" + path + "?" + r.URL.RawQuery,
	)
	if err != nil {
		http.Error(w, fmt.Sprintf("%s", err), http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		w.Header().Set("Content-Type", "application/json")
		http.Error(w, "[]", resp.StatusCode)
		return
	}
	if resp.StatusCode != http.StatusOK {
		http.Error(w, "", resp.StatusCode)
		io.Copy(w, resp.Body)
		return
	}
	// copy response header to client
	for name, value := range resp.Header {
		if strings.HasPrefix(name, "X-") || name == "Content-Type" {
			for _, v := range value {
				w.Header().Set(name, v)
			}
		}
	}

	// keys or values
	dec := json.NewDecoder(resp.Body)
	enc := json.NewEncoder(w)
	if _, t := r.Form["keys"]; t {
		var keys []string
		uniqKeyMap := make(map[string]bool)
		dec.Decode(&keys)
		for _, key := range keys {
			path := strings.Split(key, "/")
			if len(path) >= 2 {
				uniqKeyMap[path[1]] = true
			}
		}
		uniqKeys := make([]string, 0, len(uniqKeyMap))
		for key, _ := range uniqKeyMap {
			uniqKeys = append(uniqKeys, key)
		}
		sort.Strings(uniqKeys)
		enc.Encode(uniqKeys)
	} else {
		var kvps []*KVPair
		dec.Decode(&kvps)
		items := make([]Item, 0, len(kvps))
		for _, kv := range kvps {
			item := kv.NewItem()
			if itemInNodes(&item) {
				items = append(items, item)
			}
		}
		enc.Encode(items)
	}
}

func watchForTrigger(command string) {
	var index int64
	lastStatus := make(map[string]Status)
	for {
		resp, newIndex, err := callConsulAPI(
			"/v1/kv/" + Namespace + "/?recurse&wait=55s&index=" + strconv.FormatInt(index, 10),
		)
		if err != nil {
			log.Println("[error]", err)
			time.Sleep(10 * time.Second)
			continue
		}
		index = newIndex
		defer resp.Body.Close()
		var kvps []*KVPair
		dec := json.NewDecoder(resp.Body)
		dec.Decode(&kvps)

		currentItem := make(map[string]Item)
		for _, kv := range kvps {
			item := kv.NewItem()
			if !itemInNodes(&item) {
				continue
			}
			if _, exist := currentItem[item.Category]; !exist {
				currentItem[item.Category] = item
			} else if currentItem[item.Category].Status < item.Status {
				currentItem[item.Category] = item
			}
		}
		for category, item := range currentItem {
			if _, exist := lastStatus[category]; !exist {
				// at first initialze
				lastStatus[category] = item.Status
				log.Printf("[info] %s: status %s", category, item.Status)
			} else if lastStatus[category] != item.Status {
				// status changed. invoking trigger.
				log.Printf("[info] %s: status %s -> %s", category, lastStatus[category], item.Status)
				lastStatus[category] = item.Status
				b, _ := json.Marshal(item)
				err := invokePipe(command, bytes.NewReader(b))
				if err != nil {
					log.Println("[error]", err)
				}
			}
		}
		time.Sleep(1 * time.Second)
	}
}

func invokePipe(command string, src io.Reader) error {
	log.Println("[info] Invoking command:", command)
	cmd := exec.Command("sh", "-c", command)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	err = cmd.Start()
	if err != nil {
		return err
	}
	cmdCh := make(chan error)
	// src => stdin
	go func() {
		_, err := io.Copy(stdin, src)
		if err != nil {
			cmdCh <- err
		}
		stdin.Close()
	}()
	// wait for command exit
	go func() {
		cmdCh <- cmd.Wait()
	}()
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)

	cmdErr := <-cmdCh
	return cmdErr
}

func updateNodeList() {
	var index int64
	for {
		resp, newIndex, err := callConsulAPI(
			"/v1/catalog/nodes?index=" + strconv.FormatInt(index, 10) + "&wait=55s",
		)
		if err != nil {
			log.Println("[error]", err)
			time.Sleep(10 * time.Second)
			continue
		}
		index = newIndex
		defer resp.Body.Close()
		dec := json.NewDecoder(resp.Body)
		mutex.Lock()
		dec.Decode(&Nodes)
		log.Println("[info]", Nodes)
		mutex.Unlock()
		time.Sleep(1 * time.Second)
	}
}

func itemInNodes(item *Item) bool {
	mutex.RLock()
	defer mutex.RUnlock()
	for _, node := range Nodes {
		if item.Node == node.Node {
			item.Address = node.Address
			return true
		}
	}
	return false
}

func callConsulAPI(path string) (*http.Response, int64, error) {
	var index int64
	_url := "http://" + ConsulAddr + path
	log.Println("[info] get", _url)
	resp, err := http.Get(_url)
	if err != nil {
		log.Println("[error]", err)
		return nil, index, err
	}
	_indexes := resp.Header["X-Consul-Index"]
	if len(_indexes) > 0 {
		index, _ = strconv.ParseInt(_indexes[0], 10, 64)
	}
	return resp, index, nil
}
<commit_msg>Fix bug for trim prefix.<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	Namespace   = "dashboard"
	ConsulAddr  = "127.0.0.1:8500"
	Version     string
	ExtAssetDir string
	Nodes       []Node
	mutex       sync.RWMutex
)

type KVPair struct {
	Key         string
	CreateIndex int64
	ModifyIndex int64
	LockIndex   int64
	Flags       int64
	Value       []byte
}

type Status int64

const (
	Success Status = iota
	Warning
	Danger
	Info
)

func (s Status) MarshalText() ([]byte, error) {
	if s <= Danger {
		return []byte(strings.ToLower(s.String())), nil
	} else {
		return []byte(strconv.FormatInt(int64(s), 10)), nil
	}
}

type Item struct {
	Category  string `json:"category"`
	Node      string `json:"node"`
	Address   string `json:"address"`
	Timestamp string `json:"timestamp"`
	Status    Status `json:"status"`
	Key       string `json:"key"`
	Data      string `json:"data"`
}

func (kv *KVPair) NewItem() Item {
	item := Item{
		Data:      string(kv.Value),
		Timestamp: time.Unix(kv.Flags/1000, 0).Format("2006-01-02 15:04:05 -0700"),
	}
	item.Status = Status(kv.Flags % 1000)

	// kv.Key : {namespace}/{category}/{node}/{key}
	path := strings.Split(kv.Key, "/")
	item.Category = path[1]
	if len(path) >= 3 {
		item.Node = path[2]
	}
	if len(path) >= 4 {
		item.Key = path[3]
	}
	return item
}

type Node struct {
	Node    string
	Address string
}

type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

func (w gzipResponseWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

func makeGzipHandler(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			fn(w, r)
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		fn(gzr, r)
	}
}

func main() {
	var (
		port        int
		showVersion bool
		trigger     string
	)
	flag.StringVar(&Namespace, "namespace", Namespace, "Consul kv top level key name. (/v1/kv/{namespace}/...)")
	flag.IntVar(&port, "port", 3000, "http listen port")
	flag.StringVar(&ExtAssetDir, "asset", "", "Serve files located in /assets from local directory. If not specified, use built-in asset.")
	flag.BoolVar(&showVersion, "v", false, "show vesion")
	flag.BoolVar(&showVersion, "version", false, "show vesion")
	flag.StringVar(&trigger, "trigger", "", "trigger command")
	flag.Parse()

	if showVersion {
		fmt.Println("consul-kv-dashboard: version:", Version)
		return
	}

	mux := http.NewServeMux()
	mux.HandleFunc("/", makeGzipHandler(indexPage))
	mux.HandleFunc("/api/", makeGzipHandler(kvApiProxy))

	if ExtAssetDir != "" {
		mux.Handle("/assets/",
			http.StripPrefix("/assets/", http.FileServer(http.Dir(ExtAssetDir))))
	} else {
		mux.Handle("/assets/",
			http.FileServer(NewAssetFileSystem("/assets/")))
	}
	http.Handle("/", mux)

	log.Println("listen port:", port)
	log.Println("asset directory:", ExtAssetDir)
	log.Println("namespace:", Namespace)
	if trigger != "" {
		log.Println("trigger:", trigger)
		go watchForTrigger(trigger)
	}
	go updateNodeList()

	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(port), nil))
}

func indexPage(w http.ResponseWriter, r *http.Request) {
	var (
		data []byte
		err  error
	)
	if ExtAssetDir == "" {
		data, err = Asset("index.html")
	} else {
		var f *os.File
		f, err = os.Open(ExtAssetDir + "/index.html")
		data, err = ioutil.ReadAll(f)
	}
	if err != nil {
		log.Println(err)
	}
	fmt.Fprint(w, string(data))
}

func kvApiProxy(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	path := strings.TrimPrefix(r.URL.Path, "/api/")
	resp, _, err := callConsulAPI(
		"/v1/kv/" + Namespace + "/" + path + "?" + r.URL.RawQuery,
	)
	if err != nil {
		http.Error(w, fmt.Sprintf("%s", err), http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		w.Header().Set("Content-Type", "application/json")
		http.Error(w, "[]", resp.StatusCode)
		return
	}
	if resp.StatusCode != http.StatusOK {
		http.Error(w, "", resp.StatusCode)
		io.Copy(w, resp.Body)
		return
	}
	// copy response header to client
	for name, value := range resp.Header {
		if strings.HasPrefix(name, "X-") || name == "Content-Type" {
			for _, v := range value {
				w.Header().Set(name, v)
			}
		}
	}

	// keys or values
	dec := json.NewDecoder(resp.Body)
	enc := json.NewEncoder(w)
	if _, t := r.Form["keys"]; t {
		var keys []string
		uniqKeyMap := make(map[string]bool)
		dec.Decode(&keys)
		for _, key := range keys {
			path := strings.Split(key, "/")
			if len(path) >= 2 {
				uniqKeyMap[path[1]] = true
			}
		}
		uniqKeys := make([]string, 0, len(uniqKeyMap))
		for key, _ := range uniqKeyMap {
			uniqKeys = append(uniqKeys, key)
		}
		sort.Strings(uniqKeys)
		enc.Encode(uniqKeys)
	} else {
		var kvps []*KVPair
		dec.Decode(&kvps)
		items := make([]Item, 0, len(kvps))
		for _, kv := range kvps {
			item := kv.NewItem()
			if itemInNodes(&item) {
				items = append(items, item)
			}
		}
		enc.Encode(items)
	}
}

func watchForTrigger(command string) {
	var index int64
	lastStatus := make(map[string]Status)
	for {
		resp, newIndex, err := callConsulAPI(
			"/v1/kv/" + Namespace + "/?recurse&wait=55s&index=" + strconv.FormatInt(index, 10),
		)
		if err != nil {
			log.Println("[error]", err)
			time.Sleep(10 * time.Second)
			continue
		}
		index = newIndex
		defer resp.Body.Close()
		var kvps []*KVPair
		dec := json.NewDecoder(resp.Body)
		dec.Decode(&kvps)

		currentItem := make(map[string]Item)
		for _, kv := range kvps {
			item := kv.NewItem()
			if !itemInNodes(&item) {
				continue
			}
			if _, exist := currentItem[item.Category]; !exist {
				currentItem[item.Category] = item
			} else if currentItem[item.Category].Status < item.Status {
				currentItem[item.Category] = item
			}
		}
		for category, item := range currentItem {
			if _, exist := lastStatus[category]; !exist {
				// at first initialze
				lastStatus[category] = item.Status
				log.Printf("[info] %s: status %s", category, item.Status)
			} else if lastStatus[category] != item.Status {
				// status changed. invoking trigger.
				log.Printf("[info] %s: status %s -> %s", category, lastStatus[category], item.Status)
				lastStatus[category] = item.Status
				b, _ := json.Marshal(item)
				err := invokePipe(command, bytes.NewReader(b))
				if err != nil {
					log.Println("[error]", err)
				}
			}
		}
		time.Sleep(1 * time.Second)
	}
}

func invokePipe(command string, src io.Reader) error {
	log.Println("[info] Invoking command:", command)
	cmd := exec.Command("sh", "-c", command)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	err = cmd.Start()
	if err != nil {
		return err
	}
	cmdCh := make(chan error)
	// src => stdin
	go func() {
		_, err := io.Copy(stdin, src)
		if err != nil {
			cmdCh <- err
		}
		stdin.Close()
	}()
	// wait for command exit
	go func() {
		cmdCh <- cmd.Wait()
	}()
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)

	cmdErr := <-cmdCh
	return cmdErr
}

func updateNodeList() {
	var index int64
	for {
		resp, newIndex, err := callConsulAPI(
			"/v1/catalog/nodes?index=" + strconv.FormatInt(index, 10) + "&wait=55s",
		)
		if err != nil {
			log.Println("[error]", err)
			time.Sleep(10 * time.Second)
			continue
		}
		index = newIndex
		defer resp.Body.Close()
		dec := json.NewDecoder(resp.Body)
		mutex.Lock()
		dec.Decode(&Nodes)
		log.Println("[info]", Nodes)
		mutex.Unlock()
		time.Sleep(1 * time.Second)
	}
}

func itemInNodes(item *Item) bool {
	mutex.RLock()
	defer mutex.RUnlock()
	for _, node := range Nodes {
		if item.Node == node.Node {
			item.Address = node.Address
			return true
		}
	}
	return false
}

func callConsulAPI(path string) (*http.Response, int64, error) {
	var index int64
	_url := "http://" + ConsulAddr + path
	log.Println("[info] get", _url)
	resp, err := http.Get(_url)
	if err != nil {
		log.Println("[error]", err)
		return nil, index, err
	}
	_indexes := resp.Header["X-Consul-Index"]
	if len(_indexes) > 0 {
		index, _ = strconv.ParseInt(_indexes[0], 10, 64)
	}
	return resp, index, nil
}
<|endoftext|>


<commit_before>package http

import (
	"bytes"
	"errors"
	"io"
	"log"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/sparrowdb/db"
	"github.com/sparrowdb/model"
	"github.com/sparrowdb/monitor"
	"github.com/sparrowdb/spql"
	"github.com/sparrowdb/util/uuid"
)

// ServeHandler holds main http methods
type ServeHandler struct {
	dbManager     *db.DBManager
	queryExecutor *spql.QueryExecutor
}

var (
	errDatabaseNotFound = errors.New("Database not found")
	errWrongRequest     = errors.New("Wrong HTTP request")
	errEmptyQueryResult = errors.New("Empty query result")
	errWrongToken       = errors.New("Wrong token")
	errInsertImage      = errors.New("Could not insert images")
)

func (sh *ServeHandler) writeResponse(request *RequestData, result *spql.QueryResult) {
	request.responseWriter.Write(result.Value())
}

func (sh *ServeHandler) writeError(request *RequestData, query string, errs ...error) {
	result := &spql.QueryResult{}
	for _, v := range errs {
		result.AddErrorStr(v.Error())
	}

	result.AddValue(strings.Replace(query, "\n", "", -1))
	request.responseWriter.WriteHeader(404)
	request.responseWriter.Write(result.Value())
}

func (sh *ServeHandler) serveQuery(request *RequestData) {
	body := request.request.Body

	buf := new(bytes.Buffer)
	buf.ReadFrom(body)
	qStr := buf.String()

	p := spql.NewParser(qStr)
	q, err := p.ParseQuery()
	if err != nil {
		sh.writeError(request, qStr, err)
		return
	}

	results := <-sh.queryExecutor.ExecuteQuery(q)

	if results == nil {
		sh.writeError(request, qStr, errEmptyQueryResult)
		return
	}

	monitor.IncHTTPQueries()
	request.responseWriter.Header().Set("Content-Type", "application/json")
	sh.writeResponse(request, results)
}

func (sh *ServeHandler) get(request *RequestData) {
	if len(request.params) < 2 {
		sh.writeError(request, "{}", errWrongRequest)
		return
	}

	dbname := request.params[0]
	key := request.params[1]

	// Check if database exists
	sto, ok := sh.dbManager.GetDatabase(dbname)
	if !ok {
		sh.writeError(request, "{}", errDatabaseNotFound)
		return
	}

	// Async get requested data
	result := <-sh.dbManager.GetData(dbname, key)

	// Check if found requested data
	if result == nil {
		sh.writeError(request, "{}", errEmptyQueryResult)
		return
	}

	// Token verification if enabled
	if sto.Descriptor.TokenActive {
		if len(request.params) != 3 {
			sh.writeError(request, "{}", errWrongRequest)
			return
		}
		token := request.params[2]

		if token != result.Token {
			sh.writeError(request, "{}", errWrongToken)
			return
		}
	}

	request.responseWriter.Header().Set("Content-Type", "image/"+result.Ext)
	request.responseWriter.Header().Set("Content-Length", strconv.Itoa(int(result.Size)))
	request.responseWriter.Write(result.Buf)
}

func (sh *ServeHandler) upload(request *RequestData) {
	file, fhandler, err := request.request.FormFile("uploadfile")
	if err != nil {
		log.Println(err)
		return
	}
	defer file.Close()

	buf := new(bytes.Buffer)
	io.Copy(buf, file)

	dbname := request.request.FormValue("dbname")
	sto, ok := sh.dbManager.GetDatabase(dbname)

	if ok {
		var token string

		if sto.Descriptor.TokenActive {
			token = uuid.TimeUUID().String()
		}

		err := sto.InsertData(&model.DataDefinition{
			Key:   request.request.FormValue("key"),
			Token: token,

			// get file extension and remove dot before ext name
			Ext: filepath.Ext(fhandler.Filename)[1:],

			Size: uint32(len(buf.Bytes())),
			Buf:  buf.Bytes(),
		})

		if err != nil {
			sh.writeError(request, "{}", errInsertImage)
		}

		monitor.IncHTTPUploads()
	}
}

// NewServeHandler returns new ServeHandler
func NewServeHandler(dbm *db.DBManager, queryExecutor *spql.QueryExecutor) *ServeHandler {
	return &ServeHandler{
		dbManager:     dbm,
		queryExecutor: queryExecutor,
	}
}
<commit_msg>default store uuid<commit_after>package http

import (
	"bytes"
	"errors"
	"io"
	"log"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/sparrowdb/db"
	"github.com/sparrowdb/model"
	"github.com/sparrowdb/monitor"
	"github.com/sparrowdb/spql"
	"github.com/sparrowdb/util/uuid"
)

// ServeHandler holds main http methods
type ServeHandler struct {
	dbManager     *db.DBManager
	queryExecutor *spql.QueryExecutor
}

var (
	errDatabaseNotFound = errors.New("Database not found")
	errWrongRequest     = errors.New("Wrong HTTP request")
	errEmptyQueryResult = errors.New("Empty query result")
	errWrongToken       = errors.New("Wrong token")
	errInsertImage      = errors.New("Could not insert images")
)

func (sh *ServeHandler) writeResponse(request *RequestData, result *spql.QueryResult) {
	request.responseWriter.Write(result.Value())
}

func (sh *ServeHandler) writeError(request *RequestData, query string, errs ...error) {
	result := &spql.QueryResult{}
	for _, v := range errs {
		result.AddErrorStr(v.Error())
	}

	result.AddValue(strings.Replace(query, "\n", "", -1))
	request.responseWriter.WriteHeader(404)
	request.responseWriter.Write(result.Value())
}

func (sh *ServeHandler) serveQuery(request *RequestData) {
	body := request.request.Body

	buf := new(bytes.Buffer)
	buf.ReadFrom(body)
	qStr := buf.String()

	p := spql.NewParser(qStr)
	q, err := p.ParseQuery()
	if err != nil {
		sh.writeError(request, qStr, err)
		return
	}

	results := <-sh.queryExecutor.ExecuteQuery(q)

	if results == nil {
		sh.writeError(request, qStr, errEmptyQueryResult)
		return
	}

	monitor.IncHTTPQueries()
	request.responseWriter.Header().Set("Content-Type", "application/json")
	sh.writeResponse(request, results)
}

func (sh *ServeHandler) get(request *RequestData) {
	if len(request.params) < 2 {
		sh.writeError(request, "{}", errWrongRequest)
		return
	}

	dbname := request.params[0]
	key := request.params[1]

	// Check if database exists
	sto, ok := sh.dbManager.GetDatabase(dbname)
	if !ok {
		sh.writeError(request, "{}", errDatabaseNotFound)
		return
	}

	// Async get requested data
	result := <-sh.dbManager.GetData(dbname, key)

	// Check if found requested data
	if result == nil {
		sh.writeError(request, "{}", errEmptyQueryResult)
		return
	}

	// Token verification if enabled
	if sto.Descriptor.TokenActive {
		if len(request.params) != 3 {
			sh.writeError(request, "{}", errWrongRequest)
			return
		}
		token := request.params[2]

		if token != result.Token {
			sh.writeError(request, "{}", errWrongToken)
			return
		}
	}

	request.responseWriter.Header().Set("Content-Type", "image/"+result.Ext)
	request.responseWriter.Header().Set("Content-Length", strconv.Itoa(int(result.Size)))
	request.responseWriter.Write(result.Buf)
}

func (sh *ServeHandler) upload(request *RequestData) {
	file, fhandler, err := request.request.FormFile("uploadfile")
	if err != nil {
		log.Println(err)
		return
	}
	defer file.Close()

	buf := new(bytes.Buffer)
	io.Copy(buf, file)

	dbname := request.request.FormValue("dbname")
	sto, ok := sh.dbManager.GetDatabase(dbname)

	if ok {
		err := sto.InsertData(&model.DataDefinition{
			Key: request.request.FormValue("key"),

			// default store UUID to keep information of insert time
			// and eliminates attacks aimed at guessing valid URLs for photos
			Token: uuid.TimeUUID().String(),

			// get file extension and remove dot before ext name
			Ext: filepath.Ext(fhandler.Filename)[1:],

			Size: uint32(len(buf.Bytes())),
			Buf:  buf.Bytes(),
		})

		if err != nil {
			sh.writeError(request, "{}", errInsertImage)
		}

		monitor.IncHTTPUploads()
	}
}

// NewServeHandler returns new ServeHandler
func NewServeHandler(dbm *db.DBManager, queryExecutor *spql.QueryExecutor) *ServeHandler {
	return &ServeHandler{
		dbManager:     dbm,
		queryExecutor: queryExecutor,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/julienschmidt/httprouter"
	"github.com/satori/go.uuid"
)

/* Read functions
 */
func ListCheckConfiguration(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	defer PanicCatcher(w)

	returnChannel := make(chan somaResult)
	handler := handlerMap["checkConfigurationReadHandler"].(somaCheckConfigurationReadHandler)
	handler.input <- somaCheckConfigRequest{
		action: "list",
		reply:  returnChannel,
		CheckConfig: proto.CheckConfig{
			RepositoryId: params.ByName("repository"),
		},
	}
	result := <-returnChannel

	// declare here since goto does not jump over declarations
	cReq := proto.Request{}
	cReq.Filter = &proto.Filter{}
	cReq.Filter.CheckConfig = &proto.CheckConfigFilter{}
	if result.Failure() {
		goto skip
	}

	_ = DecodeJsonBody(r, &cReq)
	if cReq.Filter.CheckConfig.Name != "" {
		filtered := make([]somaCheckConfigResult, 0)
		for _, i := range result.CheckConfigs {
			if i.CheckConfig.Name == cReq.Filter.CheckConfig.Name {
				filtered = append(filtered, i)
			}
		}
		result.CheckConfigs = filtered
	}

skip:
	SendCheckConfigurationReply(&w, &result)
}

func ShowCheckConfiguration(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	defer PanicCatcher(w)

	returnChannel := make(chan somaResult)
	handler := handlerMap["checkConfigurationReadHandler"].(somaCheckConfigurationReadHandler)
	handler.input <- somaCheckConfigRequest{
		action: "show",
		reply:  returnChannel,
		CheckConfig: proto.CheckConfig{
			Id:           params.ByName("check"),
			RepositoryId: params.ByName("repository"),
		},
	}
	result := <-returnChannel

	SendCheckConfigurationReply(&w, &result)
}

/* Write functions
 */
func AddCheckConfiguration(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	defer PanicCatcher(w)

	cReq := proto.Request{}
	if err := DecodeJsonBody(r, &cReq); err != nil {
		DispatchBadRequest(&w, err)
		return
	}
	cReq.CheckConfig.Id = uuid.Nil.String()

	returnChannel := make(chan somaResult)
	handler := handlerMap["guidePost"].(guidePost)
	handler.input <- treeRequest{
		RequestType: "check",
		Action:      fmt.Sprintf("add_check_to_%s", cReq.CheckConfig.ObjectType),
		reply:       returnChannel,
		CheckConfig: somaCheckConfigRequest{
			action:      "check_configuration_new",
			CheckConfig: *cReq.CheckConfig,
		},
	}
	result := <-returnChannel
	SendCheckConfigurationReply(&w, &result)
}

/* Utility
 */
func SendCheckConfigurationReply(w *http.ResponseWriter, r *somaResult) {
	result := proto.Result{}
	if r.MarkErrors(&result) {
		goto dispatch
	}
	result.Errors = &[]string{}
	result.CheckConfigs = &[]proto.CheckConfig{}
	for _, i := range (*r).CheckConfigs {
		*result.CheckConfigs = append(*result.CheckConfigs, i.CheckConfig)
		if i.ResultError != nil {
			*result.Errors = append(*result.Errors, i.ResultError.Error())
		}
	}

dispatch:
	result.Clean()
	json, err := json.Marshal(result)
	if err != nil {
		DispatchInternalError(w, err)
		return
	}
	DispatchJsonReply(w, &json)
	return
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>Add checkconfig/delete and forward to GuidePost<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/julienschmidt/httprouter"
	"github.com/satori/go.uuid"
)

/* Read functions
 */
func ListCheckConfiguration(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	defer PanicCatcher(w)

	returnChannel := make(chan somaResult)
	handler := handlerMap["checkConfigurationReadHandler"].(somaCheckConfigurationReadHandler)
	handler.input <- somaCheckConfigRequest{
		action: "list",
		reply:  returnChannel,
		CheckConfig: proto.CheckConfig{
			RepositoryId: params.ByName("repository"),
		},
	}
	result := <-returnChannel

	// declare here since goto does not jump over declarations
	cReq := proto.Request{}
	cReq.Filter = &proto.Filter{}
	cReq.Filter.CheckConfig = &proto.CheckConfigFilter{}
	if result.Failure() {
		goto skip
	}

	_ = DecodeJsonBody(r, &cReq)
	if cReq.Filter.CheckConfig.Name != "" {
		filtered := make([]somaCheckConfigResult, 0)
		for _, i := range result.CheckConfigs {
			if i.CheckConfig.Name == cReq.Filter.CheckConfig.Name {
				filtered = append(filtered, i)
			}
		}
		result.CheckConfigs = filtered
	}

skip:
	SendCheckConfigurationReply(&w, &result)
}

func ShowCheckConfiguration(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	defer PanicCatcher(w)

	returnChannel := make(chan somaResult)
	handler := handlerMap["checkConfigurationReadHandler"].(somaCheckConfigurationReadHandler)
	handler.input <- somaCheckConfigRequest{
		action: "show",
		reply:  returnChannel,
		CheckConfig: proto.CheckConfig{
			Id:           params.ByName("check"),
			RepositoryId: params.ByName("repository"),
		},
	}
	result := <-returnChannel

	SendCheckConfigurationReply(&w, &result)
}

/* Write functions
 */
func AddCheckConfiguration(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	defer PanicCatcher(w)

	cReq := proto.Request{}
	if err := DecodeJsonBody(r, &cReq); err != nil {
		DispatchBadRequest(&w, err)
		return
	}
	cReq.CheckConfig.Id = uuid.Nil.String()

	returnChannel := make(chan somaResult)
	handler := handlerMap["guidePost"].(guidePost)
	handler.input <- treeRequest{
		RequestType: "check",
		Action:      fmt.Sprintf("add_check_to_%s", cReq.CheckConfig.ObjectType),
		reply:       returnChannel,
		CheckConfig: somaCheckConfigRequest{
			action:      "check_configuration_new",
			CheckConfig: *cReq.CheckConfig,
		},
	}
	result := <-returnChannel
	SendCheckConfigurationReply(&w, &result)
}

func DeleteCheckConfiguration(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	defer PanicCatcher(w)

	returnChannel := make(chan somaResult)
	handler := handlerMap["guidePost"].(guidePost)
	handler.input <- treeRequest{
		RequestType: `check`,
		Action:      `remove_check`,
		reply:       returnChannel,
		CheckConfig: somaCheckConfigRequest{
			action: `check_configuration_delete`,
			CheckConfig: proto.CheckConfig{
				Id:           params.ByName(`check`),
				RepositoryId: params.ByName(`repository`),
			},
		},
	}
	result := <-returnChannel
	SendCheckConfigurationReply(&w, &result)
}

/* Utility
 */
func SendCheckConfigurationReply(w *http.ResponseWriter, r *somaResult) {
	result := proto.Result{}
	if r.MarkErrors(&result) {
		goto dispatch
	}
	result.Errors = &[]string{}
	result.CheckConfigs = &[]proto.CheckConfig{}
	for _, i := range (*r).CheckConfigs {
		*result.CheckConfigs = append(*result.CheckConfigs, i.CheckConfig)
		if i.ResultError != nil {
			*result.Errors = append(*result.Errors, i.ResultError.Error())
		}
	}

dispatch:
	result.Clean()
	json, err := json.Marshal(result)
	if err != nil {
		DispatchInternalError(w, err)
		return
	}
	DispatchJsonReply(w, &json)
	return
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>package ecmd

import (
	"errors"
	"fmt"
	"github.com/distributed/ecat/ecfr"
	"time"
)

type Commander interface {
	New(datalen int) (*ExecutingCommand, error)
	Cycle() error
	Close() error
}

type ExecutingCommand struct {
	DatagramOut *ecfr.Datagram

	DatagramIn *ecfr.Datagram
	Arrived    bool
	Overlayed  bool
	Error      error
}

var NoFrame = errors.New("frame did not arrive")
var NoOverlay = errors.New("failed to overlay")

type WorkingCounterError struct {
	Command    ecfr.CommandType
	Addr32     uint32
	Want, Have uint16
}

func (e WorkingCounterError) Error() string {
	return fmt.Sprintf("working counter error, want %d, have %d on %v %#08x", e.Want,
		e.Have,
		e.Command,
		e.Addr32)
}

func ChooseDefaultError(cmd *ExecutingCommand) error {
	if !cmd.Arrived {
		return NoFrame
	}

	if !cmd.Overlayed {
		return NoOverlay
	}

	return cmd.Error
}

func IsNoFrame(err error) bool {
	return err == NoFrame
}

func IsWorkingCounterError(err error) bool {
	_, ok := err.(WorkingCounterError)
	return ok
}

func ChooseWorkingCounterError(ec *ExecutingCommand, expwc uint16) error {
	havewc := ec.DatagramIn.WorkingCounter
	if expwc != havewc {
		return WorkingCounterError{
			ec.DatagramOut.Command,
			ec.DatagramOut.Addr32,
			expwc, havewc,
		}
	}

	return nil
}

const (
	DefaultFramelossTries = 3
)

type Options struct {
	FramelossTries int
	WCDeadline     time.Time
}

func (o Options) getFramelossTries() int {
	if o.FramelossTries == 0 {
		return DefaultFramelossTries
	}
	return o.FramelossTries
}
func (o Options) getWCDeadline() time.Time { return o.WCDeadline }

func ExecuteRead8(c Commander, addr ecfr.DatagramAddress, expwc uint16) (d uint8, err error) {
	return ExecuteRead8Options(c, addr, expwc, Options{})
}

func ExecuteRead8Options(c Commander, addr ecfr.DatagramAddress, expwc uint16, opts Options) (d uint8, err error) {
	var ds []byte
	ds, err = ExecuteRead(c, addr, 1, expwc)
	if err != nil {
		return
	}
	d = xgetUint8(ds)
	return
}

func ExecuteRead16(c Commander, addr ecfr.DatagramAddress, expwc uint16) (d uint16, err error) {
	return ExecuteRead16Options(c, addr, expwc, Options{})
}

func ExecuteRead16Options(c Commander, addr ecfr.DatagramAddress, expwc uint16, opt Options) (d uint16, err error) {
	var ds []byte
	ds, err = ExecuteRead(c, addr, 2, expwc)
	if err != nil {
		return
	}
	d = xgetUint16(ds)
	return
}

func ExecuteRead32(c Commander, addr ecfr.DatagramAddress, expwc uint16) (d uint32, err error) {
	return ExecuteRead32Options(c, addr, expwc, Options{})
}

func ExecuteRead32Options(c Commander, addr ecfr.DatagramAddress, expwc uint16, opt Options) (d uint32, err error) {
	var ds []byte
	ds, err = ExecuteRead(c, addr, 4, expwc)
	if err != nil {
		return
	}
	d = xgetUint32(ds)
	return
}

func ExecuteRead(c Commander, addr ecfr.DatagramAddress, n int, expwc uint16) (d []byte, err error) {
	return ExecuteReadOptions(c, addr, n, expwc, Options{})
}

func ExecuteReadOptions(c Commander, addr ecfr.DatagramAddress, n int, expwc uint16, opts Options) (d []byte, err error) {
	nFrameLoss := 0

	var ct ecfr.CommandType
	switch addr.Type() {
	case ecfr.Positional:
		ct = ecfr.APRD
	case ecfr.Fixed:
		ct = ecfr.FPRD
	case ecfr.Broadcast:
		ct = ecfr.BRD
	default:
		err = fmt.Errorf("ExecuteReadOptions: unsupported address type %v", addr.Type())
	}

	for {
		var ec *ExecutingCommand
		ec, err = c.New(n)
		if err != nil {
			return
		}

		dgo := ec.DatagramOut
		err = dgo.SetDataLen(n)
		if err != nil {
			return
		}

		dgo.Command = ct
		dgo.Addr32 = addr.Addr32()

		err = c.Cycle()
		if err != nil {
			return
		}

		err = ChooseDefaultError(ec)
		if err != nil {
			if IsNoFrame(err) {
				nFrameLoss++
				if nFrameLoss < opts.getFramelossTries() {
					continue
				}
			}
			return
		}

		err = ChooseWorkingCounterError(ec, expwc)
		if err != nil {
			now := time.Now()
			if now.Before(opts.getWCDeadline()) {
				continue
			}
		}

		d = ec.DatagramIn.Data()
		return
	}

	panic("not reached")
}

func ExecuteWrite8(c Commander, addr ecfr.DatagramAddress, w uint8, expwc uint16) (err error) {
	return ExecuteWrite8Options(c, addr, w, expwc, Options{})
}

func ExecuteWrite8Options(c Commander, addr ecfr.DatagramAddress, w uint8, expwc uint16, opts Options) (err error) {
	ws := make([]byte, 1)
	putUint8(ws, w)
	return ExecuteWriteOptions(c, addr, ws, expwc, opts)
}

func ExecuteWrite16(c Commander, addr ecfr.DatagramAddress, w uint16, expwc uint16) (err error) {
	return ExecuteWrite16Options(c, addr, w, expwc, Options{})
}

func ExecuteWrite16Options(c Commander, addr ecfr.DatagramAddress, w uint16, expwc uint16, opts Options) (err error) {
	ws := make([]byte, 2)
	putUint16(ws, w)
	return ExecuteWriteOptions(c, addr, ws, expwc, opts)
}

func ExecuteWrite32(c Commander, addr ecfr.DatagramAddress, w uint32, expwc uint16) (err error) {
	return ExecuteWrite32Options(c, addr, w, expwc, Options{})
}

func ExecuteWrite32Options(c Commander, addr ecfr.DatagramAddress, w uint32, expwc uint16, opts Options) (err error) {
	ws := make([]byte, 4)
	putUint32(ws, w)
	return ExecuteWriteOptions(c, addr, ws, expwc, opts)
}

func ExecuteWrite(c Commander, addr ecfr.DatagramAddress, w []byte, expwc uint16) (err error) {
	return ExecuteWriteOptions(c, addr, w, expwc, Options{})
}

func ExecuteWriteOptions(c Commander, addr ecfr.DatagramAddress, w []byte, expwc uint16, opts Options) (err error) {
	nFrameLoss := 0

	var ct ecfr.CommandType
	switch addr.Type() {
	case ecfr.Positional:
		ct = ecfr.APWR
	case ecfr.Fixed:
		ct = ecfr.FPWR
	case ecfr.Broadcast:
		ct = ecfr.BWR
	default:
		err = fmt.Errorf("ExecuteWriteOptions: unsupported address type %v", addr.Type())
	}

	for {
		var ec *ExecutingCommand
		ec, err = c.New(len(w))
		if err != nil {
			return
		}

		dgo := ec.DatagramOut
		err = dgo.SetDataLen(len(w))
		if err != nil {
			return
		}
		copy(dgo.Data(), w)

		dgo.Command = ct
		dgo.Addr32 = addr.Addr32()

		err = c.Cycle()
		if err != nil {
			return
		}

		err = ChooseDefaultError(ec)
		if err != nil {
			if IsNoFrame(err) {
				nFrameLoss++
				if nFrameLoss < opts.getFramelossTries() {
					continue
				}
			}
			return
		}

		err = ChooseWorkingCounterError(ec, expwc)
		if err != nil {
			now := time.Now()
			if now.Before(opts.getWCDeadline()) {
				continue
			}
		}

		return
	}

}
<commit_msg>remote extraneous and racy setting of datagram datalen in ecmd.go. the datalen is set in the corresponding framer.<commit_after>package ecmd

import (
	"errors"
	"fmt"
	"github.com/distributed/ecat/ecfr"
	"time"
)

type Commander interface {
	New(datalen int) (*ExecutingCommand, error)
	Cycle() error
	Close() error
}

type ExecutingCommand struct {
	DatagramOut *ecfr.Datagram

	DatagramIn *ecfr.Datagram
	Arrived    bool
	Overlayed  bool
	Error      error
}

var NoFrame = errors.New("frame did not arrive")
var NoOverlay = errors.New("failed to overlay")

type WorkingCounterError struct {
	Command    ecfr.CommandType
	Addr32     uint32
	Want, Have uint16
}

func (e WorkingCounterError) Error() string {
	return fmt.Sprintf("working counter error, want %d, have %d on %v %#08x", e.Want,
		e.Have,
		e.Command,
		e.Addr32)
}

func ChooseDefaultError(cmd *ExecutingCommand) error {
	if !cmd.Arrived {
		return NoFrame
	}

	if !cmd.Overlayed {
		return NoOverlay
	}

	return cmd.Error
}

func IsNoFrame(err error) bool {
	return err == NoFrame
}

func IsWorkingCounterError(err error) bool {
	_, ok := err.(WorkingCounterError)
	return ok
}

func ChooseWorkingCounterError(ec *ExecutingCommand, expwc uint16) error {
	havewc := ec.DatagramIn.WorkingCounter
	if expwc != havewc {
		return WorkingCounterError{
			ec.DatagramOut.Command,
			ec.DatagramOut.Addr32,
			expwc, havewc,
		}
	}

	return nil
}

const (
	DefaultFramelossTries = 3
)

type Options struct {
	FramelossTries int
	WCDeadline     time.Time
}

func (o Options) getFramelossTries() int {
	if o.FramelossTries == 0 {
		return DefaultFramelossTries
	}
	return o.FramelossTries
}
func (o Options) getWCDeadline() time.Time { return o.WCDeadline }

func ExecuteRead8(c Commander, addr ecfr.DatagramAddress, expwc uint16) (d uint8, err error) {
	return ExecuteRead8Options(c, addr, expwc, Options{})
}

func ExecuteRead8Options(c Commander, addr ecfr.DatagramAddress, expwc uint16, opts Options) (d uint8, err error) {
	var ds []byte
	ds, err = ExecuteRead(c, addr, 1, expwc)
	if err != nil {
		return
	}
	d = xgetUint8(ds)
	return
}

func ExecuteRead16(c Commander, addr ecfr.DatagramAddress, expwc uint16) (d uint16, err error) {
	return ExecuteRead16Options(c, addr, expwc, Options{})
}

func ExecuteRead16Options(c Commander, addr ecfr.DatagramAddress, expwc uint16, opt Options) (d uint16, err error) {
	var ds []byte
	ds, err = ExecuteRead(c, addr, 2, expwc)
	if err != nil {
		return
	}
	d = xgetUint16(ds)
	return
}

func ExecuteRead32(c Commander, addr ecfr.DatagramAddress, expwc uint16) (d uint32, err error) {
	return ExecuteRead32Options(c, addr, expwc, Options{})
}

func ExecuteRead32Options(c Commander, addr ecfr.DatagramAddress, expwc uint16, opt Options) (d uint32, err error) {
	var ds []byte
	ds, err = ExecuteRead(c, addr, 4, expwc)
	if err != nil {
		return
	}
	d = xgetUint32(ds)
	return
}

func ExecuteRead(c Commander, addr ecfr.DatagramAddress, n int, expwc uint16) (d []byte, err error) {
	return ExecuteReadOptions(c, addr, n, expwc, Options{})
}

func ExecuteReadOptions(c Commander, addr ecfr.DatagramAddress, n int, expwc uint16, opts Options) (d []byte, err error) {
	nFrameLoss := 0

	var ct ecfr.CommandType
	switch addr.Type() {
	case ecfr.Positional:
		ct = ecfr.APRD
	case ecfr.Fixed:
		ct = ecfr.FPRD
	case ecfr.Broadcast:
		ct = ecfr.BRD
	default:
		err = fmt.Errorf("ExecuteReadOptions: unsupported address type %v", addr.Type())
	}

	for {
		var ec *ExecutingCommand
		ec, err = c.New(n)
		if err != nil {
			return
		}

		dgo := ec.DatagramOut
		dgo.Command = ct
		dgo.Addr32 = addr.Addr32()

		err = c.Cycle()
		if err != nil {
			return
		}

		err = ChooseDefaultError(ec)
		if err != nil {
			if IsNoFrame(err) {
				nFrameLoss++
				if nFrameLoss < opts.getFramelossTries() {
					continue
				}
			}
			return
		}

		err = ChooseWorkingCounterError(ec, expwc)
		if err != nil {
			now := time.Now()
			if now.Before(opts.getWCDeadline()) {
				continue
			}
		}

		d = ec.DatagramIn.Data()
		return
	}

	panic("not reached")
}

func ExecuteWrite8(c Commander, addr ecfr.DatagramAddress, w uint8, expwc uint16) (err error) {
	return ExecuteWrite8Options(c, addr, w, expwc, Options{})
}

func ExecuteWrite8Options(c Commander, addr ecfr.DatagramAddress, w uint8, expwc uint16, opts Options) (err error) {
	ws := make([]byte, 1)
	putUint8(ws, w)
	return ExecuteWriteOptions(c, addr, ws, expwc, opts)
}

func ExecuteWrite16(c Commander, addr ecfr.DatagramAddress, w uint16, expwc uint16) (err error) {
	return ExecuteWrite16Options(c, addr, w, expwc, Options{})
}

func ExecuteWrite16Options(c Commander, addr ecfr.DatagramAddress, w uint16, expwc uint16, opts Options) (err error) {
	ws := make([]byte, 2)
	putUint16(ws, w)
	return ExecuteWriteOptions(c, addr, ws, expwc, opts)
}

func ExecuteWrite32(c Commander, addr ecfr.DatagramAddress, w uint32, expwc uint16) (err error) {
	return ExecuteWrite32Options(c, addr, w, expwc, Options{})
}

func ExecuteWrite32Options(c Commander, addr ecfr.DatagramAddress, w uint32, expwc uint16, opts Options) (err error) {
	ws := make([]byte, 4)
	putUint32(ws, w)
	return ExecuteWriteOptions(c, addr, ws, expwc, opts)
}

func ExecuteWrite(c Commander, addr ecfr.DatagramAddress, w []byte, expwc uint16) (err error) {
	return ExecuteWriteOptions(c, addr, w, expwc, Options{})
}

func ExecuteWriteOptions(c Commander, addr ecfr.DatagramAddress, w []byte, expwc uint16, opts Options) (err error) {
	nFrameLoss := 0

	var ct ecfr.CommandType
	switch addr.Type() {
	case ecfr.Positional:
		ct = ecfr.APWR
	case ecfr.Fixed:
		ct = ecfr.FPWR
	case ecfr.Broadcast:
		ct = ecfr.BWR
	default:
		err = fmt.Errorf("ExecuteWriteOptions: unsupported address type %v", addr.Type())
	}

	for {
		var ec *ExecutingCommand
		ec, err = c.New(len(w))
		if err != nil {
			return
		}

		dgo := ec.DatagramOut
		copy(dgo.Data(), w)

		dgo.Command = ct
		dgo.Addr32 = addr.Addr32()

		err = c.Cycle()
		if err != nil {
			return
		}

		err = ChooseDefaultError(ec)
		if err != nil {
			if IsNoFrame(err) {
				nFrameLoss++
				if nFrameLoss < opts.getFramelossTries() {
					continue
				}
			}
			return
		}

		err = ChooseWorkingCounterError(ec, expwc)
		if err != nil {
			now := time.Now()
			if now.Before(opts.getWCDeadline()) {
				continue
			}
		}

		return
	}

}
<|endoftext|>


<commit_before>// +build ignore

// Enumerate unique keys from key/values found in the whois responses.
// To use: go run enumerate.go

package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"regexp"

	"github.com/domainr/whois"
	"github.com/domainr/whoistest"
)

func main() {
	flag.Parse()
	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func main1() error {
	fns, err := whoistest.ResponseFiles()
	if err != nil {
		return err
	}
	for _, fn := range fns {
		res, err := whois.ReadMIMEFile(fn)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error reading response file %s: %s\n", fn, err)
			continue
		}
		if res.MediaType != "text/plain" {
			continue
		}
		scan(res)
	}
	return nil
}

var (
	reEmptyLine   = regexp.MustCompile(`^\s*$`)
	reKeyValue    = regexp.MustCompile(`^\s*([^\:]*\S)\s*\:\s*(.*\S)\s*$`)
	reAltKeyValue = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*(.*\S)\s*$`)
	jpNotice      = `^\[ .+ \]$`
	deNotice      = `^% .*$`
	updated       = `^<<<.+>>>$`
	reNotice      = regexp.MustCompile(jpNotice + "|" + deNotice + "|" + updated)
)

func scan(res *whois.Response) {
	r, err := res.Reader()
	if err != nil {
		return
	}
	line := 0
	s := bufio.NewScanner(r)
	for s.Scan() {
		line++
		text := s.Text()

		if reEmptyLine.MatchString(text) {
			fmt.Printf("% 4d  EMPTY\n", line)
			continue
		}

		if m := reNotice.FindStringSubmatch(text); m != nil {
			fmt.Printf("% 4d  %- 20s  %s\n", line, "NOTICE", m[0])
			continue
		}

		if m := reAltKeyValue.FindStringSubmatch(text); m != nil {
			fmt.Printf("% 4d  %- 20s  %- 30s %s\n", line, "ALT_KEY_VALUE", m[1], m[2])
			continue
		}

		if m := reKeyValue.FindStringSubmatch(text); m != nil {
			fmt.Printf("% 4d  %- 20s  %- 30s %s\n", line, "KEY_VALUE", m[1], m[2])
			continue
		}

		fmt.Fprintf(os.Stderr, "% 4d  %- 20s  %s\n", line, "UNKNOWN", text)
	}
	fmt.Printf("\n")
}
<commit_msg>Scan for bare keys and kr notices<commit_after>// +build ignore

// Enumerate unique keys from key/values found in the whois responses.
// To use: go run enumerate.go

package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"regexp"

	"github.com/domainr/whois"
	"github.com/domainr/whoistest"
)

func main() {
	flag.Parse()
	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func main1() error {
	fns, err := whoistest.ResponseFiles()
	if err != nil {
		return err
	}
	for _, fn := range fns {
		res, err := whois.ReadMIMEFile(fn)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error reading response file %s: %s\n", fn, err)
			continue
		}
		if res.MediaType != "text/plain" {
			continue
		}
		scan(res)
	}
	return nil
}

var (
	reEmptyLine     = regexp.MustCompile(`^\s*$`)
	reKey           = regexp.MustCompile(`^\s*([^\:]*\S)\s*\:\s*$`)
	reKeyValue      = regexp.MustCompile(`^\s*([^\:]*\S)\s*\:\s*(.*\S)\s*$`)
	reAltKey        = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*$`)
	reAltKeyValue   = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*(.*\S)\s*$`)
	reIndentedValue = regexp.MustCompile(`^      \s+(.*\S)\s*$`)
	deNotice        = `^% .*$`
	jpNotice        = `^\[ .+ \]$`
	krNotice        = `^# .*$`
	updated         = `^<<<.+>>>$`
	reNotice        = regexp.MustCompile(
		deNotice + "|" + jpNotice + "|" + krNotice + "|" + updated)
)

func scan(res *whois.Response) {
	r, err := res.Reader()
	if err != nil {
		return
	}
	line := 0
	s := bufio.NewScanner(r)
	for s.Scan() {
		line++
		text := s.Text()

		if reEmptyLine.MatchString(text) {
			fmt.Printf("% 4d  EMPTY\n", line)
			continue
		}

		if m := reNotice.FindStringSubmatch(text); m != nil {
			fmt.Printf("% 4d  %- 20s  %s\n", line, "NOTICE", m[0])
			continue
		}

		if m := reAltKeyValue.FindStringSubmatch(text); m != nil {
			fmt.Printf("% 4d  %- 20s  %- 40s %s\n", line, "ALT_KEY_VALUE", m[1], m[2])
			continue
		}

		if m := reAltKey.FindStringSubmatch(text); m != nil {
			fmt.Printf("% 4d  %- 20s  %s\n", line, "ALT_KEY", m[1])
			continue
		}

		if m := reKeyValue.FindStringSubmatch(text); m != nil {
			fmt.Printf("% 4d  %- 20s  %- 40s %s\n", line, "KEY_VALUE", m[1], m[2])
			continue
		}

		if m := reKey.FindStringSubmatch(text); m != nil {
			fmt.Printf("% 4d  %- 20s  %s\n", line, "KEY", m[1])
			continue
		}

		if m := reIndentedValue.FindStringSubmatch(text); m != nil {
			fmt.Printf("% 4d  %- 20s  %- 40s %s\n", line, "INDENTED_VALUE", "", m[1])
			continue
		}

		fmt.Fprintf(os.Stderr, "% 4d  %- 20s  %s\n", line, "UNKNOWN", text)
	}
	fmt.Printf("\n")
}
<|endoftext|>


<commit_before>package find

import (
	"fmt"
	"github.com/ongaeshi/gomilk/search/file"
	"github.com/ongaeshi/gomilk/search/grep"
	"github.com/ongaeshi/gomilk/search/option"
	"github.com/ongaeshi/gomilk/search/pattern"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
)

type Finder struct {
	Out    chan *grep.Params
	Option *option.Option
}

func (self *Finder) Find(root string, pattern *pattern.Pattern) {
	results, err := self.search(root, []string{pattern.Pattern})

	if err != nil {
		close(self.Out)
		return
	}

	for _, path := range results {
		fileType := ""
		if self.Option.FilesWithRegexp == "" {
			fileType = file.IdentifyType(path)
			if fileType == file.ERROR || fileType == file.BINARY {
				continue
			}
		}
		self.Out <- &grep.Params{path, fileType, pattern}
	}

	close(self.Out)
}

func (self *Finder) search(root string, args []string) ([]string, error) {
	query   := strings.Join(args, " ")
	path, _ := filepath.Abs(root)
	url     := fmt.Sprintf("http://127.0.0.1:9292/gomilk?dir=%s&query=%s", url.QueryEscape(path), url.QueryEscape(query)) // @todo port, address

	if (self.Option.All) {
		url += "&all=1"
	}

	contents, err := readURL(url)

	if err != nil {
		return []string{}, err
	}

	// Get absolute path array from 'milk web -F'
	apaths := strings.Fields(contents)

	if (self.Option.ExpandPath) {
		// abs -> abs
		return apaths, nil

	} else {
		// abs -> relative
		currentDir, _ := filepath.Abs(".")
		rpaths := make([]string, len(apaths))

		for i, apath := range apaths {
			rpath, err := filepath.Rel(currentDir, apath)

			if (err == nil) {
				rpaths[i] = rpath
			} else {
				rpaths[i] = apaths[i]
			}
		}

		return rpaths, nil
	}
}

func readURL(url string) (string, error) {
	response, err := http.Get(url)

	if err != nil {
		return "", err
	}

	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)

	if err != nil {
		return "", err
	}

	return string(contents), nil
}

<commit_msg>It displays error message when you forget "-g" option at "milk web"<commit_after>package find

import (
	"fmt"
	"github.com/ongaeshi/gomilk/search/file"
	"github.com/ongaeshi/gomilk/search/grep"
	"github.com/ongaeshi/gomilk/search/option"
	"github.com/ongaeshi/gomilk/search/pattern"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

type Finder struct {
	Out    chan *grep.Params
	Option *option.Option
}

func (self *Finder) Find(root string, pattern *pattern.Pattern) {
	results, err := self.search(root, []string{pattern.Pattern})

	if err != nil {
		close(self.Out)
		return
	}

	for _, path := range results {
		fileType := ""
		if self.Option.FilesWithRegexp == "" {
			fileType = file.IdentifyType(path)
			if fileType == file.ERROR || fileType == file.BINARY {
				continue
			}
		}
		self.Out <- &grep.Params{path, fileType, pattern}
	}

	close(self.Out)
}

func (self *Finder) search(root string, args []string) ([]string, error) {
	query   := strings.Join(args, " ")
	path, _ := filepath.Abs(root)
	url     := fmt.Sprintf("http://127.0.0.1:9292/gomilk?dir=%s&query=%s", url.QueryEscape(path), url.QueryEscape(query)) // @todo port, address

	if (self.Option.All) {
		url += "&all=1"
	}

	contents, err := readURL(url)

	if err != nil {
		fmt.Printf("%s\n", err)
		fmt.Printf("Need \"milk web --gomilk\"\n")
		os.Exit(1)
	}

	if contents == "Error:" {
		fmt.Printf("Get %s: response is \"Error:\"\n", url)
		fmt.Printf("Need \"milk web --gomilk\"\n")
		os.Exit(1)
	}

	// Get absolute path array from 'milk web -F'
	apaths := strings.Fields(contents)

	if (self.Option.ExpandPath) {
		// abs -> abs
		return apaths, nil

	} else {
		// abs -> relative
		currentDir, _ := filepath.Abs(".")
		rpaths := make([]string, len(apaths))

		for i, apath := range apaths {
			rpath, err := filepath.Rel(currentDir, apath)

			if (err == nil) {
				rpaths[i] = rpath
			} else {
				rpaths[i] = apaths[i]
			}
		}

		return rpaths, nil
	}
}

func readURL(url string) (string, error) {
	response, err := http.Get(url)

	if err != nil {
		return "", err
	}

	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)

	if err != nil {
		return "", err
	}

	return string(contents), nil
}

<|endoftext|>


<commit_before>package gometer

import (
	"bytes"
	"fmt"
	"sort"
)

// Formatter is used to determine a format of metrics representation.
type Formatter interface {
	// Format is defined how metrics will be dumped
	// to output destination.
	Format(counters map[string]Counter) []byte
}

// NewFormatter returns new default formatter.
//
// lineSeparator determines how one line of metric
// will be separated from another.
//
// As line separator can be used any symbol: e.g. '\n', ':', '.', ','.
//
// Default format for one line of metrics is: "%v = %v".
// defaultFormatter sorts metrics by value.
func NewFormatter(lineSeparator string) Formatter {
	df := &defaultFormatter{
		lineSeparator: lineSeparator,
	}
	return df
}

type sortedMap struct {
	m map[string]Counter
	s []string
}

func (sm *sortedMap) Len() int {
	return len(sm.m)
}

func (sm *sortedMap) Less(i, j int) bool {
	return sm.m[sm.s[i]].Get() < sm.m[sm.s[j]].Get()
}

func (sm *sortedMap) Swap(i, j int) {
	sm.s[i], sm.s[j] = sm.s[j], sm.s[i]
}

func sortedKeys(m map[string]Counter) []string {
	sm := new(sortedMap)
	sm.m = m
	sm.s = make([]string, len(m))
	i := 0
	for key := range m {
		sm.s[i] = key
		i++
	}
	sort.Sort(sm)
	return sm.s
}

type defaultFormatter struct {
	lineSeparator string
}

func (f *defaultFormatter) Format(counters map[string]Counter) []byte {
	var buf bytes.Buffer

	for _, n := range sortedKeys(counters) {
		line := fmt.Sprintf("%v = %v", n, counters[n].Get()) + f.lineSeparator
		fmt.Fprintf(&buf, line)
	}

	return buf.Bytes()
}

var _ Formatter = (*defaultFormatter)(nil)
<commit_msg>fix sortedKeys to sort by key<commit_after>package gometer

import (
	"bytes"
	"fmt"
	"sort"
)

// Formatter is used to determine a format of metrics representation.
type Formatter interface {
	// Format is defined how metrics will be dumped
	// to output destination.
	Format(counters map[string]Counter) []byte
}

// NewFormatter returns new default formatter.
//
// lineSeparator determines how one line of metric
// will be separated from another.
//
// As line separator can be used any symbol: e.g. '\n', ':', '.', ','.
//
// Default format for one line of metrics is: "%v = %v".
// defaultFormatter sorts metrics by value.
func NewFormatter(lineSeparator string) Formatter {
	df := &defaultFormatter{
		lineSeparator: lineSeparator,
	}
	return df
}

func sortedKeys(m map[string]Counter) []string {
	s := make([]string, 0, len(m))
	for key := range m {
		s = append(s, key)
	}
	sort.Strings(s)
	return s
}

type defaultFormatter struct {
	lineSeparator string
}

func (f *defaultFormatter) Format(counters map[string]Counter) []byte {
	var buf bytes.Buffer

	for _, n := range sortedKeys(counters) {
		line := fmt.Sprintf("%v = %v", n, counters[n].Get()) + f.lineSeparator
		fmt.Fprintf(&buf, line)
	}

	return buf.Bytes()
}

var _ Formatter = (*defaultFormatter)(nil)
<|endoftext|>


<commit_before>package textformatter

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/mgutz/ansi"
	"github.com/sirupsen/logrus"
	"golang.org/x/crypto/ssh/terminal"
)

const defaultTimestampFormat = time.RFC3339Nano

type ColorScheme struct {
	Debug  string
	Info   string
	Warn   string
	Error  string
	Fatal  string
	Panic  string
	Prefix string
	Func   string
}

type colorFunc func(string) string

type compiledColorScheme struct {
	Debug  colorFunc
	Info   colorFunc
	Warn   colorFunc
	Error  colorFunc
	Fatal  colorFunc
	Panic  colorFunc
	Prefix colorFunc
	Func   colorFunc
}

type Instance struct {
	// Use colors if TTY detected
	UseColors bool

	// Disable timestamp logging. useful when output is redirected to logging
	// system that already adds timestamps.
	DisableTimestamp bool

	// Print level names in `lowercase` instead of `UPPERCASE`
	LowercaseLevels bool

	// Enable logging the full timestamp when a TTY is attached instead of just
	// the time passed since beginning of execution.
	FullTimestamp bool

	// Timestamp format to use for display when a full timestamp is printed.
	TimestampFormat string

	PrefixFieldName  string
	PrefixFieldWidth int
	FuncFieldName    string

	colorScheme *compiledColorScheme

	sync.Once
}

func nocolor(v string) string {
	return v
}

var (
	baseTimestamp time.Time    = time.Now()
	defaultColors *ColorScheme = &ColorScheme{
		Debug:  "black+h",
		Info:   "green",
		Warn:   "yellow",
		Error:  "red",
		Fatal:  "red+h",
		Panic:  "red+h",
		Prefix: "cyan",
		Func:   "white",
	}
	noColors *compiledColorScheme = &compiledColorScheme{
		Debug:  nocolor,
		Info:   nocolor,
		Warn:   nocolor,
		Error:  nocolor,
		Fatal:  nocolor,
		Panic:  nocolor,
		Prefix: nocolor,
		Func:   nocolor,
	}
	defaultCompiledColorScheme *compiledColorScheme = compileColorScheme(defaultColors)
)

func miniTS() float64 {
	return time.Since(baseTimestamp).Seconds()
}

func getCompiledColor(main string, fallback string) colorFunc {
	var style string
	if main != "" {
		style = main
	} else {
		style = fallback
	}
	return ansi.ColorFunc(style)
}

func compileColorScheme(s *ColorScheme) *compiledColorScheme {
	return &compiledColorScheme{
		Info:   getCompiledColor(s.Info, defaultColors.Info),
		Warn:   getCompiledColor(s.Warn, defaultColors.Warn),
		Error:  getCompiledColor(s.Error, defaultColors.Error),
		Fatal:  getCompiledColor(s.Fatal, defaultColors.Fatal),
		Panic:  getCompiledColor(s.Panic, defaultColors.Panic),
		Debug:  getCompiledColor(s.Debug, defaultColors.Debug),
		Prefix: getCompiledColor(s.Prefix, defaultColors.Prefix),
		Func:   getCompiledColor(s.Func, defaultColors.Func),
	}
}

func (f *Instance) checkIfTerminal(w io.Writer) bool {
	switch v := w.(type) {
	case *os.File:
		return terminal.IsTerminal(int(v.Fd()))
	default:
		return false
	}
}

func (f *Instance) SetColorScheme(colorScheme *ColorScheme) {
	f.colorScheme = compileColorScheme(colorScheme)
}

func (f *Instance) Format(entry *logrus.Entry) ([]byte, error) {
	// init
	f.Once.Do(func() {
		if len(f.PrefixFieldName) == 0 {
			f.PrefixFieldName = "__p"
		}
		if len(f.FuncFieldName) == 0 {
			f.FuncFieldName = "__f"
		}
		if len(f.TimestampFormat) == 0 {
			f.TimestampFormat = defaultTimestampFormat
		}
		if f.colorScheme == nil {
			if f.UseColors {
				f.colorScheme = defaultCompiledColorScheme
			} else {
				f.colorScheme = noColors
			}
		}
	})

	var buf *bytes.Buffer
	if entry.Buffer != nil {
		buf = entry.Buffer
	} else {
		buf = &bytes.Buffer{}
	}

	var levelColor colorFunc
	var levelText string
	switch entry.Level {
	case logrus.InfoLevel:
		levelColor = f.colorScheme.Info
	case logrus.WarnLevel:
		levelColor = f.colorScheme.Warn
	case logrus.ErrorLevel:
		levelColor = f.colorScheme.Error
	case logrus.FatalLevel:
		levelColor = f.colorScheme.Fatal
	case logrus.PanicLevel:
		levelColor = f.colorScheme.Panic
	default:
		levelColor = f.colorScheme.Debug
	}

	if entry.Level != logrus.WarnLevel {
		levelText = entry.Level.String()
	} else {
		levelText = "warn"
	}

	if !f.LowercaseLevels {
		levelText = strings.ToUpper(levelText)
	}

	if !f.DisableTimestamp {
		var ts string
		if !f.FullTimestamp {
			ts = fmt.Sprintf("[%f]", miniTS())
		} else {
			ts = fmt.Sprintf("[%s]", entry.Time.Format(f.TimestampFormat))
		}
		fmt.Fprint(buf, levelColor(ts), " ")
	}

	fmt.Fprint(buf, levelColor(fmt.Sprintf("%5s", levelText)))

	var fstr string

	// Prefix
	if v, ok := entry.Data[f.PrefixFieldName]; ok {
		fstr = fmt.Sprintf("%v", v)
	} else {
		fstr = f.PrefixFieldName + "<missing>"
	}
	flen := len(fstr)

	fmt.Fprint(buf, " ", f.colorScheme.Prefix(fstr))

	if flen < f.PrefixFieldWidth {
		fmt.Fprint(buf, strings.Repeat(" ", int(f.PrefixFieldWidth-flen)+1))
	} else {
		fmt.Fprint(buf, " ")
	}

	// Func
	if v, ok := entry.Data[f.FuncFieldName]; ok {
		fmt.Fprint(buf, " ", f.colorScheme.Func(fmt.Sprintf("%v", v)))
	}

	// Message
	fmt.Fprint(buf, " ", levelColor(entry.Message))

	var errpresent bool
	if v, ok := entry.Data[logrus.ErrorKey]; ok {
		errpresent = true
		printField(buf, logrus.ErrorKey, v, f.colorScheme.Func, levelColor, true)
	}

	keys := make([]string, 0, len(entry.Data))
	for k := range entry.Data {
		switch k {
		case f.PrefixFieldName, f.FuncFieldName, logrus.ErrorKey:
			continue
		default:
			keys = append(keys, k)
		}
	}
	sort.Strings(keys)

	for n, k := range keys {
		v := entry.Data[k]
		printField(buf, k, v, f.colorScheme.Func, levelColor, n == 0 && !errpresent)
	}
	if errpresent || len(keys) > 0 {
		fmt.Fprint(buf, ")")
	}
	fmt.Fprint(buf, "\n")

	return buf.Bytes(), nil
}

func printField(w io.Writer, key string, val interface{}, kcolor, vcolor colorFunc, first bool) {
	if first {
		fmt.Fprint(w, " (")
	} else {
		fmt.Fprint(w, " ")
	}
	switch v := val.(type) {
	case fmt.Stringer:
		fmt.Fprintf(w, "%s=%s", kcolor(key), vcolor(v.String()))
	case error:
		fmt.Fprintf(w, "%s={%s}", kcolor(key), vcolor(v.Error()))
	default:
		fmt.Fprintf(w, "%s=%s", kcolor(key), vcolor(fmt.Sprintf("%#v", v)))
	}
}
<commit_msg>Level name now prints after prefix<commit_after>package textformatter

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/mgutz/ansi"
	"github.com/sirupsen/logrus"
	"golang.org/x/crypto/ssh/terminal"
)

const defaultTimestampFormat = time.RFC3339Nano

type ColorScheme struct {
	Debug  string
	Info   string
	Warn   string
	Error  string
	Fatal  string
	Panic  string
	Prefix string
	Func   string
}

type colorFunc func(string) string

type compiledColorScheme struct {
	Debug  colorFunc
	Info   colorFunc
	Warn   colorFunc
	Error  colorFunc
	Fatal  colorFunc
	Panic  colorFunc
	Prefix colorFunc
	Func   colorFunc
}

type Instance struct {
	// Use colors if TTY detected
	UseColors bool

	// Disable timestamp logging. useful when output is redirected to logging
	// system that already adds timestamps.
	DisableTimestamp bool

	// Print level names in `lowercase` instead of `UPPERCASE`
	LowercaseLevels bool

	// Enable logging the full timestamp when a TTY is attached instead of just
	// the time passed since beginning of execution.
	FullTimestamp bool

	// Timestamp format to use for display when a full timestamp is printed.
	TimestampFormat string

	PrefixFieldName  string
	PrefixFieldWidth int
	FuncFieldName    string

	colorScheme *compiledColorScheme

	sync.Once
}

func nocolor(v string) string {
	return v
}

var (
	baseTimestamp time.Time    = time.Now()
	defaultColors *ColorScheme = &ColorScheme{
		Debug:  "black+h",
		Info:   "green",
		Warn:   "yellow",
		Error:  "red",
		Fatal:  "red+h",
		Panic:  "red+h",
		Prefix: "cyan",
		Func:   "white",
	}
	noColors *compiledColorScheme = &compiledColorScheme{
		Debug:  nocolor,
		Info:   nocolor,
		Warn:   nocolor,
		Error:  nocolor,
		Fatal:  nocolor,
		Panic:  nocolor,
		Prefix: nocolor,
		Func:   nocolor,
	}
	defaultCompiledColorScheme *compiledColorScheme = compileColorScheme(defaultColors)
)

func miniTS() float64 {
	return time.Since(baseTimestamp).Seconds()
}

func getCompiledColor(main string, fallback string) colorFunc {
	var style string
	if main != "" {
		style = main
	} else {
		style = fallback
	}
	return ansi.ColorFunc(style)
}

func compileColorScheme(s *ColorScheme) *compiledColorScheme {
	return &compiledColorScheme{
		Info:   getCompiledColor(s.Info, defaultColors.Info),
		Warn:   getCompiledColor(s.Warn, defaultColors.Warn),
		Error:  getCompiledColor(s.Error, defaultColors.Error),
		Fatal:  getCompiledColor(s.Fatal, defaultColors.Fatal),
		Panic:  getCompiledColor(s.Panic, defaultColors.Panic),
		Debug:  getCompiledColor(s.Debug, defaultColors.Debug),
		Prefix: getCompiledColor(s.Prefix, defaultColors.Prefix),
		Func:   getCompiledColor(s.Func, defaultColors.Func),
	}
}

func (f *Instance) checkIfTerminal(w io.Writer) bool {
	switch v := w.(type) {
	case *os.File:
		return terminal.IsTerminal(int(v.Fd()))
	default:
		return false
	}
}

func (f *Instance) SetColorScheme(colorScheme *ColorScheme) {
	f.colorScheme = compileColorScheme(colorScheme)
}

func (f *Instance) Format(entry *logrus.Entry) ([]byte, error) {
	// init
	f.Once.Do(func() {
		if len(f.PrefixFieldName) == 0 {
			f.PrefixFieldName = "__p"
		}
		if len(f.FuncFieldName) == 0 {
			f.FuncFieldName = "__f"
		}
		if len(f.TimestampFormat) == 0 {
			f.TimestampFormat = defaultTimestampFormat
		}
		if f.colorScheme == nil {
			if f.UseColors {
				f.colorScheme = defaultCompiledColorScheme
			} else {
				f.colorScheme = noColors
			}
		}
	})

	var buf *bytes.Buffer
	if entry.Buffer != nil {
		buf = entry.Buffer
	} else {
		buf = &bytes.Buffer{}
	}

	var levelColor colorFunc
	var levelText string
	switch entry.Level {
	case logrus.InfoLevel:
		levelColor = f.colorScheme.Info
	case logrus.WarnLevel:
		levelColor = f.colorScheme.Warn
	case logrus.ErrorLevel:
		levelColor = f.colorScheme.Error
	case logrus.FatalLevel:
		levelColor = f.colorScheme.Fatal
	case logrus.PanicLevel:
		levelColor = f.colorScheme.Panic
	default:
		levelColor = f.colorScheme.Debug
	}

	if entry.Level != logrus.WarnLevel {
		levelText = entry.Level.String()
	} else {
		levelText = "warn"
	}

	if !f.LowercaseLevels {
		levelText = strings.ToUpper(levelText)
	}

	if !f.DisableTimestamp {
		var ts string
		if !f.FullTimestamp {
			ts = fmt.Sprintf("[%f]", miniTS())
		} else {
			ts = fmt.Sprintf("[%s]", entry.Time.Format(f.TimestampFormat))
		}
		fmt.Fprint(buf, levelColor(ts), " ")
	}

	var fstr string

	// Prefix
	if v, ok := entry.Data[f.PrefixFieldName]; ok {
		fstr = fmt.Sprintf("%v", v)
	} else {
		fstr = f.PrefixFieldName + "<missing>"
	}
	flen := len(fstr)

	fmt.Fprint(buf, " ", f.colorScheme.Prefix(fstr))

	if flen < f.PrefixFieldWidth {
		fmt.Fprint(buf, strings.Repeat(" ", int(f.PrefixFieldWidth-flen)+1))
	} else {
		fmt.Fprint(buf, " ")
	}

	fmt.Fprint(buf, levelColor(fmt.Sprintf("%5s", levelText)))

	// Func
	if v, ok := entry.Data[f.FuncFieldName]; ok {
		fmt.Fprint(buf, " ", f.colorScheme.Func(fmt.Sprintf("%v", v)))
	}

	// Message
	fmt.Fprint(buf, " ", levelColor(entry.Message))

	var errpresent bool
	if v, ok := entry.Data[logrus.ErrorKey]; ok {
		errpresent = true
		printField(buf, logrus.ErrorKey, v, f.colorScheme.Func, levelColor, true)
	}

	keys := make([]string, 0, len(entry.Data))
	for k := range entry.Data {
		switch k {
		case f.PrefixFieldName, f.FuncFieldName, logrus.ErrorKey:
			continue
		default:
			keys = append(keys, k)
		}
	}
	sort.Strings(keys)

	for n, k := range keys {
		v := entry.Data[k]
		printField(buf, k, v, f.colorScheme.Func, levelColor, n == 0 && !errpresent)
	}
	if errpresent || len(keys) > 0 {
		fmt.Fprint(buf, ")")
	}
	fmt.Fprint(buf, "\n")

	return buf.Bytes(), nil
}

func printField(w io.Writer, key string, val interface{}, kcolor, vcolor colorFunc, first bool) {
	if first {
		fmt.Fprint(w, " (")
	} else {
		fmt.Fprint(w, " ")
	}
	switch v := val.(type) {
	case fmt.Stringer:
		fmt.Fprintf(w, "%s=%s", kcolor(key), vcolor(v.String()))
	case error:
		fmt.Fprintf(w, "%s={%s}", kcolor(key), vcolor(v.Error()))
	default:
		fmt.Fprintf(w, "%s=%s", kcolor(key), vcolor(fmt.Sprintf("%#v", v)))
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Alexandre Fiori
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"bytes"
	"database/sql"
	"encoding/binary"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/fiorix/go-redis/redis"
	"github.com/fiorix/go-web/httpxtra"
	_ "github.com/mattn/go-sqlite3"
)

type Settings struct {
	XMLName      xml.Name `xml:"Server"`
	Debug        bool     `xml:"debug,attr"`
	XHeaders     bool     `xml:"xheaders,attr"`
	Addr         string   `xml:"addr,attr"`
	DocumentRoot string
	IPDB         struct {
		File      string `xml:",attr"`
		CacheSize string `xml:",attr"`
	}
	Limit struct {
		MaxRequests int `xml:",attr"`
		Expire      int `xml:",attr"`
	}
	Redis []string `xml:"Redis>Addr"`
}

var conf *Settings

func main() {
	if buf, err := ioutil.ReadFile("freegeoip.conf"); err != nil {
		panic(err)
	} else {
		conf = &Settings{}
		if err := xml.Unmarshal(buf, conf); err != nil {
			panic(err)
		}
	}
	http.Handle("/", http.FileServer(http.Dir(conf.DocumentRoot)))
	h := GeoipHandler()
	http.HandleFunc("/csv/", h)
	http.HandleFunc("/xml/", h)
	http.HandleFunc("/json/", h)
	server := http.Server{
		Addr: conf.Addr,
		Handler: httpxtra.Handler{
			Logger:   logger,
			XHeaders: conf.XHeaders,
		},
		ReadTimeout:  15 * time.Second,
		WriteTimeout: 15 * time.Second,
	}
	log.Println("FreeGeoIP server starting")
	if e := httpxtra.ListenAndServe(server); e != nil {
		log.Println(e.Error())
	}
}

func logger(r *http.Request, created time.Time, status, bytes int) {
	//fmt.Println(httpxtra.ApacheCommonLog(r, created, status, bytes))
	log.Printf("HTTP %d %s %s (%s) :: %s",
		status,
		r.Method,
		r.URL.Path,
		r.RemoteAddr,
		time.Since(created))
}

// GeoipHandler handles GET on /csv, /xml and /json.
func GeoipHandler() http.HandlerFunc {
	db, err := sql.Open("sqlite3", conf.IPDB.File)
	if err != nil {
		panic(err)
	}
	_, err = db.Exec("PRAGMA cache_size=" + conf.IPDB.CacheSize)
	if err != nil {
		panic(err)
	}
	stmt, err := db.Prepare(query)
	if err != nil {
		panic(err)
	}
	//defer stmt.Close()
	rc := redis.New(conf.Redis...)
	return func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "GET":
			w.Header().Set("Access-Control-Allow-Origin", "*")
		case "OPTIONS":
			w.Header().Set("Access-Control-Allow-Origin", "*")
			w.Header().Set("Content-Type", "text/plain")
			w.Header().Set("Access-Control-Allow-Methods", "GET")
			w.Header().Set("Access-Control-Allow-Headers", "X-Requested-With")
			w.WriteHeader(200)
			return
		default:
			w.Header().Set("Allow", "GET, OPTIONS")
			http.Error(w, http.StatusText(405), 405)
			return
		}
		// GET
		// Check quota
		var ipkey string
		if ip, _, err := net.SplitHostPort(r.RemoteAddr); err != nil {
			ipkey = r.RemoteAddr // support for XHeaders
		} else {
			ipkey = ip
		}
		if qcs, err := rc.Get(ipkey); err != nil {
			if conf.Debug {
				log.Println("Redis error:", err.Error())
			}
			http.Error(w, http.StatusText(503), 503) // redis down
			return
		} else if qcs == "" {
			if err := rc.Set(ipkey, "1"); err == nil {
				rc.Expire(ipkey, conf.Limit.Expire)
			}
		} else if qc, _ := strconv.Atoi(qcs); qc < conf.Limit.MaxRequests {
			rc.Incr(ipkey)
		} else {
			// Out of quota, soz :(
			http.Error(w, http.StatusText(403), 403)
			return
		}
		// Parse URL and build the query.
		var ip string
		a := strings.SplitN(r.URL.Path, "/", 3)
		if len(a) == 3 && a[2] != "" {
			// e.g. /csv/google.com
			addrs, err := net.LookupHost(a[2])
			if err != nil {
				http.Error(w, http.StatusText(404), 404)
				return
			}
			ip = addrs[0]
		} else {
			ip = ipkey
		}
		geoip, err := GeoipLookup(stmt, ip)
		if err != nil {
			http.NotFound(w, r)
			return
		}
		switch a[1][0] {
		case 'c':
			w.Header().Set("Content-Type", "application/csv")
			fmt.Fprintf(w, `"%s","%s","%s","%s","%s","%s",`+
				`"%s","%0.4f","%0.4f","%s","%s"`+"\r\n",
				geoip.Ip,
				geoip.CountryCode, geoip.CountryName,
				geoip.RegionCode, geoip.RegionName,
				geoip.CityName, geoip.ZipCode,
				geoip.Latitude, geoip.Longitude,
				geoip.MetroCode, geoip.AreaCode)
		case 'j':
			resp, err := json.Marshal(geoip)
			if err != nil {
				if conf.Debug {
					log.Println("JSON error:", err.Error())
				}
				http.NotFound(w, r)
				return
			}
			callback := r.FormValue("callback")
			if callback != "" {
				w.Header().Set("Content-Type", "text/javascript")
				fmt.Fprintf(w, "%s(%s);", callback, resp)
			} else {
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintf(w, "%s", resp)
			}
		case 'x':
			w.Header().Set("Content-Type", "application/xml")
			resp, err := xml.MarshalIndent(geoip, "", " ")
			if err != nil {
				if conf.Debug {
					log.Println("XML error:", err.Error())
				}
				http.Error(w, http.StatusText(500), 500)
				return
			}
			fmt.Fprintf(w, xml.Header+"%s\n", resp)
		}
	}
}

const query = `SELECT
  city_location.country_code,
  country_blocks.country_name,
  city_location.region_code,
  region_names.region_name,
  city_location.city_name,
  city_location.postal_code,
  city_location.latitude,
  city_location.longitude,
  city_location.metro_code,
  city_location.area_code
FROM city_blocks
  NATURAL JOIN city_location
  INNER JOIN country_blocks ON
    city_location.country_code = country_blocks.country_code
  LEFT OUTER JOIN region_names ON
    city_location.country_code = region_names.country_code
    AND
    city_location.region_code = region_names.region_code
WHERE city_blocks.ip_start <= ?
ORDER BY city_blocks.ip_start DESC LIMIT 1`

func GeoipLookup(stmt *sql.Stmt, ip string) (*GeoIP, error) {
	IP := net.ParseIP(ip)
	reserved := false
	for _, net := range reservedIPs {
		if net.Contains(IP) {
			reserved = true
			break
		}
	}
	geoip := GeoIP{Ip: ip}
	if reserved {
		geoip.CountryCode = "RD"
		geoip.CountryName = "Reserved"
	} else {
		var uintIP uint32
		b := bytes.NewBuffer(IP.To4())
		binary.Read(b, binary.BigEndian, &uintIP)
		if err := stmt.QueryRow(uintIP).Scan(
			&geoip.CountryCode,
			&geoip.CountryName,
			&geoip.RegionCode,
			&geoip.RegionName,
			&geoip.CityName,
			&geoip.ZipCode,
			&geoip.Latitude,
			&geoip.Longitude,
			&geoip.MetroCode,
			&geoip.AreaCode,
		); err != nil {
			fmt.Println("ERR:", err)
			return nil, err
		}
	}
	return &geoip, nil
}

type GeoIP struct {
	XMLName     xml.Name `json:"-" xml:"Response"`
	Ip          string   `json:"ip"`
	CountryCode string   `json:"country_code"`
	CountryName string   `json:"country_name"`
	RegionCode  string   `json:"region_code"`
	RegionName  string   `json:"region_name"`
	CityName    string   `json:"city" xml:"City"`
	ZipCode     string   `json:"zipcode"`
	Latitude    float32  `json:"latitude"`
	Longitude   float32  `json:"longitude"`
	MetroCode   string   `json:"metro_code"`
	AreaCode    string   `json:"areacode"`
}

// http://en.wikipedia.org/wiki/Reserved_IP_addresses
var reservedIPs = []net.IPNet{
	{net.IPv4(0, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(10, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(100, 64, 0, 0), net.IPv4Mask(255, 192, 0, 0)},
	{net.IPv4(127, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(169, 254, 0, 0), net.IPv4Mask(255, 255, 0, 0)},
	{net.IPv4(172, 16, 0, 0), net.IPv4Mask(255, 240, 0, 0)},
	{net.IPv4(192, 0, 0, 0), net.IPv4Mask(255, 255, 255, 248)},
	{net.IPv4(192, 0, 2, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(192, 88, 99, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(192, 168, 0, 0), net.IPv4Mask(255, 255, 0, 0)},
	{net.IPv4(198, 18, 0, 0), net.IPv4Mask(255, 254, 0, 0)},
	{net.IPv4(198, 51, 100, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(203, 0, 113, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(224, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)},
	{net.IPv4(240, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)},
	{net.IPv4(255, 255, 255, 255), net.IPv4Mask(255, 255, 255, 255)},
}
<commit_msg>Do not print error msg<commit_after>// Copyright 2013 Alexandre Fiori
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"bytes"
	"database/sql"
	"encoding/binary"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/fiorix/go-redis/redis"
	"github.com/fiorix/go-web/httpxtra"
	_ "github.com/mattn/go-sqlite3"
)

type Settings struct {
	XMLName      xml.Name `xml:"Server"`
	Debug        bool     `xml:"debug,attr"`
	XHeaders     bool     `xml:"xheaders,attr"`
	Addr         string   `xml:"addr,attr"`
	DocumentRoot string
	IPDB         struct {
		File      string `xml:",attr"`
		CacheSize string `xml:",attr"`
	}
	Limit struct {
		MaxRequests int `xml:",attr"`
		Expire      int `xml:",attr"`
	}
	Redis []string `xml:"Redis>Addr"`
}

var conf *Settings

func main() {
	if buf, err := ioutil.ReadFile("freegeoip.conf"); err != nil {
		panic(err)
	} else {
		conf = &Settings{}
		if err := xml.Unmarshal(buf, conf); err != nil {
			panic(err)
		}
	}
	http.Handle("/", http.FileServer(http.Dir(conf.DocumentRoot)))
	h := GeoipHandler()
	http.HandleFunc("/csv/", h)
	http.HandleFunc("/xml/", h)
	http.HandleFunc("/json/", h)
	server := http.Server{
		Addr: conf.Addr,
		Handler: httpxtra.Handler{
			Logger:   logger,
			XHeaders: conf.XHeaders,
		},
		ReadTimeout:  15 * time.Second,
		WriteTimeout: 15 * time.Second,
	}
	log.Println("FreeGeoIP server starting")
	if e := httpxtra.ListenAndServe(server); e != nil {
		log.Println(e.Error())
	}
}

func logger(r *http.Request, created time.Time, status, bytes int) {
	//fmt.Println(httpxtra.ApacheCommonLog(r, created, status, bytes))
	log.Printf("HTTP %d %s %s (%s) :: %s",
		status,
		r.Method,
		r.URL.Path,
		r.RemoteAddr,
		time.Since(created))
}

// GeoipHandler handles GET on /csv, /xml and /json.
func GeoipHandler() http.HandlerFunc {
	db, err := sql.Open("sqlite3", conf.IPDB.File)
	if err != nil {
		panic(err)
	}
	_, err = db.Exec("PRAGMA cache_size=" + conf.IPDB.CacheSize)
	if err != nil {
		panic(err)
	}
	stmt, err := db.Prepare(query)
	if err != nil {
		panic(err)
	}
	//defer stmt.Close()
	rc := redis.New(conf.Redis...)
	return func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "GET":
			w.Header().Set("Access-Control-Allow-Origin", "*")
		case "OPTIONS":
			w.Header().Set("Access-Control-Allow-Origin", "*")
			w.Header().Set("Content-Type", "text/plain")
			w.Header().Set("Access-Control-Allow-Methods", "GET")
			w.Header().Set("Access-Control-Allow-Headers", "X-Requested-With")
			w.WriteHeader(200)
			return
		default:
			w.Header().Set("Allow", "GET, OPTIONS")
			http.Error(w, http.StatusText(405), 405)
			return
		}
		// GET
		// Check quota
		var ipkey string
		if ip, _, err := net.SplitHostPort(r.RemoteAddr); err != nil {
			ipkey = r.RemoteAddr // support for XHeaders
		} else {
			ipkey = ip
		}
		if qcs, err := rc.Get(ipkey); err != nil {
			if conf.Debug {
				log.Println("Redis error:", err.Error())
			}
			http.Error(w, http.StatusText(503), 503) // redis down
			return
		} else if qcs == "" {
			if err := rc.Set(ipkey, "1"); err == nil {
				rc.Expire(ipkey, conf.Limit.Expire)
			}
		} else if qc, _ := strconv.Atoi(qcs); qc < conf.Limit.MaxRequests {
			rc.Incr(ipkey)
		} else {
			// Out of quota, soz :(
			http.Error(w, http.StatusText(403), 403)
			return
		}
		// Parse URL and build the query.
		var ip string
		a := strings.SplitN(r.URL.Path, "/", 3)
		if len(a) == 3 && a[2] != "" {
			// e.g. /csv/google.com
			addrs, err := net.LookupHost(a[2])
			if err != nil {
				http.Error(w, http.StatusText(404), 404)
				return
			}
			ip = addrs[0]
		} else {
			ip = ipkey
		}
		geoip, err := GeoipLookup(stmt, ip)
		if err != nil {
			http.NotFound(w, r)
			return
		}
		switch a[1][0] {
		case 'c':
			w.Header().Set("Content-Type", "application/csv")
			fmt.Fprintf(w, `"%s","%s","%s","%s","%s","%s",`+
				`"%s","%0.4f","%0.4f","%s","%s"`+"\r\n",
				geoip.Ip,
				geoip.CountryCode, geoip.CountryName,
				geoip.RegionCode, geoip.RegionName,
				geoip.CityName, geoip.ZipCode,
				geoip.Latitude, geoip.Longitude,
				geoip.MetroCode, geoip.AreaCode)
		case 'j':
			resp, err := json.Marshal(geoip)
			if err != nil {
				if conf.Debug {
					log.Println("JSON error:", err.Error())
				}
				http.NotFound(w, r)
				return
			}
			callback := r.FormValue("callback")
			if callback != "" {
				w.Header().Set("Content-Type", "text/javascript")
				fmt.Fprintf(w, "%s(%s);", callback, resp)
			} else {
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintf(w, "%s", resp)
			}
		case 'x':
			w.Header().Set("Content-Type", "application/xml")
			resp, err := xml.MarshalIndent(geoip, "", " ")
			if err != nil {
				if conf.Debug {
					log.Println("XML error:", err.Error())
				}
				http.Error(w, http.StatusText(500), 500)
				return
			}
			fmt.Fprintf(w, xml.Header+"%s\n", resp)
		}
	}
}

const query = `SELECT
  city_location.country_code,
  country_blocks.country_name,
  city_location.region_code,
  region_names.region_name,
  city_location.city_name,
  city_location.postal_code,
  city_location.latitude,
  city_location.longitude,
  city_location.metro_code,
  city_location.area_code
FROM city_blocks
  NATURAL JOIN city_location
  INNER JOIN country_blocks ON
    city_location.country_code = country_blocks.country_code
  LEFT OUTER JOIN region_names ON
    city_location.country_code = region_names.country_code
    AND
    city_location.region_code = region_names.region_code
WHERE city_blocks.ip_start <= ?
ORDER BY city_blocks.ip_start DESC LIMIT 1`

func GeoipLookup(stmt *sql.Stmt, ip string) (*GeoIP, error) {
	IP := net.ParseIP(ip)
	reserved := false
	for _, net := range reservedIPs {
		if net.Contains(IP) {
			reserved = true
			break
		}
	}
	geoip := GeoIP{Ip: ip}
	if reserved {
		geoip.CountryCode = "RD"
		geoip.CountryName = "Reserved"
	} else {
		var uintIP uint32
		b := bytes.NewBuffer(IP.To4())
		binary.Read(b, binary.BigEndian, &uintIP)
		if err := stmt.QueryRow(uintIP).Scan(
			&geoip.CountryCode,
			&geoip.CountryName,
			&geoip.RegionCode,
			&geoip.RegionName,
			&geoip.CityName,
			&geoip.ZipCode,
			&geoip.Latitude,
			&geoip.Longitude,
			&geoip.MetroCode,
			&geoip.AreaCode,
		); err != nil {
			return nil, err
		}
	}
	return &geoip, nil
}

type GeoIP struct {
	XMLName     xml.Name `json:"-" xml:"Response"`
	Ip          string   `json:"ip"`
	CountryCode string   `json:"country_code"`
	CountryName string   `json:"country_name"`
	RegionCode  string   `json:"region_code"`
	RegionName  string   `json:"region_name"`
	CityName    string   `json:"city" xml:"City"`
	ZipCode     string   `json:"zipcode"`
	Latitude    float32  `json:"latitude"`
	Longitude   float32  `json:"longitude"`
	MetroCode   string   `json:"metro_code"`
	AreaCode    string   `json:"areacode"`
}

// http://en.wikipedia.org/wiki/Reserved_IP_addresses
var reservedIPs = []net.IPNet{
	{net.IPv4(0, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(10, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(100, 64, 0, 0), net.IPv4Mask(255, 192, 0, 0)},
	{net.IPv4(127, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(169, 254, 0, 0), net.IPv4Mask(255, 255, 0, 0)},
	{net.IPv4(172, 16, 0, 0), net.IPv4Mask(255, 240, 0, 0)},
	{net.IPv4(192, 0, 0, 0), net.IPv4Mask(255, 255, 255, 248)},
	{net.IPv4(192, 0, 2, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(192, 88, 99, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(192, 168, 0, 0), net.IPv4Mask(255, 255, 0, 0)},
	{net.IPv4(198, 18, 0, 0), net.IPv4Mask(255, 254, 0, 0)},
	{net.IPv4(198, 51, 100, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(203, 0, 113, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(224, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)},
	{net.IPv4(240, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)},
	{net.IPv4(255, 255, 255, 255), net.IPv4Mask(255, 255, 255, 255)},
}
<|endoftext|>


<commit_before>package gostub

import (
	"errors"
	"os"
	"testing"
	"time"
)

func TestStubTime(t *testing.T) {
	var timeNow = time.Now

	var fakeTime = time.Date(2015, 7, 1, 0, 0, 0, 0, time.UTC)
	StubFunc(&timeNow, fakeTime)
	expectVal(t, fakeTime, timeNow())
}

func TestReturnErr(t *testing.T) {
	var osRemove = os.Remove

	StubFunc(&osRemove, nil)
	expectVal(t, nil, osRemove("test"))

	e := errors.New("err")
	StubFunc(&osRemove, e)
	expectVal(t, e, osRemove("test"))
}

func TestStubHostname(t *testing.T) {
	var osHostname = os.Hostname

	StubFunc(&osHostname, "fakehost", nil)
	hostname, err := osHostname()
	expectVal(t, "fakehost", hostname)
	expectVal(t, nil, err)

	var errNoHost = errors.New("no hostname")
	StubFunc(&osHostname, "", errNoHost)
	hostname, err = osHostname()
	expectVal(t, "", hostname)
	expectVal(t, errNoHost, err)
}

func TestStubReturnFunc(t *testing.T) {
	var retFunc = func() func() error {
		return func() error {
			return errors.New("err")
		}
	}

	var errInception = errors.New("in limbo")
	StubFunc(&retFunc, func() error {
		return errInception
	})
	expectVal(t, errInception, retFunc()())
}
<commit_msg>Add tests for invalid arguments to StubFunc<commit_after>package gostub

import (
	"errors"
	"os"
	"testing"
	"time"
)

func TestStubTime(t *testing.T) {
	var timeNow = time.Now

	var fakeTime = time.Date(2015, 7, 1, 0, 0, 0, 0, time.UTC)
	StubFunc(&timeNow, fakeTime)
	expectVal(t, fakeTime, timeNow())
}

func TestReturnErr(t *testing.T) {
	var osRemove = os.Remove

	StubFunc(&osRemove, nil)
	expectVal(t, nil, osRemove("test"))

	e := errors.New("err")
	StubFunc(&osRemove, e)
	expectVal(t, e, osRemove("test"))
}

func TestStubHostname(t *testing.T) {
	var osHostname = os.Hostname

	StubFunc(&osHostname, "fakehost", nil)
	hostname, err := osHostname()
	expectVal(t, "fakehost", hostname)
	expectVal(t, nil, err)

	var errNoHost = errors.New("no hostname")
	StubFunc(&osHostname, "", errNoHost)
	hostname, err = osHostname()
	expectVal(t, "", hostname)
	expectVal(t, errNoHost, err)
}

func TestStubReturnFunc(t *testing.T) {
	var retFunc = func() func() error {
		return func() error {
			return errors.New("err")
		}
	}

	var errInception = errors.New("in limbo")
	StubFunc(&retFunc, func() error {
		return errInception
	})
	expectVal(t, errInception, retFunc()())
}

func TestStubFuncFail(t *testing.T) {
	var osHostname = os.Hostname
	var s string

	tests := []struct {
		desc     string
		toStub   interface{}
		stubVals []interface{}
		wantErr  string
	}{
		{
			desc:     "toStub is not a function",
			toStub:   &s,
			stubVals: []interface{}{"fakehost", nil},
			wantErr:  "to stub must be a pointer to a function",
		},
		{
			desc:     "toStub is not a pointer",
			toStub:   osHostname,
			stubVals: []interface{}{"fakehost", nil},
			wantErr:  "to stub must be a pointer to a function",
		},
		{
			desc:     "wrong number of stubVals",
			toStub:   &osHostname,
			stubVals: []interface{}{"fakehost"},
			wantErr:  "func type has 2 return values, but only 1 stub values provided",
		},
	}

	for _, tt := range tests {
		func() {
			defer expectPanic(t, tt.desc, tt.wantErr)
			StubFunc(tt.toStub, tt.stubVals...)
		}()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"
	"text/template"
)

// Go Template Function Map here

var templateFunctions = template.FuncMap{
	// simple additon function useful for counters in loops
	"Add": func(a int, b int) int {
		return a + b
	},

	// strip function for removing characters from text
	"Strip": func(s string, rmv string) string {
		return strings.Replace(s, rmv, "", -1)
	},

	// file function for reading text from a given file under the files folder
	"File": func(filename string) (string, error) {

		p := job.tplFiles[0]
		f := filepath.Join(filepath.Dir(p), "..", "files", filename)
		fmt.Println(f)
		b, err := ioutil.ReadFile(f)
		if err != nil {
			log.Fatal("Error reading the template file: ", err)
			return "", err
		}
		return string(b), nil
	},

	// Get get does an HTTP Get request of the given url and returns the output string
	"GET": func(url string) (string, error) {
		resp, err := Get(url)
		if err != nil {
			return "", err
		}

		return resp, nil
	},

	// S3Read reads content of file from s3 and returns string contents
	"S3Read": func(url string) (string, error) {
		resp, err := S3Read(url)
		if err != nil {
			return "", err
		}
		return resp, nil
	},
}
<commit_msg>Add Increment function<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"
)

// Go Template Function Map here
var increment = 0

var templateFunctions = template.FuncMap{
	// simple additon function useful for counters
	"Add": func(a int, b int) int {
		return a + b
	},

	// strip function for removing characters from text
	"Strip": func(s string, rmv string) string {
		return strings.Replace(s, rmv, "", -1)
	},

	// Inc function returns an incremented value for each call.
	"Inc": func() string {
		increment = increment + 1
		return strconv.Itoa(increment)
	},

	// file function for reading text from a given file under the files folder
	"File": func(filename string) (string, error) {

		p := job.tplFiles[0]
		f := filepath.Join(filepath.Dir(p), "..", "files", filename)
		fmt.Println(f)
		b, err := ioutil.ReadFile(f)
		if err != nil {
			log.Fatal("Error reading the template file: ", err)
			return "", err
		}
		return string(b), nil
	},

	// Get get does an HTTP Get request of the given url and returns the output string
	"GET": func(url string) (string, error) {
		resp, err := Get(url)
		if err != nil {
			return "", err
		}

		return resp, nil
	},

	// S3Read reads content of file from s3 and returns string contents
	"S3Read": func(url string) (string, error) {
		resp, err := S3Read(url)
		if err != nil {
			return "", err
		}
		return resp, nil
	},
}
<|endoftext|>