<commit_before>// https://rsdn.ru/article/baseserv/pe_coff.xml
package main

import (
	"encoding/binary"
	"fmt"
	"os"
)

type debug_info struct {
	CodeId  string
	DebugId string
}

type MZHeader struct {
	Signature int16      // 0x00-0x02 0x5A4D
	Unused    [0x3A]byte // 0x02-0x3C
	PEOffset  int32      // 0x3C-0x40
}

type RvaAndSize struct {
	VirtualAddress int32
	VirtualSize    int32
}

type PEHeader struct {
	Signature                   int32      // 0x00-0x04 0x50450000
	Machine                     int16      // 0x04-0x06
	NumberOfSections            int16      // 0x06-0x08
	TimeDateStamp               int32      // 0x08-0x0C
	PointerToSymbolTable        int32      // 0x0C-0x10
	NumberOfSymbolTable         int32      // 0x10-0x14
	SizeOfOptionalHeader        int16      // 0x14-0x16
	Characteristics             int16      // 0x16-0x18
	StandadCOFFFields           [0x1C]byte // 0x18-0x34
	ImageBase                   int32      // 0x34-0x38
	SectionAlignment            int32      // 0x38-0x3C
	FileAlignment               int32      // 0x3C-0x40
	MajorOperatingSystemVersion int16      // 0x40-0x42
	MinorOperatingSystemVersion int16      // 0x40-0x42
	MajorImageVersion           int16      // 0x42-0x44
	MinorImageVersion           int16      // 0x44-0x46
	MajorSubsystemVersion       int16      // 0x46-0x48
	MinorSubsystemVersion       int16      // 0x48-0x4A
	Win32VersionValue           int32      // 0x4A-0x50
	SizeOfImage                 int32      // 0x50-0x54
	SizeOfHeaders               int32      // 0x54-0x58
	CheckSum                    int32      // 0x58-0x5C
	Subsystem                   int16      // 0x5C-0x5E
	DllCharacteristics          int16      // 0x5E-0x60
	SizeOfStackReserve          int32      // 0x60-0x64
	SizeOfStackCommit           int32      // 0x64-0x68
	SizeOfHeapReserve           int32      // 0x68-0x6C
	SizeOfHeapCommit            int32      // 0x6C-0x70
	LoaderFlags                 int32      // 0x70-0x74
	NumberOfRvaAndSizes         int32      // 0x74-0x78
}

type PESection struct {
	Name                 [0x08]byte
	VirtualSize          int32
	VirtualAddress       int32
	SizeOfRawData        int32
	PointerToRawData     int32
	PointerToRelocations int32
	PointerToLinenumbers int32
	NumberOfRelocations  int16
	NumberOfLinenumbers  int16
	Characteristics      int32
}
type PEDebugDirectory struct {
	Characteristics  int32
	TimeDateStamp    int32
	MajorVersion     int16
	MinorVersion     int16
	Type             int32
	SizeOfData       int32
	AddressOfRawData int32
	PointerToRawData int32
}

func read_debug_info(file *os.File) debug_info {
	var mz MZHeader
	var pe PEHeader
	binary.Read(file, binary.LittleEndian, &mz)

	fmt.Printf("MZ signature: %04X\n", mz.Signature)
	fmt.Printf("PE offset: %08X\n", mz.PEOffset)

	file.Seek(int64(mz.PEOffset), 0)
	binary.Read(file, binary.LittleEndian, &pe)

	fmt.Printf("PE signature: %08X\n", pe.Signature)
	fmt.Printf("PE timestamp: %08X\n", pe.TimeDateStamp)
	fmt.Printf("PE image size: %08X\n", pe.SizeOfImage)

	fmt.Printf("Sections count: %d\n", pe.NumberOfSections)
	fmt.Printf("Sections alignment: %d\n", pe.SectionAlignment)
	fmt.Printf("Size of headers: %d\n", pe.SizeOfHeaders)

	var debug_rva RvaAndSize
	if pe.NumberOfRvaAndSizes < 7 {
		// todo: opss...
		fmt.Println("OPS....")
	}
	for i := 0; i < 7; i++ {
		binary.Read(file, binary.LittleEndian, &debug_rva)
	}

	file.Seek(int64(mz.PEOffset)+int64(pe.SizeOfOptionalHeader)+0x18, 0)

	fmt.Printf("Section offset: %08X\n", int64(mz.PEOffset)+int64(pe.SizeOfOptionalHeader)+0x18)
	rdata := [8]byte{'.', 'r', 'd', 'a', 't', 'a'}
	debug_dir_offest := int64(0)
	for i := int16(0); i < pe.NumberOfSections; i++ {
		var section PESection
		binary.Read(file, binary.LittleEndian, &section)
		fmt.Printf("%d: %s\n", i, section.Name)
		if section.Name == rdata {
			debug_dir_offest = int64(section.PointerToRawData + debug_rva.VirtualAddress - section.VirtualAddress)
			break
		}
	}

	if debug_dir_offest > 0 {
		file.Seek(int64(debug_dir_offest), 0)
		var debug_dir PEDebugDirectory
		fmt.Printf("IMAGE_DEBUG_DIRECTORY offset: %X (%d)\n", debug_dir_offest, binary.Size(&debug_dir))
		for i := 0; i < int(debug_rva.VirtualSize)/binary.Size(&debug_dir); i++ {
			binary.Read(file, binary.LittleEndian, &debug_dir)
			fmt.Printf("   %d: %d\n", i, debug_dir.Type)
			if debug_dir.Type == 2 {
				fmt.Printf("RSDS offset: %X\n", debug_dir.PointerToRawData)
				break
			}
		}
	}

	return debug_info{
		fmt.Sprintf("%X%x", pe.TimeDateStamp, pe.SizeOfImage),
		"debug",
	}
}

func main() {
	file, _ := os.Open("sample/hello.exe")
	info := read_debug_info(file)
	fmt.Printf("Code ID: %s\n", info.CodeId)
}
<commit_msg>Load debug id from RSDS header<commit_after>// https://rsdn.ru/article/baseserv/pe_coff.xml
package main

import (
	"encoding/binary"
	"fmt"
	"os"
)

type debug_info struct {
	CodeId  string
	DebugId string
}

type MZHeader struct {
	Signature int16      // 0x00-0x02 0x5A4D
	Unused    [0x3A]byte // 0x02-0x3C
	PEOffset  int32      // 0x3C-0x40
}

type RvaAndSize struct {
	VirtualAddress int32
	VirtualSize    int32
}

type PEHeader struct {
	Signature                   int32      // 0x00-0x04 0x50450000
	Machine                     int16      // 0x04-0x06
	NumberOfSections            int16      // 0x06-0x08
	TimeDateStamp               int32      // 0x08-0x0C
	PointerToSymbolTable        int32      // 0x0C-0x10
	NumberOfSymbolTable         int32      // 0x10-0x14
	SizeOfOptionalHeader        int16      // 0x14-0x16
	Characteristics             int16      // 0x16-0x18
	StandadCOFFFields           [0x1C]byte // 0x18-0x34
	ImageBase                   int32      // 0x34-0x38
	SectionAlignment            int32      // 0x38-0x3C
	FileAlignment               int32      // 0x3C-0x40
	MajorOperatingSystemVersion int16      // 0x40-0x42
	MinorOperatingSystemVersion int16      // 0x40-0x42
	MajorImageVersion           int16      // 0x42-0x44
	MinorImageVersion           int16      // 0x44-0x46
	MajorSubsystemVersion       int16      // 0x46-0x48
	MinorSubsystemVersion       int16      // 0x48-0x4A
	Win32VersionValue           int32      // 0x4A-0x50
	SizeOfImage                 int32      // 0x50-0x54
	SizeOfHeaders               int32      // 0x54-0x58
	CheckSum                    int32      // 0x58-0x5C
	Subsystem                   int16      // 0x5C-0x5E
	DllCharacteristics          int16      // 0x5E-0x60
	SizeOfStackReserve          int32      // 0x60-0x64
	SizeOfStackCommit           int32      // 0x64-0x68
	SizeOfHeapReserve           int32      // 0x68-0x6C
	SizeOfHeapCommit            int32      // 0x6C-0x70
	LoaderFlags                 int32      // 0x70-0x74
	NumberOfRvaAndSizes         int32      // 0x74-0x78
}

type PESection struct {
	Name                 [0x08]byte
	VirtualSize          int32
	VirtualAddress       int32
	SizeOfRawData        int32
	PointerToRawData     int32
	PointerToRelocations int32
	PointerToLinenumbers int32
	NumberOfRelocations  int16
	NumberOfLinenumbers  int16
	Characteristics      int32
}

type PEDebugDirectory struct {
	Characteristics  int32
	TimeDateStamp    int32
	MajorVersion     int16
	MinorVersion     int16
	Type             int32
	SizeOfData       int32
	AddressOfRawData int32
	PointerToRawData int32
}

type RSDSHeader struct {
	Signature     int32      // 0x00-0x04 0x53445352
	GUID          [0x10]byte // 0x04-0x14
	TimeDateStamp int32      // 0x14-0x18
}

func read_debug_info(file *os.File) debug_info {
	var mz MZHeader
	var pe PEHeader
	binary.Read(file, binary.LittleEndian, &mz)

	fmt.Printf("MZ signature: %04X\n", mz.Signature)
	fmt.Printf("PE offset: %08X\n", mz.PEOffset)

	file.Seek(int64(mz.PEOffset), 0)
	binary.Read(file, binary.LittleEndian, &pe)

	fmt.Printf("PE signature: %08X\n", pe.Signature)
	fmt.Printf("PE timestamp: %08X\n", pe.TimeDateStamp)
	fmt.Printf("PE image size: %08X\n", pe.SizeOfImage)

	fmt.Printf("Sections count: %d\n", pe.NumberOfSections)
	fmt.Printf("Sections alignment: %d\n", pe.SectionAlignment)
	fmt.Printf("Size of headers: %d\n", pe.SizeOfHeaders)

	var debug_rva RvaAndSize
	if pe.NumberOfRvaAndSizes < 7 {
		// todo: opss...
		fmt.Println("OPS....")
	}
	for i := 0; i < 7; i++ {
		binary.Read(file, binary.LittleEndian, &debug_rva)
	}

	file.Seek(int64(mz.PEOffset)+int64(pe.SizeOfOptionalHeader)+0x18, 0)

	fmt.Printf("Section offset: %08X\n", int64(mz.PEOffset)+int64(pe.SizeOfOptionalHeader)+0x18)
	rdata := [8]byte{'.', 'r', 'd', 'a', 't', 'a'}
	debug_dir_offest := int64(0)
	for i := int16(0); i < pe.NumberOfSections; i++ {
		var section PESection
		binary.Read(file, binary.LittleEndian, &section)
		fmt.Printf("%d: %s\n", i, section.Name)
		if section.Name == rdata {
			debug_dir_offest = int64(section.PointerToRawData + debug_rva.VirtualAddress - section.VirtualAddress)
			break
		}
	}

	rsds_offset := int64(0)
	if debug_dir_offest > 0 {
		file.Seek(debug_dir_offest, 0)
		var debug_dir PEDebugDirectory
		fmt.Printf("IMAGE_DEBUG_DIRECTORY offset: %X (%d)\n", debug_dir_offest, binary.Size(&debug_dir))
		for i := 0; i < int(debug_rva.VirtualSize)/binary.Size(&debug_dir); i++ {
			binary.Read(file, binary.LittleEndian, &debug_dir)
			fmt.Printf("   %d: %d\n", i, debug_dir.Type)
			if debug_dir.Type == 2 {
				fmt.Printf("RSDS offset: %X\n", debug_dir.PointerToRawData)
				rsds_offset = int64(debug_dir.PointerToRawData)
				break
			}
		}
	}

	var rsds RSDSHeader
	if rsds_offset > 0 {
		file.Seek(rsds_offset, 0)
		binary.Read(file, binary.LittleEndian, &rsds)
	}

	fmt.Printf("RSDS signature: %08X\n", rsds.Signature)
	fmt.Printf("RSDS timestamp: %08X\n", rsds.TimeDateStamp)

	return debug_info{
		fmt.Sprintf("%X%x", pe.TimeDateStamp, pe.SizeOfImage),
		fmt.Sprintf("%02X%02X%02X%02X%02X%02X%02X%02X%16X%d",
			rsds.GUID[3], rsds.GUID[2], rsds.GUID[1], rsds.GUID[0],
			rsds.GUID[5], rsds.GUID[4],
			rsds.GUID[7], rsds.GUID[6],
			rsds.GUID[8:],
			rsds.TimeDateStamp),
	}
}

func main() {
	file, _ := os.Open("sample/hello.exe")
	info := read_debug_info(file)
	fmt.Printf("Code ID: %s\n", info.CodeId)
	fmt.Printf("Debug ID: %s\n", info.DebugId)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"

	"gopkg.in/redis.v3"
)

var (
	version       string
	redisDB       int64
	redisAddr     string
	documentation = `Redis Dumper

This script dumps all the entries from one Redis DB into a file in the redis protocol format.
See here (http://redis.io/topics/protocol) and here (http://redis.io/topics/mass-insert).
This allows use to pipe the resulting file directly into redis with pipe command like this

> cat redis_db_0_dump.rdb | redis-cli --pipe

This script is especially created to get contents from AWS Elasticache but works with all Redis instances

`
)

func init() {
	flag.Int64Var(&redisDB, "db", 0, "Indicate which db to process")
	flag.StringVar(&redisAddr, "address", "localhost:6379", "Redis address (url and port)")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, documentation)
		fmt.Fprintf(os.Stderr, "Usage of Redis Dumper:\n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\nCurrent Version: %s\n", version)
	}
}

func main() {
	flag.Parse()
	log.Println("Start processing")

	client := redis.NewClient(&redis.Options{
		DB:   redisDB,
		Addr: redisAddr,
	})

	file, writer := createFile()
	defer file.Close()

	var cursor int64
	for {
		var keys []string
		var err error
		cursor, keys, err = client.Scan(cursor, "", 1000).Result()
		if err != nil {
			log.Fatalf("Couldn't iterate through set: %v", err)
		}

		for _, key := range keys {
			dump, err := client.Dump(key).Result()
			if err != nil {
				log.Printf("ERROR: couldn't dump key %s: %v", key, err)
				return
			}
			writer.WriteString(createRestoreCommand(key, dump))
		}
		writer.Flush()

		if cursor == 0 {
			break
		}
	}

	log.Println("End processing")
}

func createRestoreCommand(key, dump string) string {
	proto := "*4\r\n$7\r\nRESTORE\r\n"
	key_proto := "$" + strconv.Itoa(len(key)) + "\r\n" + key + "\r\n"
	ttl_proto := "$1\r\n0\r\n"
	dump_proto := "$" + strconv.Itoa(len(dump)) + "\r\n" + dump + "\r\n"

	return proto + key_proto + ttl_proto + dump_proto
}

func createFile() (*os.File, *bufio.Writer) {
	file, err := os.Create(fmt.Sprintf("redis_db_%d_dump.rdb", redisDB))
	if err != nil {
		log.Fatalf("Couldn't create file: %v", err)
	}

	return file, bufio.NewWriter(file)
}
<commit_msg>Dump original key TTL <commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"gopkg.in/redis.v3"
)

var (
	version       string
	redisDB       int64
	redisAddr     string
	documentation = `Redis Dumper

This script dumps all the entries from one Redis DB into a file in the redis protocol format.
See here (http://redis.io/topics/protocol) and here (http://redis.io/topics/mass-insert).
This allows use to pipe the resulting file directly into redis with pipe command like this

> cat redis_db_0_dump.rdb | redis-cli --pipe

This script is especially created to get contents from AWS Elasticache but works with all Redis instances

`
)

const restoreCommand = "*4\r\n$7\r\nRESTORE\r\n"

func init() {
	flag.Int64Var(&redisDB, "db", 0, "Indicate which db to process")
	flag.StringVar(&redisAddr, "address", "localhost:6379", "Redis address (url and port)")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, documentation)
		fmt.Fprintf(os.Stderr, "Usage of Redis Dumper:\n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\nCurrent Version: %s\n", version)
	}
}

func main() {
	flag.Parse()
	log.Println("Start processing")

	client := redis.NewClient(&redis.Options{
		DB:   redisDB,
		Addr: redisAddr,
	})

	file, writer := createFile()
	defer file.Close()

	var cursor int64
	for {
		var keys []string
		var err error
		cursor, keys, err = client.Scan(cursor, "", 1000).Result()
		if err != nil {
			log.Fatalf("Couldn't iterate through set: %v", err)
		}

		for _, key := range keys {
			processKey(client, writer, key)
		}
		writer.Flush()

		if cursor == 0 {
			break
		}
	}

	log.Println("End processing")
}

func processKey(client *redis.Client, writer *bufio.Writer, key string) {
	dump, err := client.Dump(key).Result()
	if err != nil {
		log.Printf("ERROR: couldn't dump key %s: %v", key, err)
		return
	}

	ttl, err := client.TTL(key).Result()
	if err != nil {
		log.Printf("ERROR: couldn't dump key %s: %v", key, err)
		return
	}

	writer.WriteString(createRestoreCommand(key, dump, &ttl))
}

func createRestoreCommand(key, dump string, ttl *time.Duration) string {
	// TODO: need to check if seconds contains numbers after ,!!!
	ttlString := strconv.Itoa(int(ttl.Seconds() * 1000))

	result := restoreCommand

	for _, val := range [3]string{key, ttlString, dump} {
		result += "$" + strconv.Itoa(len(val)) + "\r\n" + val + "\r\n"
	}

	return result
}

func createFile() (*os.File, *bufio.Writer) {
	file, err := os.Create(fmt.Sprintf("redis_db_%d_dump.rdb", redisDB))
	if err != nil {
		log.Fatalf("Couldn't create file: %v", err)
	}

	return file, bufio.NewWriter(file)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"github.com/gophergala/go_report/check"
	"gopkg.in/mgo.v2"
	"labix.org/v2/mgo/bson"
)

var (
	mongoURL        = "mongodb://localhost:27017"
	mongoDatabase   = "goreportcard"
	mongoCollection = "reports"
)

func getMongoCollection() (*mgo.Collection, error) {
	session, err := mgo.Dial(mongoURL)
	if err != nil {
		return nil, err
	}
	c := session.DB(mongoDatabase).C(mongoCollection)
	return c, nil
}

func homeHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Serving home page")
	if r.URL.Path[1:] == "" {
		http.ServeFile(w, r, "templates/home.html")
	} else {
		http.NotFound(w, r)
	}
}

func assetsHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Serving " + r.URL.Path[1:])
	http.ServeFile(w, r, r.URL.Path[1:])
}

func orgRepoNames(url string) (string, string) {
	dir := strings.TrimSuffix(url, ".git")
	split := strings.Split(dir, "/")
	org := split[len(split)-2]
	repoName := split[len(split)-1]

	return org, repoName
}

func dirName(url string) string {
	org, repoName := orgRepoNames(url)

	return fmt.Sprintf("repos/src/github.com/%s/%s", org, repoName)
}

func clone(url string) error {
	org, _ := orgRepoNames(url)
	if err := os.Mkdir(fmt.Sprintf("repos/src/github.com/%s", org), 0755); err != nil && !os.IsExist(err) {
		return fmt.Errorf("could not create dir: %v", err)
	}
	dir := dirName(url)
	_, err := os.Stat(dir)
	if os.IsNotExist(err) {
		cmd := exec.Command("git", "clone", "--depth", "1", "--single-branch", url, dir)
		if err := cmd.Run(); err != nil {
			return fmt.Errorf("could not run git clone: %v", err)
		}
	} else if err != nil {
		return fmt.Errorf("could not stat dir: %v", err)
	} else {
		cmd := exec.Command("git", "-C", dir, "pull")
		if err := cmd.Run(); err != nil {
			return fmt.Errorf("could not pull repo: %v", err)
		}
	}

	return nil
}

type score struct {
	Name          string              `json:"name"`
	Description   string              `json:"description"`
	FileSummaries []check.FileSummary `json:"file_summaries"`
	Percentage    float64             `json:"percentage"`
}

type checksResp struct {
	Checks      []score   `json:"checks"`
	Average     float64   `json:"average"`
	Files       int       `json:"files"`
	Issues      int       `json:"issues"`
	Repo        string    `json:"repo"`
	LastRefresh time.Time `json:"last_refresh"`
}

func checkHandler(w http.ResponseWriter, r *http.Request) {
	repo := r.FormValue("repo")
	url := repo
	if !strings.HasPrefix(url, "https://github.com/") {
		url = "https://github.com/" + url
	}

	w.Header().Set("Content-Type", "application/json")

	// if this is a GET request, fetch from cached version in mongo
	if r.Method == "GET" {
		// try and fetch from mongo
		coll, err := getMongoCollection()
		if err != nil {
			log.Println("Failed to get mongo collection during GET: ", err)
		} else {
			resp := checksResp{}
			err := coll.Find(bson.M{"repo": repo}).One(&resp)
			if err != nil {
				log.Println("Failed to fetch from mongo: ", err)
			} else {
				resp.LastRefresh = resp.LastRefresh.UTC()
				b, err := json.Marshal(resp)
				if err != nil {
					log.Println("ERROR: could not marshal json:", err)
					http.Error(w, err.Error(), 500)
					return
				}
				w.Write(b)
				log.Println("Loaded from cache!", repo)
				return
			}
		}
	}

	err := clone(url)
	if err != nil {
		log.Println("ERROR: could not clone repo: ", err)
		http.Error(w, fmt.Sprintf("Could not clone repo: %v", err), 500)
		return
	}

	dir := dirName(url)
	filenames, err := check.GoFiles(dir)
	if err != nil {
		log.Println("ERROR: could not get filenames: ", err)
		http.Error(w, fmt.Sprintf("Could not get filenames: %v", err), 500)
		return
	}
	checks := []check.Check{check.GoFmt{Dir: dir, Filenames: filenames},
		check.GoVet{Dir: dir, Filenames: filenames},
		check.GoLint{Dir: dir, Filenames: filenames},
		check.GoCyclo{Dir: dir, Filenames: filenames},
	}

	ch := make(chan score)
	for _, c := range checks {
		go func(c check.Check) {
			p, summaries, err := c.Percentage()
			if err != nil {
				log.Printf("ERROR: (%s) %v", c.Name(), err)
			}
			s := score{
				Name:          c.Name(),
				Description:   c.Description(),
				FileSummaries: summaries,
				Percentage:    p,
			}
			ch <- s
		}(c)
	}

	resp := checksResp{Repo: repo,
		Files:       len(filenames),
		LastRefresh: time.Now().UTC()}
	var avg float64
	var issues = make(map[string]bool)
	for i := 0; i < len(checks); i++ {
		s := <-ch
		resp.Checks = append(resp.Checks, s)
		avg += s.Percentage
		for _, fs := range s.FileSummaries {
			issues[fs.Filename] = true
		}
	}

	resp.Average = avg / float64(len(checks))
	resp.Issues = len(issues)

	b, err := json.Marshal(resp)
	if err != nil {
		log.Println("ERROR: could not marshal json:", err)
		http.Error(w, err.Error(), 500)
		return
	}
	w.Write(b)

	// write to mongo
	coll, err := getMongoCollection()
	if err != nil {
		log.Println("Failed to get mongo collection: ", err)
	} else {
		log.Println("Writing to mongo...")
		_, err := coll.Upsert(bson.M{"Repo": repo}, resp)
		if err != nil {
			log.Println("Mongo writing error:", err)
		}
	}
}

func reportHandler(w http.ResponseWriter, r *http.Request, org, repo string) {
	http.ServeFile(w, r, "templates/home.html")
}

func makeReportHandler(fn func(http.ResponseWriter, *http.Request, string, string)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		validPath := regexp.MustCompile(`^/report/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)$`)

		m := validPath.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r, m[1], m[2])
	}
}

func main() {
	if err := os.MkdirAll("repos/src/github.com", 0755); err != nil && !os.IsExist(err) {
		log.Fatal("ERROR: could not create repos dir: ", err)
	}

	http.HandleFunc("/assets/", assetsHandler)
	http.HandleFunc("/checks", checkHandler)
	http.HandleFunc("/report/", makeReportHandler(reportHandler))
	http.HandleFunc("/", homeHandler)

	fmt.Println("Running on 127.0.01:8080...")
	log.Fatal(http.ListenAndServe("127.0.0.1:8080", nil))
}
<commit_msg>log repo name when not found in mongo<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"github.com/gophergala/go_report/check"
	"gopkg.in/mgo.v2"
	"labix.org/v2/mgo/bson"
)

var (
	mongoURL        = "mongodb://localhost:27017"
	mongoDatabase   = "goreportcard"
	mongoCollection = "reports"
)

func getMongoCollection() (*mgo.Collection, error) {
	session, err := mgo.Dial(mongoURL)
	if err != nil {
		return nil, err
	}
	c := session.DB(mongoDatabase).C(mongoCollection)
	return c, nil
}

func homeHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Serving home page")
	if r.URL.Path[1:] == "" {
		http.ServeFile(w, r, "templates/home.html")
	} else {
		http.NotFound(w, r)
	}
}

func assetsHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Serving " + r.URL.Path[1:])
	http.ServeFile(w, r, r.URL.Path[1:])
}

func orgRepoNames(url string) (string, string) {
	dir := strings.TrimSuffix(url, ".git")
	split := strings.Split(dir, "/")
	org := split[len(split)-2]
	repoName := split[len(split)-1]

	return org, repoName
}

func dirName(url string) string {
	org, repoName := orgRepoNames(url)

	return fmt.Sprintf("repos/src/github.com/%s/%s", org, repoName)
}

func clone(url string) error {
	org, _ := orgRepoNames(url)
	if err := os.Mkdir(fmt.Sprintf("repos/src/github.com/%s", org), 0755); err != nil && !os.IsExist(err) {
		return fmt.Errorf("could not create dir: %v", err)
	}
	dir := dirName(url)
	_, err := os.Stat(dir)
	if os.IsNotExist(err) {
		cmd := exec.Command("git", "clone", "--depth", "1", "--single-branch", url, dir)
		if err := cmd.Run(); err != nil {
			return fmt.Errorf("could not run git clone: %v", err)
		}
	} else if err != nil {
		return fmt.Errorf("could not stat dir: %v", err)
	} else {
		cmd := exec.Command("git", "-C", dir, "pull")
		if err := cmd.Run(); err != nil {
			return fmt.Errorf("could not pull repo: %v", err)
		}
	}

	return nil
}

type score struct {
	Name          string              `json:"name"`
	Description   string              `json:"description"`
	FileSummaries []check.FileSummary `json:"file_summaries"`
	Percentage    float64             `json:"percentage"`
}

type checksResp struct {
	Checks      []score   `json:"checks"`
	Average     float64   `json:"average"`
	Files       int       `json:"files"`
	Issues      int       `json:"issues"`
	Repo        string    `json:"repo"`
	LastRefresh time.Time `json:"last_refresh"`
}

func checkHandler(w http.ResponseWriter, r *http.Request) {
	repo := r.FormValue("repo")
	url := repo
	if !strings.HasPrefix(url, "https://github.com/") {
		url = "https://github.com/" + url
	}

	w.Header().Set("Content-Type", "application/json")

	// if this is a GET request, fetch from cached version in mongo
	if r.Method == "GET" {
		// try and fetch from mongo
		coll, err := getMongoCollection()
		if err != nil {
			log.Println("Failed to get mongo collection during GET: ", err)
		} else {
			resp := checksResp{}
			err := coll.Find(bson.M{"repo": repo}).One(&resp)
			if err != nil {
				log.Printf("Failed to fetch %q from mongo: %v", repo, err)
			} else {
				resp.LastRefresh = resp.LastRefresh.UTC()
				b, err := json.Marshal(resp)
				if err != nil {
					log.Println("ERROR: could not marshal json:", err)
					http.Error(w, err.Error(), 500)
					return
				}
				w.Write(b)
				log.Println("Loaded from cache!", repo)
				return
			}
		}
	}

	err := clone(url)
	if err != nil {
		log.Println("ERROR: could not clone repo: ", err)
		http.Error(w, fmt.Sprintf("Could not clone repo: %v", err), 500)
		return
	}

	dir := dirName(url)
	filenames, err := check.GoFiles(dir)
	if err != nil {
		log.Println("ERROR: could not get filenames: ", err)
		http.Error(w, fmt.Sprintf("Could not get filenames: %v", err), 500)
		return
	}
	checks := []check.Check{check.GoFmt{Dir: dir, Filenames: filenames},
		check.GoVet{Dir: dir, Filenames: filenames},
		check.GoLint{Dir: dir, Filenames: filenames},
		check.GoCyclo{Dir: dir, Filenames: filenames},
	}

	ch := make(chan score)
	for _, c := range checks {
		go func(c check.Check) {
			p, summaries, err := c.Percentage()
			if err != nil {
				log.Printf("ERROR: (%s) %v", c.Name(), err)
			}
			s := score{
				Name:          c.Name(),
				Description:   c.Description(),
				FileSummaries: summaries,
				Percentage:    p,
			}
			ch <- s
		}(c)
	}

	resp := checksResp{Repo: repo,
		Files:       len(filenames),
		LastRefresh: time.Now().UTC()}
	var avg float64
	var issues = make(map[string]bool)
	for i := 0; i < len(checks); i++ {
		s := <-ch
		resp.Checks = append(resp.Checks, s)
		avg += s.Percentage
		for _, fs := range s.FileSummaries {
			issues[fs.Filename] = true
		}
	}

	resp.Average = avg / float64(len(checks))
	resp.Issues = len(issues)

	b, err := json.Marshal(resp)
	if err != nil {
		log.Println("ERROR: could not marshal json:", err)
		http.Error(w, err.Error(), 500)
		return
	}
	w.Write(b)

	// write to mongo
	coll, err := getMongoCollection()
	if err != nil {
		log.Println("Failed to get mongo collection: ", err)
	} else {
		log.Println("Writing to mongo...")
		_, err := coll.Upsert(bson.M{"Repo": repo}, resp)
		if err != nil {
			log.Println("Mongo writing error:", err)
		}
	}
}

func reportHandler(w http.ResponseWriter, r *http.Request, org, repo string) {
	http.ServeFile(w, r, "templates/home.html")
}

func makeReportHandler(fn func(http.ResponseWriter, *http.Request, string, string)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		validPath := regexp.MustCompile(`^/report/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)$`)

		m := validPath.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r, m[1], m[2])
	}
}

func main() {
	if err := os.MkdirAll("repos/src/github.com", 0755); err != nil && !os.IsExist(err) {
		log.Fatal("ERROR: could not create repos dir: ", err)
	}

	http.HandleFunc("/assets/", assetsHandler)
	http.HandleFunc("/checks", checkHandler)
	http.HandleFunc("/report/", makeReportHandler(reportHandler))
	http.HandleFunc("/", homeHandler)

	fmt.Println("Running on 127.0.01:8080...")
	log.Fatal(http.ListenAndServe("127.0.0.1:8080", nil))
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/starmanmartin/simple-fs"
)

const (
	install = "install"
	test    = "test"
)

var runTypes = []string{install, test}

var (
	lastPart                                        *regexp.Regexp
	isTest, isBenchTest, isExecute, isWatch         bool
	newRoot, packageName, currentPath, outputString string
	restArgs                                        []string
)

func init() {
	lastPart, _ = regexp.Compile(`[^\\/]*$`)

	flag.BoolVar(&isTest, "t", false, "Run as Test")
	flag.BoolVar(&isBenchTest, "b", false, "Bench tests (only if test)")
	flag.BoolVar(&isExecute, "e", false, "Execute (only if not test)")
	flag.BoolVar(&isWatch, "w", false, "Execute (only if not test)")
	flag.StringVar(&outputString, "p", "", "Make Package")
}

func getCmd(cmdCommand []string) *exec.Cmd {
	parts := cmdCommand
	head := parts[0]
	parts = parts[1:len(parts)]

	cmd := exec.Command(head, parts...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd
}

func exeCmd(cmdCommand []string) (*exec.Cmd, error) {
	cmd := getCmd(cmdCommand)

	if err := cmd.Start(); err != nil {
		return cmd, err
	}

	return cmd, nil
}

func buildCommand(packageName string) []string {
	buffer := make([]string, 0, 6)

	buffer = append(buffer, "go")

	if isTest {
		buffer = append(buffer, "test")
		if isBenchTest {
			buffer = append(buffer, "-bench=.")
		}
	} else {
		buffer = append(buffer, "install")
	}

	buffer = append(buffer, "-v")

	return buffer
}

func handelPathArgs() (string, string, []string, error) {
	args := flag.Args()
	if len(args) == 0 {
		return "", "", nil, errors.New("No Args")
	}

	if len(args) == 1 || args[0][:11] == "github.com/" {
		dir, err := os.Getwd()
		if err != nil {
			return "", "", nil, (err)
		}

		return dir, args[0], args[1:], nil
	}

	absPath, err := filepath.Abs(args[0])
	if err != nil {
		return "", "", nil, (err)
	}

	return absPath, args[1], args[1:], nil
}

func copyPackage(dir, packageName, funcName string) (isPackage bool, err error) {
	if len(outputString) == 0 {
		return
	}

	isPackage = true
	dest := dir + "/bin/" + funcName + "/"
	src := dir + "/src/" + packageName + "/"

	output := strings.Split(outputString, " ")

	for _, dirName := range output {
		err = fs.CopyFolder(src+dirName, dest+dirName)
		if err != nil {
			return
		}
	}

	return
}

func main() {
	flag.Parse()
	var err error
	newRoot, packageName, restArgs, err = handelPathArgs()
	if err != nil {
		log.Fatal(err)
		return
	}
	currentPath := os.Getenv("GOPATH")
	defer func() {
		log.Println("Done!!")
		os.Setenv("GOPATH", currentPath)
	}()
	newPath := []string{newRoot, ";", currentPath}

	os.Setenv("GOPATH", strings.Join(newPath, ""))
	runBuild()
}

func runBuild() {
	buildCommandList := buildCommand(packageName)
	buildCommandList = append(buildCommandList, packageName)
	_, err := exeCmd(buildCommandList)
	if err != nil {
		log.Fatal(err)
	}

	funcName := lastPart.FindString(packageName)
	isPackage, err := copyPackage(newRoot, packageName, funcName)

	if err != nil {
		log.Fatal(err)
		return
	} else if isPackage {
		fs.SyncFile(newRoot+"/bin/"+funcName+".exe", newRoot+"/bin/"+funcName+"/"+funcName+".exe")
		funcName = funcName + "/" + funcName
	}

	if isExecute && !isTest {
		log.Printf("Running %s\n", funcName)
		executionPath := newRoot + "/bin/" + funcName + ".exe"
		exArgs := []string{executionPath}
		exArgs = append(exArgs, restArgs...)
		if isWatch {
			watch(exArgs, newRoot+"/src/"+packageName)
		} else {
			_, err := exeCmd(exArgs)
			if err != nil {
				log.Fatal(err)
				return
			}
		}
	} else {
		log.Printf("Builded %s\n", funcName)
	}
}

func watch(args []string, rootPath string) {
	done := make(chan error, 1)
	doneWithoutErr := make(chan bool, 1)

	cmd := getCmd(args)

	go func() {
		err := cmd.Run()
		if err != nil {
			done <- err
		} else {
			doneWithoutErr <- true
		}
	}()

	restart := make(chan bool, 1)

	go func() {
		ticker := time.NewTicker(5 * time.Second)
		lastChaek := time.Now()
		for _ = range ticker.C {
			isUpdated, _ := fs.CheckIfFolderUpdated(rootPath, lastChaek)
			if isUpdated {
				restart <- true
				ticker.Stop()
			}
		}
	}()

	select {
	case <-restart:
		select {
		case <-doneWithoutErr:
			log.Println("process restarted")
			runBuild()
		default:
			if err := cmd.Process.Kill(); err != nil {
				log.Fatal("failed to kill: ", err)
			}

			log.Println("process restarted")
			runBuild()

		}

	case err := <-done:
		if err != nil {
			log.Fatal("process done with error = %v", err)
		} else {
			log.Print("process done gracefully without error")
		}
	}

}
<commit_msg>bug fix - waiting on execution<commit_after>package main

import (
	"errors"
	"flag"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/starmanmartin/simple-fs"
)

const (
	install = "install"
	test    = "test"
)

var runTypes = []string{install, test}

var (
	lastPart                                        *regexp.Regexp
	isTest, isBenchTest, isExecute, isWatch         bool
	newRoot, packageName, currentPath, outputString string
	restArgs                                        []string
)

func init() {
	lastPart, _ = regexp.Compile(`[^\\/]*$`)

	flag.BoolVar(&isTest, "t", false, "Run as Test")
	flag.BoolVar(&isBenchTest, "b", false, "Bench tests (only if test)")
	flag.BoolVar(&isExecute, "e", false, "Execute (only if not test)")
	flag.BoolVar(&isWatch, "w", false, "Execute (only if not test)")
	flag.StringVar(&outputString, "p", "", "Make Package")
}

func getCmd(cmdCommand []string) *exec.Cmd {
	parts := cmdCommand
	head := parts[0]
	parts = parts[1:len(parts)]

	cmd := exec.Command(head, parts...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd
}

func exeCmd(cmdCommand []string) (*exec.Cmd, error) {
	cmd := getCmd(cmdCommand)

	if err := cmd.Run(); err != nil {
		return cmd, err
	}

	return cmd, nil
}

func buildCommand(packageName string) []string {
	buffer := make([]string, 0, 6)

	buffer = append(buffer, "go")

	if isTest {
		buffer = append(buffer, "test")
		if isBenchTest {
			buffer = append(buffer, "-bench=.")
		}
	} else {
		buffer = append(buffer, "install")
	}

	buffer = append(buffer, "-v")

	return buffer
}

func handelPathArgs() (string, string, []string, error) {
	args := flag.Args()
	if len(args) == 0 {
		return "", "", nil, errors.New("No Args")
	}

	if len(args) == 1 || args[0][:11] == "github.com/" {
		dir, err := os.Getwd()
		if err != nil {
			return "", "", nil, (err)
		}

		return dir, args[0], args[1:], nil
	}

	absPath, err := filepath.Abs(args[0])
	if err != nil {
		return "", "", nil, (err)
	}

	return absPath, args[1], args[1:], nil
}

func copyPackage(dir, packageName, funcName string) (isPackage bool, err error) {
	if len(outputString) == 0 {
		return
	}

	isPackage = true
	dest := dir + "/bin/" + funcName + "/"
	src := dir + "/src/" + packageName + "/"

	output := strings.Split(outputString, " ")

	for _, dirName := range output {
		err = fs.CopyFolder(src+dirName, dest+dirName)
		if err != nil {
			return
		}
	}

	return
}

func main() {
	flag.Parse()
	var err error
	newRoot, packageName, restArgs, err = handelPathArgs()
	if err != nil {
		log.Fatal(err)
		return
	}
	currentPath := os.Getenv("GOPATH")
	defer func() {
		log.Println("Done!!")
		os.Setenv("GOPATH", currentPath)
	}()
	newPath := []string{newRoot, ";", currentPath}

	os.Setenv("GOPATH", strings.Join(newPath, ""))
	runBuild()
}

func runBuild() {
	buildCommandList := buildCommand(packageName)
	buildCommandList = append(buildCommandList, packageName)
    _, err := exeCmd(buildCommandList)
	if err != nil {
		log.Fatal(err)
	}

	funcName := lastPart.FindString(packageName)
	isPackage, err := copyPackage(newRoot, packageName, funcName)

	if err != nil {
		log.Fatal(err)
		return
	} else if isPackage {
		fs.SyncFile(newRoot+"/bin/"+funcName+".exe", newRoot+"/bin/"+funcName+"/"+funcName+".exe")
		funcName = funcName + "/" + funcName
	}

	if isExecute && !isTest {
		log.Printf("Running %s\n", funcName)
		executionPath := newRoot + "/bin/" + funcName + ".exe"
		exArgs := []string{executionPath}
		exArgs = append(exArgs, restArgs...)
		if isWatch {
			watch(exArgs, newRoot+"/src/"+packageName)
		} else {
			_, err := exeCmd(exArgs)
			if err != nil {
				log.Fatal(err)
				return
			}
		}
	} else {
		log.Printf("Builded %s\n", funcName)
	}
}

func watch(args []string, rootPath string) {
	done := make(chan error, 1)
	doneWithoutErr := make(chan bool, 1)

	cmd := getCmd(args)

	go func() {
		err := cmd.Run()
		if err != nil {
			done <- err
		} else {
			doneWithoutErr <- true
		}
	}()

	restart := make(chan bool, 1)

	go func() {
		ticker := time.NewTicker(5 * time.Second)
		lastChaek := time.Now()
		for _ = range ticker.C {
			isUpdated, _ := fs.CheckIfFolderUpdated(rootPath, lastChaek)
			if isUpdated {
				restart <- true
				ticker.Stop()
			}
		}
	}()

	select {
	case <-restart:
		select {
		case <-doneWithoutErr:
			log.Println("process restarted")
			runBuild()
		default:
			if err := cmd.Process.Kill(); err != nil {
				log.Fatal("failed to kill: ", err)
			}

			log.Println("process restarted")
			runBuild()

		}

	case err := <-done:
		if err != nil {
			log.Fatal("process done with error = %v", err)
		} else {
			log.Print("process done gracefully without error")
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"go/importer"
	"go/types"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

func main() {
	outputFile := flag.String("o", "-", "File to output to. Blank or - for stdin")
	templateFile := flag.String("t", "", "File to use as template for sprintf. if blank, just list the types")
	fmtStr := flag.String("f", "%s", "Format string to use on each type before sending to the template")

	flag.Parse()
	var err error

	tmpl := "%s"
	if *templateFile != "" {
		var bytes []byte
		bytes, err = ioutil.ReadFile(*templateFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		tmpl = string(bytes)
	}

	var wr io.WriteCloser = os.Stdout
	if *outputFile != "" && *outputFile != "-" {
		wr, err = os.Create(*outputFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}
	importer := importer.Default()

	// aaaallllrighty that's all the flag stuff outta the way
	// now we read all the packages and fmt.Fprintf(wr, tmpl, types)
	var types []string
	hasFailed := false

	for _, p := range flag.Args() {
		var pkg *types.Package
		pkg, err = importer.Import(p)
		if err != nil {
			fmt.Println(err)
			hasFailed = true
			continue
		}
		pkgName := pkg.Name()
		pkgPath := pkg.Path()
		scope := pkg.Scope()
		names := scope.Names()
		for _, name := range names {
			obj := scope.Lookup(name)
			inScopeRef := fmt.Sprintf("%s.%s", pkgName, name)
			fullNameWithPath := fmt.Sprintf("%s.%s", pkgPath, name)
			if obj.Exported() && obj.Type().String() == fullNameWithPath {
				types = append(types, fmt.Sprintf(*fmtStr, inScopeRef))
			}
		}

	}

	fmt.Fprintf(wr, tmpl, strings.Join(types, "\n"))
	err = wr.Close()
	if err != nil {
		fmt.Println(err)
		hasFailed = true
	}
	if hasFailed {
		os.Exit(1)
	}

}
<commit_msg>fix gen/list_types.go<commit_after>package main

import (
	"flag"
	"fmt"
	"go/importer"
	gotypes "go/types"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

func main() {
	outputFile := flag.String("o", "-", "File to output to. Blank or - for stdin")
	templateFile := flag.String("t", "", "File to use as template for sprintf. if blank, just list the types")
	fmtStr := flag.String("f", "%s", "Format string to use on each type before sending to the template")

	flag.Parse()
	var err error

	tmpl := "%s"
	if *templateFile != "" {
		var bytes []byte
		bytes, err = ioutil.ReadFile(*templateFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		tmpl = string(bytes)
	}

	var wr io.WriteCloser = os.Stdout
	if *outputFile != "" && *outputFile != "-" {
		wr, err = os.Create(*outputFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}
	importer := importer.Default()

	// aaaallllrighty that's all the flag stuff outta the way
	// now we read all the packages and fmt.Fprintf(wr, tmpl, types)
	var types []string
	hasFailed := false

	for _, p := range flag.Args() {
		var pkg *gotypes.Package
		pkg, err = importer.Import(p)
		if err != nil {
			fmt.Println(err)
			hasFailed = true
			continue
		}
		pkgName := pkg.Name()
		pkgPath := pkg.Path()
		scope := pkg.Scope()
		names := scope.Names()
		for _, name := range names {
			obj := scope.Lookup(name)
			inScopeRef := fmt.Sprintf("%s.%s", pkgName, name)
			fullNameWithPath := fmt.Sprintf("%s.%s", pkgPath, name)
			if obj.Exported() && obj.Type().String() == fullNameWithPath {
				types = append(types, fmt.Sprintf(*fmtStr, inScopeRef))
			}
		}

	}

	fmt.Fprintf(wr, tmpl, strings.Join(types, "\n"))
	err = wr.Close()
	if err != nil {
		fmt.Println(err)
		hasFailed = true
	}
	if hasFailed {
		os.Exit(1)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"

	abcicli "github.com/tendermint/abci/client"
	"github.com/tendermint/abci/types"
	"github.com/tendermint/abci/version"
	"github.com/tendermint/tmlibs/log"
	"github.com/urfave/cli"
)

// Structure for data passed to print response.
type response struct {
	// generic abci response
	Data []byte
	Code types.CodeType
	Log  string

	Query *queryResponse
}

type queryResponse struct {
	Key    []byte
	Value  []byte
	Height uint64
	Proof  []byte
}

// client is a global variable so it can be reused by the console
var client abcicli.Client

var logger log.Logger

func main() {

	//workaround for the cli library (https://github.com/urfave/cli/issues/565)
	cli.OsExiter = func(_ int) {}

	app := cli.NewApp()
	app.Name = "abci-cli"
	app.Usage = "abci-cli [command] [args...]"
	app.Version = version.Version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: "tcp://127.0.0.1:46658",
			Usage: "address of application socket",
		},
		cli.StringFlag{
			Name:  "abci",
			Value: "socket",
			Usage: "socket or grpc",
		},
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "print the command and results as if it were a console session",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:  "batch",
			Usage: "Run a batch of abci commands against an application",
			Action: func(c *cli.Context) error {
				return cmdBatch(app, c)
			},
		},
		{
			Name:  "console",
			Usage: "Start an interactive abci console for multiple commands",
			Action: func(c *cli.Context) error {
				return cmdConsole(app, c)
			},
		},
		{
			Name:  "echo",
			Usage: "Have the application echo a message",
			Action: func(c *cli.Context) error {
				return cmdEcho(c)
			},
		},
		{
			Name:  "info",
			Usage: "Get some info about the application",
			Action: func(c *cli.Context) error {
				return cmdInfo(c)
			},
		},
		{
			Name:  "set_option",
			Usage: "Set an option on the application",
			Action: func(c *cli.Context) error {
				return cmdSetOption(c)
			},
		},
		{
			Name:  "deliver_tx",
			Usage: "Deliver a new tx to application",
			Action: func(c *cli.Context) error {
				return cmdDeliverTx(c)
			},
		},
		{
			Name:  "check_tx",
			Usage: "Validate a tx",
			Action: func(c *cli.Context) error {
				return cmdCheckTx(c)
			},
		},
		{
			Name:  "commit",
			Usage: "Commit the application state and return the Merkle root hash",
			Action: func(c *cli.Context) error {
				return cmdCommit(c)
			},
		},
		{
			Name:  "query",
			Usage: "Query application state",
			Action: func(c *cli.Context) error {
				return cmdQuery(c)
			},
		},
	}
	app.Before = before
	err := app.Run(os.Args)
	if err != nil {
		logger.Error(err.Error())
		os.Exit(1)
	}

}

func before(c *cli.Context) error {
	if logger == nil {
		logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
	}
	if client == nil {
		var err error
		client, err = abcicli.NewClient(c.GlobalString("address"), c.GlobalString("abci"), false)
		if err != nil {
			logger.Error(err.Error())
			os.Exit(1)
		}
		client.SetLogger(logger.With("module", "abci-client"))
	}
	return nil
}

// badCmd is called when we invoke with an invalid first argument (just for console for now)
func badCmd(c *cli.Context, cmd string) {
	fmt.Println("Unknown command:", cmd)
	fmt.Println("Please try one of the following:")
	fmt.Println("")
	cli.DefaultAppComplete(c)
}

//Generates new Args array based off of previous call args to maintain flag persistence
func persistentArgs(line []byte) []string {

	//generate the arguments to run from orginal os.Args
	// to maintain flag arguments
	args := os.Args
	args = args[:len(args)-1] // remove the previous command argument

	if len(line) > 0 { //prevents introduction of extra space leading to argument parse errors
		args = append(args, strings.Split(string(line), " ")...)
	}
	return args
}

//--------------------------------------------------------------------------------

func cmdBatch(app *cli.App, c *cli.Context) error {
	bufReader := bufio.NewReader(os.Stdin)
	for {
		line, more, err := bufReader.ReadLine()
		if more {
			return errors.New("Input line is too long")
		} else if err == io.EOF {
			break
		} else if len(line) == 0 {
			continue
		} else if err != nil {
			return err
		}

		args := persistentArgs(line)
		app.Run(args) //cli prints error within its func call
	}
	return nil
}

func cmdConsole(app *cli.App, c *cli.Context) error {
	// don't hard exit on mistyped commands (eg. check vs check_tx)
	app.CommandNotFound = badCmd

	for {
		fmt.Printf("\n> ")
		bufReader := bufio.NewReader(os.Stdin)
		line, more, err := bufReader.ReadLine()
		if more {
			return errors.New("Input is too long")
		} else if err != nil {
			return err
		}

		args := persistentArgs(line)
		app.Run(args) //cli prints error within its func call
	}
}

// Have the application echo a message
func cmdEcho(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command echo takes 1 argument")
	}
	resEcho := client.EchoSync(args[0])
	printResponse(c, response{
		Data: resEcho.Data,
	})
	return nil
}

// Get some info from the application
func cmdInfo(c *cli.Context) error {
	resInfo, err := client.InfoSync()
	if err != nil {
		return err
	}
	printResponse(c, response{
		Data: []byte(resInfo.Data),
	})
	return nil
}

// Set an option on the application
func cmdSetOption(c *cli.Context) error {
	args := c.Args()
	if len(args) != 2 {
		return errors.New("Command set_option takes 2 arguments (key, value)")
	}
	resSetOption := client.SetOptionSync(args[0], args[1])
	printResponse(c, response{
		Log: resSetOption.Log,
	})
	return nil
}

// Append a new tx to application
func cmdDeliverTx(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command deliver_tx takes 1 argument")
	}
	txBytes, err := stringOrHexToBytes(c.Args()[0])
	if err != nil {
		return err
	}
	res := client.DeliverTxSync(txBytes)
	printResponse(c, response{
		Code: res.Code,
		Data: res.Data,
		Log:  res.Log,
	})
	return nil
}

// Validate a tx
func cmdCheckTx(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command check_tx takes 1 argument")
	}
	txBytes, err := stringOrHexToBytes(c.Args()[0])
	if err != nil {
		return err
	}
	res := client.CheckTxSync(txBytes)
	printResponse(c, response{
		Code: res.Code,
		Data: res.Data,
		Log:  res.Log,
	})
	return nil
}

// Get application Merkle root hash
func cmdCommit(c *cli.Context) error {
	res := client.CommitSync()
	printResponse(c, response{
		Code: res.Code,
		Data: res.Data,
		Log:  res.Log,
	})
	return nil
}

// Query application state
// TODO: Make request and response support all fields.
func cmdQuery(c *cli.Context) error {
	args := c.Args()

	if len(args) == 0 {
		return errors.New("Command query takes 1 or more arguments")
	}

	queryBytes, err := stringOrHexToBytes(args[0])
	if err != nil {
		return err
	}

	var path = "/store"
	if len(args) > 1 {
		path = args[1]
	}

	var height uint64
	if len(args) > 2 {
		height, _ = strconv.ParseUint(args[2], 10, 64)
	}

	var prove = true
	if len(args) > 3 {
		prove, _ = strconv.ParseBool(args[3])
	}

	resQuery, err := client.QuerySync(types.RequestQuery{
		Data:   queryBytes,
		Path:   path,
		Height: height,
		Prove:  prove,
	})
	if err != nil {
		return err
	}
	printResponse(c, response{
		Code: resQuery.Code,
		Log:  resQuery.Log,
		Query: &queryResponse{
			Key:    resQuery.Key,
			Value:  resQuery.Value,
			Height: resQuery.Height,
			Proof:  resQuery.Proof,
		},
	})
	return nil
}

//--------------------------------------------------------------------------------

func printResponse(c *cli.Context, rsp response) {

	verbose := c.GlobalBool("verbose")

	if verbose {
		fmt.Println(">", c.Command.Name, strings.Join(c.Args(), " "))
	}

	if !rsp.Code.IsOK() {
		fmt.Printf("-> code: %s\n", rsp.Code.String())
	}
	if len(rsp.Data) != 0 {
		fmt.Printf("-> data: %s\n", rsp.Data)
		fmt.Printf("-> data.hex: %X\n", rsp.Data)
	}
	if rsp.Log != "" {
		fmt.Printf("-> log: %s\n", rsp.Log)
	}

	if rsp.Query != nil {
		fmt.Printf("-> height: %d\n", rsp.Query.Height)
		if rsp.Query.Key != nil {
			fmt.Printf("-> key: %s\n", rsp.Query.Key)
			fmt.Printf("-> key.hex: %X\n", rsp.Query.Key)
		}
		if rsp.Query.Value != nil {
			fmt.Printf("-> value: %s\n", rsp.Query.Value)
			fmt.Printf("-> value.hex: %X\n", rsp.Query.Value)
		}
		if rsp.Query.Proof != nil {
			fmt.Printf("-> proof: %X\n", rsp.Query.Proof)
		}
	}

	if verbose {
		fmt.Println("")
	}

}

// NOTE: s is interpreted as a string unless prefixed with 0x
func stringOrHexToBytes(s string) ([]byte, error) {
	if len(s) > 2 && strings.ToLower(s[:2]) == "0x" {
		b, err := hex.DecodeString(s[2:])
		if err != nil {
			err = fmt.Errorf("Error decoding hex argument: %s", err.Error())
			return nil, err
		}
		return b, nil
	}

	if !strings.HasPrefix(s, "\"") || !strings.HasSuffix(s, "\"") {
		err := fmt.Errorf("Invalid string arg: \"%s\". Must be quoted or a \"0x\"-prefixed hex string", s)
		return nil, err
	}

	return []byte(s[1 : len(s)-1]), nil
}
<commit_msg>cmd: query params are flags<commit_after>package main

import (
	"bufio"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	abcicli "github.com/tendermint/abci/client"
	"github.com/tendermint/abci/types"
	"github.com/tendermint/abci/version"
	"github.com/tendermint/tmlibs/log"
	"github.com/urfave/cli"
)

// Structure for data passed to print response.
type response struct {
	// generic abci response
	Data []byte
	Code types.CodeType
	Log  string

	Query *queryResponse
}

type queryResponse struct {
	Key    []byte
	Value  []byte
	Height uint64
	Proof  []byte
}

// client is a global variable so it can be reused by the console
var client abcicli.Client

var logger log.Logger

func main() {

	//workaround for the cli library (https://github.com/urfave/cli/issues/565)
	cli.OsExiter = func(_ int) {}

	app := cli.NewApp()
	app.Name = "abci-cli"
	app.Usage = "abci-cli [command] [args...]"
	app.Version = version.Version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: "tcp://127.0.0.1:46658",
			Usage: "address of application socket",
		},
		cli.StringFlag{
			Name:  "abci",
			Value: "socket",
			Usage: "socket or grpc",
		},
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "print the command and results as if it were a console session",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:  "batch",
			Usage: "Run a batch of abci commands against an application",
			Action: func(c *cli.Context) error {
				return cmdBatch(app, c)
			},
		},
		{
			Name:  "console",
			Usage: "Start an interactive abci console for multiple commands",
			Action: func(c *cli.Context) error {
				return cmdConsole(app, c)
			},
		},
		{
			Name:  "echo",
			Usage: "Have the application echo a message",
			Action: func(c *cli.Context) error {
				return cmdEcho(c)
			},
		},
		{
			Name:  "info",
			Usage: "Get some info about the application",
			Action: func(c *cli.Context) error {
				return cmdInfo(c)
			},
		},
		{
			Name:  "set_option",
			Usage: "Set an option on the application",
			Action: func(c *cli.Context) error {
				return cmdSetOption(c)
			},
		},
		{
			Name:  "deliver_tx",
			Usage: "Deliver a new tx to application",
			Action: func(c *cli.Context) error {
				return cmdDeliverTx(c)
			},
		},
		{
			Name:  "check_tx",
			Usage: "Validate a tx",
			Action: func(c *cli.Context) error {
				return cmdCheckTx(c)
			},
		},
		{
			Name:  "commit",
			Usage: "Commit the application state and return the Merkle root hash",
			Action: func(c *cli.Context) error {
				return cmdCommit(c)
			},
		},
		{
			Name:  "query",
			Usage: "Query application state",
			Action: func(c *cli.Context) error {
				return cmdQuery(c)
			},
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "path",
					Value: "/store",
					Usage: "Path to prefix the query with",
				},
				cli.IntFlag{
					Name:  "height",
					Value: 0,
					Usage: "Height to query the blockchain at",
				},
				cli.BoolFlag{
					Name:  "prove",
					Usage: "Whether or not to return a merkle proof of the query result",
				},
			},
		},
	}
	app.Before = before
	err := app.Run(os.Args)
	if err != nil {
		logger.Error(err.Error())
		os.Exit(1)
	}

}

func before(c *cli.Context) error {
	if logger == nil {
		logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
	}
	if client == nil {
		var err error
		client, err = abcicli.NewClient(c.GlobalString("address"), c.GlobalString("abci"), false)
		if err != nil {
			logger.Error(err.Error())
			os.Exit(1)
		}
		client.SetLogger(logger.With("module", "abci-client"))
	}
	return nil
}

// badCmd is called when we invoke with an invalid first argument (just for console for now)
func badCmd(c *cli.Context, cmd string) {
	fmt.Println("Unknown command:", cmd)
	fmt.Println("Please try one of the following:")
	fmt.Println("")
	cli.DefaultAppComplete(c)
}

//Generates new Args array based off of previous call args to maintain flag persistence
func persistentArgs(line []byte) []string {

	//generate the arguments to run from orginal os.Args
	// to maintain flag arguments
	args := os.Args
	args = args[:len(args)-1] // remove the previous command argument

	if len(line) > 0 { //prevents introduction of extra space leading to argument parse errors
		args = append(args, strings.Split(string(line), " ")...)
	}
	return args
}

//--------------------------------------------------------------------------------

func cmdBatch(app *cli.App, c *cli.Context) error {
	bufReader := bufio.NewReader(os.Stdin)
	for {
		line, more, err := bufReader.ReadLine()
		if more {
			return errors.New("Input line is too long")
		} else if err == io.EOF {
			break
		} else if len(line) == 0 {
			continue
		} else if err != nil {
			return err
		}

		args := persistentArgs(line)
		app.Run(args) //cli prints error within its func call
	}
	return nil
}

func cmdConsole(app *cli.App, c *cli.Context) error {
	// don't hard exit on mistyped commands (eg. check vs check_tx)
	app.CommandNotFound = badCmd

	for {
		fmt.Printf("\n> ")
		bufReader := bufio.NewReader(os.Stdin)
		line, more, err := bufReader.ReadLine()
		if more {
			return errors.New("Input is too long")
		} else if err != nil {
			return err
		}

		args := persistentArgs(line)
		app.Run(args) //cli prints error within its func call
	}
}

// Have the application echo a message
func cmdEcho(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command echo takes 1 argument")
	}
	resEcho := client.EchoSync(args[0])
	printResponse(c, response{
		Data: resEcho.Data,
	})
	return nil
}

// Get some info from the application
func cmdInfo(c *cli.Context) error {
	resInfo, err := client.InfoSync()
	if err != nil {
		return err
	}
	printResponse(c, response{
		Data: []byte(resInfo.Data),
	})
	return nil
}

// Set an option on the application
func cmdSetOption(c *cli.Context) error {
	args := c.Args()
	if len(args) != 2 {
		return errors.New("Command set_option takes 2 arguments (key, value)")
	}
	resSetOption := client.SetOptionSync(args[0], args[1])
	printResponse(c, response{
		Log: resSetOption.Log,
	})
	return nil
}

// Append a new tx to application
func cmdDeliverTx(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command deliver_tx takes 1 argument")
	}
	txBytes, err := stringOrHexToBytes(c.Args()[0])
	if err != nil {
		return err
	}
	res := client.DeliverTxSync(txBytes)
	printResponse(c, response{
		Code: res.Code,
		Data: res.Data,
		Log:  res.Log,
	})
	return nil
}

// Validate a tx
func cmdCheckTx(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command check_tx takes 1 argument")
	}
	txBytes, err := stringOrHexToBytes(c.Args()[0])
	if err != nil {
		return err
	}
	res := client.CheckTxSync(txBytes)
	printResponse(c, response{
		Code: res.Code,
		Data: res.Data,
		Log:  res.Log,
	})
	return nil
}

// Get application Merkle root hash
func cmdCommit(c *cli.Context) error {
	res := client.CommitSync()
	printResponse(c, response{
		Code: res.Code,
		Data: res.Data,
		Log:  res.Log,
	})
	return nil
}

// Query application state
func cmdQuery(c *cli.Context) error {
	args := c.Args()

	if len(args) != 1 {
		return errors.New("Command query takes 1 argument, the query bytes")
	}

	queryBytes, err := stringOrHexToBytes(args[0])
	if err != nil {
		return err
	}

	path := c.String("path")
	height := c.Int("height")
	prove := c.Bool("prove")

	resQuery, err := client.QuerySync(types.RequestQuery{
		Data:   queryBytes,
		Path:   path,
		Height: uint64(height),
		Prove:  prove,
	})
	if err != nil {
		return err
	}
	printResponse(c, response{
		Code: resQuery.Code,
		Log:  resQuery.Log,
		Query: &queryResponse{
			Key:    resQuery.Key,
			Value:  resQuery.Value,
			Height: resQuery.Height,
			Proof:  resQuery.Proof,
		},
	})
	return nil
}

//--------------------------------------------------------------------------------

func printResponse(c *cli.Context, rsp response) {

	verbose := c.GlobalBool("verbose")

	if verbose {
		fmt.Println(">", c.Command.Name, strings.Join(c.Args(), " "))
	}

	if !rsp.Code.IsOK() {
		fmt.Printf("-> code: %s\n", rsp.Code.String())
	}
	if len(rsp.Data) != 0 {
		fmt.Printf("-> data: %s\n", rsp.Data)
		fmt.Printf("-> data.hex: %X\n", rsp.Data)
	}
	if rsp.Log != "" {
		fmt.Printf("-> log: %s\n", rsp.Log)
	}

	if rsp.Query != nil {
		fmt.Printf("-> height: %d\n", rsp.Query.Height)
		if rsp.Query.Key != nil {
			fmt.Printf("-> key: %s\n", rsp.Query.Key)
			fmt.Printf("-> key.hex: %X\n", rsp.Query.Key)
		}
		if rsp.Query.Value != nil {
			fmt.Printf("-> value: %s\n", rsp.Query.Value)
			fmt.Printf("-> value.hex: %X\n", rsp.Query.Value)
		}
		if rsp.Query.Proof != nil {
			fmt.Printf("-> proof: %X\n", rsp.Query.Proof)
		}
	}

	if verbose {
		fmt.Println("")
	}

}

// NOTE: s is interpreted as a string unless prefixed with 0x
func stringOrHexToBytes(s string) ([]byte, error) {
	if len(s) > 2 && strings.ToLower(s[:2]) == "0x" {
		b, err := hex.DecodeString(s[2:])
		if err != nil {
			err = fmt.Errorf("Error decoding hex argument: %s", err.Error())
			return nil, err
		}
		return b, nil
	}

	if !strings.HasPrefix(s, "\"") || !strings.HasSuffix(s, "\"") {
		err := fmt.Errorf("Invalid string arg: \"%s\". Must be quoted or a \"0x\"-prefixed hex string", s)
		return nil, err
	}

	return []byte(s[1 : len(s)-1]), nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	log "github.com/Sirupsen/logrus"
	"sync"
	"time"
)

// sqlite3 cannot use a single connection concurrently - thus the mutex
type sqliteDB struct {
	db    *sql.DB
	mutex sync.Mutex
}

func newSqliteDB(filename string) (*sqliteDB, error) {
	database, err := sql.Open("sqlite3", filename)
	if err != nil {
		return nil, err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS payloads(id TEXT, size INTEGER, sha1 TEXT, sha256 TEXT, ver_build INTEGER, ver_branch INTEGER, ver_patch INTEGER, ver_timestamp INTEGER)")
	if err != nil {
		return nil, err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS channel_payload_rel(payload TEXT, channel TEXT)")
	if err != nil {
		return nil, err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS client(id TEXT, name TEXT)")
	if err != nil {
		return nil, err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS channel_client_rel(client TEXT, channel TEXT)")
	if err != nil {
		return nil, err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS events(client TEXT, type INTEGER, result INTEGER, timestamp INTEGER)")
	if err != nil {
		return nil, err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS channel_settings(channel TEXT, force_downgrade INTEGER DEFAULT 0)")
	if err != nil {
		return nil, err
	}

	return &sqliteDB{db: database}, nil
}

func (u *sqliteDB) Close() error {
	return u.db.Close()
}

func (u *sqliteDB) AttachPayloadToChannel(id, channel string) error {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	q, err := u.db.Prepare("INSERT INTO channel_payload_rel (payload,channel) VALUES (?, ?);")
	if err != nil {
		return err
	}

	_, err = q.Exec(id, channel)
	if err != nil {
		return err
	}

	return nil
}

func (u *sqliteDB) AddPayload(id, sha1, sha256 string, size int64, version payloadVersion) error {
	u.mutex.Lock()
	defer u.mutex.Unlock()
	q, err := u.db.Prepare("INSERT INTO payloads (id,size,sha1,sha256,ver_build,ver_branch,ver_patch,ver_timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?);")
	if err != nil {
		return err
	}
	_, err = q.Exec(id, size, sha1, sha256, version.build, version.branch, version.patch, version.timestamp.Unix())
	if err != nil {
		return err
	}

	log.Debugf("DB: added payload '%v', size=%v, version=%v.%v.%v+%v, sha1=%v, sha256=%v,", id, size, version.build, version.branch, version.patch, version.timestamp.Unix(), sha1, sha256)

	return nil
}

func (u *sqliteDB) DeletePayload(id string) error {
	u.mutex.Lock()
	defer u.mutex.Unlock()
	tx, err := u.db.Begin()
	if err != nil {
		return err
	}

	_, err = tx.Exec("DELETE from payloads WHERE id=?;", id)
	if err != nil {
		return err
	}

	_, err = tx.Exec("DELETE from channel_payload_rel WHERE payload=?;", id)
	if err != nil {
		return err
	}

	_, err = tx.Exec("DELETE from channel_settings WHERE channel=?;", id)
	if err != nil {
		return err
	}

	return tx.Commit()
}

func (u *sqliteDB) GetNewerPayload(currentVersion payloadVersion, channel string) (*payload, error) {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	q, err := u.db.Prepare(`SELECT id,size,sha1,sha256,ver_build,ver_branch,ver_patch,ver_timestamp,ifnull(force_downgrade,0) FROM payloads AS P
		JOIN channel_payload_rel AS R ON P.id=R.payload
		LEFT OUTER JOIN channel_settings AS S ON S.channel=R.channel
		WHERE R.channel=?
		ORDER BY ver_build DESC, ver_branch DESC, ver_patch DESC, ver_timestamp DESC LIMIT 1;`)
	if err != nil {
		return nil, err
	}

	result := q.QueryRow(channel)

	var p payload
	var latest payloadVersion
	var forceDowngrade int
	var latestTimestamp int64
	err = result.Scan(&p.ID, &p.Size, &p.SHA1, &p.SHA256, &latest.build, &latest.branch, &latest.patch, &latestTimestamp, &forceDowngrade)
	if err != nil {
		return nil, err
	}
	latest.timestamp = time.Unix(latestTimestamp, 0).UTC()

	if forceDowngrade == 0 {
		if latest.IsGreater(currentVersion) {
			return &p, nil
		}
		return nil, nil
	}

	// forceDowngrade != 0
	if latest.IsEqual(currentVersion) == false {
		return &p, nil
	}

	return nil, nil
}

func (u *sqliteDB) ListChannels() ([]string, error) {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	result, err := u.db.Query("SELECT DISTINCT channel FROM channel_payload_rel;")
	if err != nil {
		return nil, err
	}

	channels := []string{}

	for result.Next() {
		var chanName string
		err = result.Scan(&chanName)
		if err != nil {
			return nil, err
		}
		channels = append(channels, chanName)
	}

	return channels, nil
}

func (u *sqliteDB) ListImages(channel string) ([]payload, error) {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	q, err := u.db.Prepare("SELECT id,ver_build,ver_branch,ver_patch,ver_timestamp,sha1,sha256,size FROM payloads AS P JOIN channel_payload_rel AS R ON P.id=R.payload WHERE R.channel=? ORDER BY ver_build, ver_branch, ver_patch, ver_timestamp;")
	if err != nil {
		return nil, err
	}

	result, err := q.Query(channel)
	if err != nil {
		return nil, err
	}

	out := []payload{}

	for result.Next() {
		var image payload

		var ver payloadVersion
		var timestamp int64

		err = result.Scan(&image.ID, &ver.build, &ver.branch, &ver.patch, &timestamp, &image.SHA1, &image.SHA256, &image.Size)
		if err != nil {
			return nil, err
		}

		ver.timestamp = time.Unix(timestamp, 0).UTC()
		image.Version = ver.String()
		out = append(out, image)
	}

	return out, nil
}

func (u *sqliteDB) LogEvent(client string, evType, evResult int) error {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	q, err := u.db.Prepare("INSERT INTO events (client,type,result,timestamp) VALUES (?, ?, ?, ?);")
	if err != nil {
		return err
	}

	_, err = q.Exec(client, evType, evResult, time.Now().UTC().Unix())

	return err
}

type Event struct {
	MachineID string
	Type      int
	Result    int
	Timestamp string
}

func (u *sqliteDB) GetEvents() ([]Event, error) {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	q, err := u.db.Prepare("SELECT client,type,result,timestamp FROM events ORDER BY timestamp ASC;")
	if err != nil {
		return nil, err
	}

	result, err := q.Query()
	if err != nil {
		return nil, err
	}

	out := []Event{}

	for result.Next() {
		var ev Event

		var timestamp int64

		err = result.Scan(&ev.MachineID, &ev.Type, &ev.Result, &timestamp)
		if err != nil {
			return nil, err
		}

		ev.Timestamp = time.Unix(timestamp, 0).UTC().String()
		out = append(out, ev)
	}

	return out, nil
}

func (u *sqliteDB) SetChannelForceDowngrade(channel string, value bool) error {
	var intValue int

	if value {
		intValue = 1
	} else {
		intValue = 0
	}

	u.mutex.Lock()
	defer u.mutex.Unlock()

	result, err := u.db.Exec("UPDATE channel_settings SET force_downgrade=? WHERE channel=?", intValue, channel)
	if err != nil {
		return err
	}

	affected, err := result.RowsAffected()
	if err != nil {
		return err
	}

	if affected == 0 {
		_, err = u.db.Exec("INSERT OR IGNORE INTO channel_settings (channel, force_downgrade) VALUES (?, ?);", channel, intValue)
		if err != nil {
			return err
		}
	}

	return nil
}

func (u *sqliteDB) GetChannelForceDowngrade(channel string) (bool, error) {
	row := u.db.QueryRow("SELECT force_downgrade FROM channel_settings WHERE channel=?;", channel)

	var intValue int
	err := row.Scan(&intValue)
	if err != nil {
		// unset, returning default
		if err == sql.ErrNoRows {
			return false, nil
		}

		return false, err
	}

	if intValue == 0 {
		return false, nil
	}

	return true, nil
}
<commit_msg>move database initialization to separate function<commit_after>package main

import (
	"database/sql"
	log "github.com/Sirupsen/logrus"
	"sync"
	"time"
)

// sqlite3 cannot use a single connection concurrently - thus the mutex
type sqliteDB struct {
	db    *sql.DB
	mutex sync.Mutex
}

func newSqliteDB(filename string) (*sqliteDB, error) {
	database, err := sql.Open("sqlite3", filename)
	if err != nil {
		return nil, err
	}

	err = initStructure(database)
	if err != nil {
		return nil, err
	}

	return &sqliteDB{db: database}, nil
}

func initStructure(database *sql.DB) error {
	_, err := database.Exec("CREATE TABLE IF NOT EXISTS payloads(id TEXT, size INTEGER, sha1 TEXT, sha256 TEXT, ver_build INTEGER, ver_branch INTEGER, ver_patch INTEGER, ver_timestamp INTEGER)")
	if err != nil {
		return err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS channel_payload_rel(payload TEXT, channel TEXT)")
	if err != nil {
		return err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS client(id TEXT, name TEXT)")
	if err != nil {
		return err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS channel_client_rel(client TEXT, channel TEXT)")
	if err != nil {
		return err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS events(client TEXT, type INTEGER, result INTEGER, timestamp INTEGER)")
	if err != nil {
		return err
	}

	_, err = database.Exec("CREATE TABLE IF NOT EXISTS channel_settings(channel TEXT, force_downgrade INTEGER DEFAULT 0)")
	if err != nil {
		return err
	}

	return nil
}

func (u *sqliteDB) Close() error {
	return u.db.Close()
}

func (u *sqliteDB) AttachPayloadToChannel(id, channel string) error {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	q, err := u.db.Prepare("INSERT INTO channel_payload_rel (payload,channel) VALUES (?, ?);")
	if err != nil {
		return err
	}

	_, err = q.Exec(id, channel)
	if err != nil {
		return err
	}

	return nil
}

func (u *sqliteDB) AddPayload(id, sha1, sha256 string, size int64, version payloadVersion) error {
	u.mutex.Lock()
	defer u.mutex.Unlock()
	q, err := u.db.Prepare("INSERT INTO payloads (id,size,sha1,sha256,ver_build,ver_branch,ver_patch,ver_timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?);")
	if err != nil {
		return err
	}
	_, err = q.Exec(id, size, sha1, sha256, version.build, version.branch, version.patch, version.timestamp.Unix())
	if err != nil {
		return err
	}

	log.Debugf("DB: added payload '%v', size=%v, version=%v.%v.%v+%v, sha1=%v, sha256=%v,", id, size, version.build, version.branch, version.patch, version.timestamp.Unix(), sha1, sha256)

	return nil
}

func (u *sqliteDB) DeletePayload(id string) error {
	u.mutex.Lock()
	defer u.mutex.Unlock()
	tx, err := u.db.Begin()
	if err != nil {
		return err
	}

	_, err = tx.Exec("DELETE from payloads WHERE id=?;", id)
	if err != nil {
		return err
	}

	_, err = tx.Exec("DELETE from channel_payload_rel WHERE payload=?;", id)
	if err != nil {
		return err
	}

	_, err = tx.Exec("DELETE from channel_settings WHERE channel=?;", id)
	if err != nil {
		return err
	}

	return tx.Commit()
}

func (u *sqliteDB) GetNewerPayload(currentVersion payloadVersion, channel string) (*payload, error) {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	q, err := u.db.Prepare(`SELECT id,size,sha1,sha256,ver_build,ver_branch,ver_patch,ver_timestamp,ifnull(force_downgrade,0) FROM payloads AS P
		JOIN channel_payload_rel AS R ON P.id=R.payload
		LEFT OUTER JOIN channel_settings AS S ON S.channel=R.channel
		WHERE R.channel=?
		ORDER BY ver_build DESC, ver_branch DESC, ver_patch DESC, ver_timestamp DESC LIMIT 1;`)
	if err != nil {
		return nil, err
	}

	result := q.QueryRow(channel)

	var p payload
	var latest payloadVersion
	var forceDowngrade int
	var latestTimestamp int64
	err = result.Scan(&p.ID, &p.Size, &p.SHA1, &p.SHA256, &latest.build, &latest.branch, &latest.patch, &latestTimestamp, &forceDowngrade)
	if err != nil {
		return nil, err
	}
	latest.timestamp = time.Unix(latestTimestamp, 0).UTC()

	if forceDowngrade == 0 {
		if latest.IsGreater(currentVersion) {
			return &p, nil
		}
		return nil, nil
	}

	// forceDowngrade != 0
	if latest.IsEqual(currentVersion) == false {
		return &p, nil
	}

	return nil, nil
}

func (u *sqliteDB) ListChannels() ([]string, error) {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	result, err := u.db.Query("SELECT DISTINCT channel FROM channel_payload_rel;")
	if err != nil {
		return nil, err
	}

	channels := []string{}

	for result.Next() {
		var chanName string
		err = result.Scan(&chanName)
		if err != nil {
			return nil, err
		}
		channels = append(channels, chanName)
	}

	return channels, nil
}

func (u *sqliteDB) ListImages(channel string) ([]payload, error) {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	q, err := u.db.Prepare("SELECT id,ver_build,ver_branch,ver_patch,ver_timestamp,sha1,sha256,size FROM payloads AS P JOIN channel_payload_rel AS R ON P.id=R.payload WHERE R.channel=? ORDER BY ver_build, ver_branch, ver_patch, ver_timestamp;")
	if err != nil {
		return nil, err
	}

	result, err := q.Query(channel)
	if err != nil {
		return nil, err
	}

	out := []payload{}

	for result.Next() {
		var image payload

		var ver payloadVersion
		var timestamp int64

		err = result.Scan(&image.ID, &ver.build, &ver.branch, &ver.patch, &timestamp, &image.SHA1, &image.SHA256, &image.Size)
		if err != nil {
			return nil, err
		}

		ver.timestamp = time.Unix(timestamp, 0).UTC()
		image.Version = ver.String()
		out = append(out, image)
	}

	return out, nil
}

func (u *sqliteDB) LogEvent(client string, evType, evResult int) error {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	q, err := u.db.Prepare("INSERT INTO events (client,type,result,timestamp) VALUES (?, ?, ?, ?);")
	if err != nil {
		return err
	}

	_, err = q.Exec(client, evType, evResult, time.Now().UTC().Unix())

	return err
}

type Event struct {
	MachineID string
	Type      int
	Result    int
	Timestamp string
}

func (u *sqliteDB) GetEvents() ([]Event, error) {
	u.mutex.Lock()
	defer u.mutex.Unlock()

	q, err := u.db.Prepare("SELECT client,type,result,timestamp FROM events ORDER BY timestamp ASC;")
	if err != nil {
		return nil, err
	}

	result, err := q.Query()
	if err != nil {
		return nil, err
	}

	out := []Event{}

	for result.Next() {
		var ev Event

		var timestamp int64

		err = result.Scan(&ev.MachineID, &ev.Type, &ev.Result, &timestamp)
		if err != nil {
			return nil, err
		}

		ev.Timestamp = time.Unix(timestamp, 0).UTC().String()
		out = append(out, ev)
	}

	return out, nil
}

func (u *sqliteDB) SetChannelForceDowngrade(channel string, value bool) error {
	var intValue int

	if value {
		intValue = 1
	} else {
		intValue = 0
	}

	u.mutex.Lock()
	defer u.mutex.Unlock()

	result, err := u.db.Exec("UPDATE channel_settings SET force_downgrade=? WHERE channel=?", intValue, channel)
	if err != nil {
		return err
	}

	affected, err := result.RowsAffected()
	if err != nil {
		return err
	}

	if affected == 0 {
		_, err = u.db.Exec("INSERT OR IGNORE INTO channel_settings (channel, force_downgrade) VALUES (?, ?);", channel, intValue)
		if err != nil {
			return err
		}
	}

	return nil
}

func (u *sqliteDB) GetChannelForceDowngrade(channel string) (bool, error) {
	row := u.db.QueryRow("SELECT force_downgrade FROM channel_settings WHERE channel=?;", channel)

	var intValue int
	err := row.Scan(&intValue)
	if err != nil {
		// unset, returning default
		if err == sql.ErrNoRows {
			return false, nil
		}

		return false, err
	}

	if intValue == 0 {
		return false, nil
	}

	return true, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/raintank-metric/metric_tank/consolidation"
	"runtime"
	"sort"
)

// doRecover is the handler that turns panics into returns from the top level of getTarget.
func doRecover(errp *error) {
	e := recover()
	if e != nil {
		if _, ok := e.(runtime.Error); ok {
			panic(e)
		}
		if err, ok := e.(error); ok {
			*errp = err
		} else if errStr, ok := e.(string); ok {
			*errp = errors.New(errStr)
		} else {
			*errp = fmt.Errorf("%v", e)
		}
	}
	return
}

func divide(pointsA, pointsB []Point) []Point {
	if len(pointsA) != len(pointsB) {
		panic(fmt.Errorf("divide of a series with len %d by a series with len %d", len(pointsA), len(pointsB)))
	}
	out := make([]Point, len(pointsA))
	for i, a := range pointsA {
		b := pointsB[i]
		out[i] = Point{a.Val / b.Val, a.Ts}
	}
	return out
}

func consolidate(in []Point, num int, consolidator consolidation.Consolidator) []Point {
	aggFunc := consolidation.GetAggFunc(consolidator)
	buf := make([]float64, num)
	bufpos := -1
	outLen := len(in) / num
	if len(in)%num != 0 {
		outLen += 1
	}
	points := make([]Point, 0, outLen)
	for inpos, p := range in {
		bufpos = inpos % num
		buf[bufpos] = p.Val
		if bufpos == num-1 {
			points = append(points, Point{aggFunc(buf), p.Ts})
		}
	}
	if bufpos != -1 && bufpos < num-1 {
		// we have an incomplete buf of some points that didn't get aggregated yet
		points = append(points, Point{aggFunc(buf[:bufpos+1]), in[len(in)-1].Ts})
	}
	return points
}

// returns how many points should be aggregated together so that you end up with as many points as possible,
// but never more than maxPoints
func aggEvery(numPoints, maxPoints uint32) int {
	return int((numPoints + maxPoints - 1) / maxPoints)
}

type planOption struct {
	archive  string
	interval uint32
	intestim bool
	points   uint32
	comment  string
}

type plan []planOption

func (a plan) Len() int           { return len(a) }
func (a plan) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a plan) Less(i, j int) bool { return a[i].points > a[j].points }

func getTarget(req Req, aggSettings []aggSetting, metaCache *MetaCache) (points []Point, err error) {
	defer doRecover(&err)
	archive := -1 // -1 means original data, 0 last agg level, 1 2nd last, etc.

	p := make([]planOption, len(aggSettings)+1)
	guess := false

	// note: the metacache is clearly not a perfect all-knowning entity, it just knows the last interval of metrics seen since program start
	// and we assume we can use that interval through history.
	// TODO: no support for interval changes, metrics not seen yet, missing datablocks, ...
	meta := metaCache.Get(req.key)
	interval := uint32(meta.interval)

	// we don't have the data yet, let's assume the interval is 10 seconds
	if interval == 0 {
		guess = true
		interval = 10
	}
	numPoints := (req.to - req.from) / interval

	p[0] = planOption{"raw", interval, guess, numPoints, ""}

	aggs := aggSettingsSpanDesc(aggSettings)
	sort.Sort(aggs)
	finished := false
	for i, aggSetting := range aggs {
		numPointsHere := (req.to - req.from) / aggSetting.span
		p[i+1] = planOption{fmt.Sprintf("agg %d", i), aggSetting.span, false, numPointsHere, ""}
		if numPointsHere >= req.minPoints && !finished {
			archive = i
			interval = aggSetting.span
			numPoints = numPointsHere
			finished = true
		}
	}

	p[archive+1].comment = "<-- chosen"

	// note, it should always be safe to dynamically switch on/off consolidation based on how well our data stacks up against the request
	// i.e. whether your data got consolidated or not, it should be pretty equivalent.
	// for that reason, stdev should not be done as a consolidation. but sos is still useful for when we explicitly (and always, not optionally) want the stdev.

	readConsolidated := (archive != -1)                 // do we need to read from a downsampled series?
	runtimeConsolidation := (numPoints > req.maxPoints) // do we need to compress any points at runtime?

	log.Debug("getTarget()         %s", req)
	log.Debug("type   interval   points")
	sortedPlan := plan(p)
	sort.Sort(sortedPlan)
	for _, opt := range p {
		iStr := fmt.Sprintf("%d", opt.interval)
		if opt.intestim {
			iStr = fmt.Sprintf("%d (guess)", opt.interval)
		}
		log.Debug("%-6s %-10s %-6d %s", opt.archive, iStr, opt.points, opt.comment)
	}
	log.Debug("runtimeConsolidation: %t", runtimeConsolidation)

	if !readConsolidated && !runtimeConsolidation {
		return getSeries(req.key, consolidation.None, 0, req.from, req.to), nil
	} else if !readConsolidated && runtimeConsolidation {
		return consolidate(
			getSeries(req.key, consolidation.None, 0, req.from, req.to),
			aggEvery(numPoints, req.maxPoints),
			req.consolidator), nil
	} else if readConsolidated && !runtimeConsolidation {
		if req.consolidator == consolidation.Avg {
			return divide(
				getSeries(req.key, consolidation.Sum, interval, req.from, req.to),
				getSeries(req.key, consolidation.Cnt, interval, req.from, req.to),
			), nil
		} else {
			return getSeries(req.key, req.consolidator, interval, req.from, req.to), nil
		}
	} else {
		// readConsolidated && runtimeConsolidation
		aggNum := aggEvery(numPoints, req.maxPoints)
		if req.consolidator == consolidation.Avg {
			return divide(
				consolidate(
					getSeries(req.key, consolidation.Sum, interval, req.from, req.to),
					aggNum,
					consolidation.Sum),
				consolidate(
					getSeries(req.key, consolidation.Cnt, interval, req.from, req.to),
					aggNum,
					consolidation.Cnt),
			), nil
		} else {
			return consolidate(
				getSeries(req.key, req.consolidator, interval, req.from, req.to),
				aggNum, req.consolidator), nil
		}
	}
}

func logLoad(typ, key string, from, to uint32) {
	log.Debug("load from %-6s %-20s %d - %d (%s - %s) span:%ds", typ, key, from, to, TS(from), TS(to), to-from-1)
}

func aggMetricKey(key, archive string, aggSpan uint32) string {
	return fmt.Sprintf("%s_%s_%d", key, archive, aggSpan)
}

// getSeries just gets the needed raw iters from mem and/or cassandra, based on from/to
// it can query for data within aggregated archives, by using fn min/max/sos/sum/cnt and providing the matching agg span.
func getSeries(key string, consolidator consolidation.Consolidator, aggSpan, fromUnix, toUnix uint32) []Point {
	iters := make([]Iter, 0)
	memIters := make([]Iter, 0)
	oldest := toUnix
	if metric, ok := metrics.Get(key); ok {
		if consolidator != consolidation.None {
			logLoad("memory", aggMetricKey(key, consolidator.Archive(), aggSpan), fromUnix, toUnix)
			oldest, memIters = metric.GetAggregated(consolidator, aggSpan, fromUnix, toUnix)
		} else {
			logLoad("memory", key, fromUnix, toUnix)
			oldest, memIters = metric.Get(fromUnix, toUnix)
		}
	}
	if oldest > fromUnix {
		reqSpanBoth.Value(int64(toUnix - fromUnix))
		if consolidator != consolidation.None {
			key = aggMetricKey(key, consolidator.Archive(), aggSpan)
		}
		logLoad("cassan", key, fromUnix, oldest)
		storeIters, err := searchCassandra(key, fromUnix, oldest)
		if err != nil {
			panic(err)
		}
		iters = append(iters, storeIters...)
	} else {
		reqSpanMem.Value(int64(toUnix - fromUnix))
	}
	iters = append(iters, memIters...)

	points := make([]Point, 0)
	for _, iter := range iters {
		total := 0
		good := 0
		for iter.Next() {
			total += 1
			ts, val := iter.Values()
			if ts >= fromUnix && ts < toUnix {
				good += 1
				points = append(points, Point{val, ts})
			}
		}
		log.Debug("getSeries: iter %s  values good/total %d/%d", iter.cmt, good, total)
	}
	return points
}
<commit_msg>fix: properly align timestamp of consolidation output for uneven input<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/raintank-metric/metric_tank/consolidation"
	"runtime"
	"sort"
)

// doRecover is the handler that turns panics into returns from the top level of getTarget.
func doRecover(errp *error) {
	e := recover()
	if e != nil {
		if _, ok := e.(runtime.Error); ok {
			panic(e)
		}
		if err, ok := e.(error); ok {
			*errp = err
		} else if errStr, ok := e.(string); ok {
			*errp = errors.New(errStr)
		} else {
			*errp = fmt.Errorf("%v", e)
		}
	}
	return
}

func divide(pointsA, pointsB []Point) []Point {
	if len(pointsA) != len(pointsB) {
		panic(fmt.Errorf("divide of a series with len %d by a series with len %d", len(pointsA), len(pointsB)))
	}
	out := make([]Point, len(pointsA))
	for i, a := range pointsA {
		b := pointsB[i]
		out[i] = Point{a.Val / b.Val, a.Ts}
	}
	return out
}

func consolidate(in []Point, num int, consolidator consolidation.Consolidator) []Point {
	aggFunc := consolidation.GetAggFunc(consolidator)
	buf := make([]float64, num)
	bufpos := -1
	outLen := len(in) / num
	if len(in)%num != 0 {
		outLen += 1
	}
	points := make([]Point, 0, outLen)
	for inpos, p := range in {
		bufpos = inpos % num
		buf[bufpos] = p.Val
		if bufpos == num-1 {
			points = append(points, Point{aggFunc(buf), p.Ts})
		}
	}
	if bufpos != -1 && bufpos < num-1 {
		// we have an incomplete buf of some points that didn't get aggregated yet
		// we must also aggregate it and add it, and the timestamp of this point must be what it would have been
		// if the buf would have been complete, i.e. points in the consolidation output should be evenly spaced.
		// obviously we can only figure out the interval if we have at least 2 points
		var lastTs uint32
		if len(in) == 1 {
			lastTs = in[0].Ts
		} else {
			interval := in[len(in)-1].Ts - in[len(in)-2].Ts
			// len 10, num 3 -> 3*4 values supposedly -> "in[11].Ts" -> in[9].Ts + 2*interval
			lastTs = in[len(in)-1].Ts + uint32(num-len(in)%num)*interval
		}
		points = append(points, Point{aggFunc(buf[:bufpos+1]), lastTs})
	}
	return points
}

// returns how many points should be aggregated together so that you end up with as many points as possible,
// but never more than maxPoints
func aggEvery(numPoints, maxPoints uint32) int {
	return int((numPoints + maxPoints - 1) / maxPoints)
}

type planOption struct {
	archive  string
	interval uint32
	intestim bool
	points   uint32
	comment  string
}

type plan []planOption

func (a plan) Len() int           { return len(a) }
func (a plan) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a plan) Less(i, j int) bool { return a[i].points > a[j].points }

func getTarget(req Req, aggSettings []aggSetting, metaCache *MetaCache) (points []Point, err error) {
	defer doRecover(&err)
	archive := -1 // -1 means original data, 0 last agg level, 1 2nd last, etc.

	p := make([]planOption, len(aggSettings)+1)
	guess := false

	// note: the metacache is clearly not a perfect all-knowning entity, it just knows the last interval of metrics seen since program start
	// and we assume we can use that interval through history.
	// TODO: no support for interval changes, metrics not seen yet, missing datablocks, ...
	meta := metaCache.Get(req.key)
	interval := uint32(meta.interval)

	// we don't have the data yet, let's assume the interval is 10 seconds
	if interval == 0 {
		guess = true
		interval = 10
	}
	numPoints := (req.to - req.from) / interval

	p[0] = planOption{"raw", interval, guess, numPoints, ""}

	aggs := aggSettingsSpanDesc(aggSettings)
	sort.Sort(aggs)
	finished := false
	for i, aggSetting := range aggs {
		numPointsHere := (req.to - req.from) / aggSetting.span
		p[i+1] = planOption{fmt.Sprintf("agg %d", i), aggSetting.span, false, numPointsHere, ""}
		if numPointsHere >= req.minPoints && !finished {
			archive = i
			interval = aggSetting.span
			numPoints = numPointsHere
			finished = true
		}
	}

	p[archive+1].comment = "<-- chosen"

	// note, it should always be safe to dynamically switch on/off consolidation based on how well our data stacks up against the request
	// i.e. whether your data got consolidated or not, it should be pretty equivalent.
	// for that reason, stdev should not be done as a consolidation. but sos is still useful for when we explicitly (and always, not optionally) want the stdev.

	readConsolidated := (archive != -1)                 // do we need to read from a downsampled series?
	runtimeConsolidation := (numPoints > req.maxPoints) // do we need to compress any points at runtime?

	log.Debug("getTarget()         %s", req)
	log.Debug("type   interval   points")
	sortedPlan := plan(p)
	sort.Sort(sortedPlan)
	for _, opt := range p {
		iStr := fmt.Sprintf("%d", opt.interval)
		if opt.intestim {
			iStr = fmt.Sprintf("%d (guess)", opt.interval)
		}
		log.Debug("%-6s %-10s %-6d %s", opt.archive, iStr, opt.points, opt.comment)
	}
	log.Debug("runtimeConsolidation: %t", runtimeConsolidation)

	if !readConsolidated && !runtimeConsolidation {
		return getSeries(req.key, consolidation.None, 0, req.from, req.to), nil
	} else if !readConsolidated && runtimeConsolidation {
		return consolidate(
			getSeries(req.key, consolidation.None, 0, req.from, req.to),
			aggEvery(numPoints, req.maxPoints),
			req.consolidator), nil
	} else if readConsolidated && !runtimeConsolidation {
		if req.consolidator == consolidation.Avg {
			return divide(
				getSeries(req.key, consolidation.Sum, interval, req.from, req.to),
				getSeries(req.key, consolidation.Cnt, interval, req.from, req.to),
			), nil
		} else {
			return getSeries(req.key, req.consolidator, interval, req.from, req.to), nil
		}
	} else {
		// readConsolidated && runtimeConsolidation
		aggNum := aggEvery(numPoints, req.maxPoints)
		if req.consolidator == consolidation.Avg {
			return divide(
				consolidate(
					getSeries(req.key, consolidation.Sum, interval, req.from, req.to),
					aggNum,
					consolidation.Sum),
				consolidate(
					getSeries(req.key, consolidation.Cnt, interval, req.from, req.to),
					aggNum,
					consolidation.Cnt),
			), nil
		} else {
			return consolidate(
				getSeries(req.key, req.consolidator, interval, req.from, req.to),
				aggNum, req.consolidator), nil
		}
	}
}

func logLoad(typ, key string, from, to uint32) {
	log.Debug("load from %-6s %-20s %d - %d (%s - %s) span:%ds", typ, key, from, to, TS(from), TS(to), to-from-1)
}

func aggMetricKey(key, archive string, aggSpan uint32) string {
	return fmt.Sprintf("%s_%s_%d", key, archive, aggSpan)
}

// getSeries just gets the needed raw iters from mem and/or cassandra, based on from/to
// it can query for data within aggregated archives, by using fn min/max/sos/sum/cnt and providing the matching agg span.
func getSeries(key string, consolidator consolidation.Consolidator, aggSpan, fromUnix, toUnix uint32) []Point {
	iters := make([]Iter, 0)
	memIters := make([]Iter, 0)
	oldest := toUnix
	if metric, ok := metrics.Get(key); ok {
		if consolidator != consolidation.None {
			logLoad("memory", aggMetricKey(key, consolidator.Archive(), aggSpan), fromUnix, toUnix)
			oldest, memIters = metric.GetAggregated(consolidator, aggSpan, fromUnix, toUnix)
		} else {
			logLoad("memory", key, fromUnix, toUnix)
			oldest, memIters = metric.Get(fromUnix, toUnix)
		}
	}
	if oldest > fromUnix {
		reqSpanBoth.Value(int64(toUnix - fromUnix))
		if consolidator != consolidation.None {
			key = aggMetricKey(key, consolidator.Archive(), aggSpan)
		}
		logLoad("cassan", key, fromUnix, oldest)
		storeIters, err := searchCassandra(key, fromUnix, oldest)
		if err != nil {
			panic(err)
		}
		iters = append(iters, storeIters...)
	} else {
		reqSpanMem.Value(int64(toUnix - fromUnix))
	}
	iters = append(iters, memIters...)

	points := make([]Point, 0)
	for _, iter := range iters {
		total := 0
		good := 0
		for iter.Next() {
			total += 1
			ts, val := iter.Values()
			if ts >= fromUnix && ts < toUnix {
				good += 1
				points = append(points, Point{val, ts})
			}
		}
		log.Debug("getSeries: iter %s  values good/total %d/%d", iter.cmt, good, total)
	}
	return points
}
<|endoftext|>


<commit_before>// Copyright 2013 Martin Schnabel. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ot

import (
	"encoding/json"
	"testing"
)

func TestOpsCount(t *testing.T) {
	var o Ops
	checklen := func(bl, tl int) {
		ret, del, ins := o.Count()
		if l := ret + del; l != bl {
			t.Errorf("base len %d != %d", l, bl)
		}
		if l := ret + ins; l != tl {
			t.Errorf("taget len %d != %d", l, tl)
		}
	}
	checklen(0, 0)
	o = append(o, Op{N: 5})
	checklen(5, 5)
	o = append(o, Op{S: "abc"})
	checklen(5, 8)
	o = append(o, Op{N: 2})
	checklen(7, 10)
	o = append(o, Op{N: -2})
	checklen(9, 10)
}

func TestOpsMerge(t *testing.T) {
	o := Ops{
		{N: 5}, {N: 2}, {},
		{S: "lo"}, {S: "rem"}, {},
		{N: -3}, {N: -2}, {},
	}
	if mo := Merge(o); len(mo) != 3 {
		t.Errorf("got %+v", mo)
	}
}

func TestOpsEqual(t *testing.T) {
	var a, b Ops
	if !a.Equal(b) || !b.Equal(a) {
		t.Errorf("expect equal %v %v", a, b)
	}
	a = Ops{{N: 7}, {S: "lorem"}, {N: -5}}
	if a.Equal(b) || b.Equal(a) {
		t.Errorf("expect not equal %v %v", a, b)
	}
	b = Ops{{N: 7}, {S: "lorem"}, {N: -5}}
	if !a.Equal(b) || !b.Equal(a) {
		t.Errorf("expect equal %v %v", a, b)
	}
}

func TestOpsEncoding(t *testing.T) {
	e := `[7,"lorem",-5]`
	o := Ops{{N: 7}, {S: "lorem"}, {N: -5}}
	oe, err := json.Marshal(o)
	if err != nil {
		t.Error(err)
	}
	if string(oe) != e {
		t.Errorf("expected %s got %s", e, oe)
	}
	var eo Ops
	err = json.Unmarshal([]byte(e), &eo)
	if err != nil {
		t.Error(err)
	}
	if !o.Equal(eo) {
		t.Errorf("expected %v got %v", o, eo)
	}
}

var composeTests = []struct {
	a, b, ab Ops
}{
	{
		a:  Ops{{N: 3}},
		b:  Ops{{N: 1}, {S: "tag"}, {N: 2}},
		ab: Ops{{N: 1}, {S: "tag"}, {N: 2}},
	},
	{
		a:  Ops{{N: 1}, {S: "tag"}, {N: 2}},
		b:  Ops{{N: 4}, {N: -2}},
		ab: Ops{{N: 1}, {S: "tag"}, {N: -2}},
	},
}

func TestOpsCompose(t *testing.T) {
	for _, c := range composeTests {
		ab, err := Compose(c.a, c.b)
		if err != nil {
			t.Error(err)
		}
		if !ab.Equal(c.ab) {
			t.Errorf("expected %v got %v", c.ab, ab)
		}
	}
}

var transformTests = []struct {
	a, b, a1, b1 Ops
}{
	{
		a:  Ops{{N: 1}, {S: "tag"}, {N: 2}},
		b:  Ops{{N: 2}, {N: -1}},
		a1: Ops{{N: 1}, {S: "tag"}, {N: 1}},
		b1: Ops{{N: 5}, {N: -1}},
	},
	{
		a:  Ops{{N: 1}, {S: "tag"}, {N: 2}},
		b:  Ops{{N: 1}, {S: "tag"}, {N: 2}},
		a1: Ops{{N: 1}, {S: "tag"}, {N: 5}},
		b1: Ops{{N: 4}, {S: "tag"}, {N: 2}},
	},
}

func TestOpsTransform(t *testing.T) {
	for _, c := range transformTests {
		a1, b1, err := Transform(c.a, c.b)
		if err != nil {
			t.Error(err)
		}
		if !a1.Equal(c.a1) {
			t.Errorf("expected %v got %v", c.a1, a1)
		}
		if !b1.Equal(c.b1) {
			t.Errorf("expected %v got %v", c.b1, b1)
		}
	}
}
<commit_msg>adds more go operational transformation tests<commit_after>// Copyright 2013 Martin Schnabel. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ot

import (
	"encoding/json"
	"testing"
)

func TestOpsCount(t *testing.T) {
	var o Ops
	checklen := func(bl, tl int) {
		ret, del, ins := o.Count()
		if l := ret + del; l != bl {
			t.Errorf("base len %d != %d", l, bl)
		}
		if l := ret + ins; l != tl {
			t.Errorf("taget len %d != %d", l, tl)
		}
	}
	checklen(0, 0)
	o = append(o, Op{N: 5})
	checklen(5, 5)
	o = append(o, Op{S: "abc"})
	checklen(5, 8)
	o = append(o, Op{N: 2})
	checklen(7, 10)
	o = append(o, Op{N: -2})
	checklen(9, 10)
}

func TestOpsMerge(t *testing.T) {
	o := Ops{
		{N: 5}, {N: 2}, {},
		{S: "lo"}, {S: "rem"}, {},
		{N: -3}, {N: -2}, {},
	}
	if mo := Merge(o); len(mo) != 3 {
		t.Errorf("got %+v", mo)
	}
}

func TestOpsEqual(t *testing.T) {
	var a, b Ops
	if !a.Equal(b) || !b.Equal(a) {
		t.Errorf("expect equal %v %v", a, b)
	}
	a = Ops{{N: 7}, {S: "lorem"}, {N: -5}}
	if a.Equal(b) || b.Equal(a) {
		t.Errorf("expect not equal %v %v", a, b)
	}
	b = Ops{{N: 7}, {S: "lorem"}, {N: -5}}
	if !a.Equal(b) || !b.Equal(a) {
		t.Errorf("expect equal %v %v", a, b)
	}
}

func TestOpsEncoding(t *testing.T) {
	e := `[7,"lorem",-5]`
	o := Ops{{N: 7}, {S: "lorem"}, {N: -5}}
	oe, err := json.Marshal(o)
	if err != nil {
		t.Error(err)
	}
	if string(oe) != e {
		t.Errorf("expected %s got %s", e, oe)
	}
	var eo Ops
	err = json.Unmarshal([]byte(e), &eo)
	if err != nil {
		t.Error(err)
	}
	if !o.Equal(eo) {
		t.Errorf("expected %v got %v", o, eo)
	}
}

var composeTests = []struct {
	a, b, ab Ops
}{
	{
		a:  Ops{{N: 3}, {N: -1}},
		b:  Ops{{N: 1}, {S: "tag"}, {N: 2}},
		ab: Ops{{N: 1}, {S: "tag"}, {N: 2}, {N: -1}},
	},
	{
		a:  Ops{{N: 1}, {S: "tag"}, {N: 2}},
		b:  Ops{{N: 4}, {N: -2}},
		ab: Ops{{N: 1}, {S: "tag"}, {N: -2}},
	},
	{
		a:  Ops{{N: 1}, {S: "tag"}},
		b:  Ops{{N: 2}, {N: -2}},
		ab: Ops{{N: 1}, {S: "t"}},
	},
}

func TestOpsCompose(t *testing.T) {
	for _, c := range composeTests {
		ab, err := Compose(c.a, c.b)
		if err != nil {
			t.Error(err)
		}
		if !ab.Equal(c.ab) {
			t.Errorf("expected %v got %v", c.ab, ab)
		}
	}
}

var transformTests = []struct {
	a, b, a1, b1 Ops
}{
	{
		a:  Ops{{N: 1}, {S: "tag"}, {N: 2}},
		b:  Ops{{N: 2}, {N: -1}},
		a1: Ops{{N: 1}, {S: "tag"}, {N: 1}},
		b1: Ops{{N: 5}, {N: -1}},
	},
	{
		a:  Ops{{N: 1}, {S: "tag"}, {N: 2}},
		b:  Ops{{N: 1}, {S: "tag"}, {N: 2}},
		a1: Ops{{N: 1}, {S: "tag"}, {N: 5}},
		b1: Ops{{N: 4}, {S: "tag"}, {N: 2}},
	},
	{
		a:  Ops{{N: 1}, {N: -2}},
		b:  Ops{{N: 2}, {N: -1}},
		a1: Ops{{N: 1}, {N: -1}},
		b1: Ops{{N: 1}},
	},
	{
		a:  Ops{{N: 2}, {N: -1}},
		b:  Ops{{N: 1}, {N: -2}},
		a1: Ops{{N: 1}},
		b1: Ops{{N: 1}, {N: -1}},
	},
}

func TestOpsTransform(t *testing.T) {
	for _, c := range transformTests {
		a1, b1, err := Transform(c.a, c.b)
		if err != nil {
			t.Error(err)
		}
		if !a1.Equal(c.a1) {
			t.Errorf("expected %v got %v", c.a1, a1)
		}
		if !b1.Equal(c.b1) {
			t.Errorf("expected %v got %v", c.b1, b1)
		}
	}
}
<|endoftext|>