<commit_before>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-id3"
	"github.com/dustin/goexif/exif"
)

var uploadWg = sync.WaitGroup{}

var uploadFlags = flag.NewFlagSet("upload", flag.ExitOnError)
var uploadVerbose = uploadFlags.Bool("v", false, "Verbose")
var uploadDelete = uploadFlags.Bool("delete", false,
	"Delete locally missing items")
var uploadMeta = uploadFlags.Bool("meta", false,
	"Store meta info in userData for items")
var uploadWorkers = uploadFlags.Int("workers", 4, "Number of upload workers")
var uploadRevs = uploadFlags.Int("revs", 0,
	"Number of old revisions to keep (-1 == all)")
var uploadIgnore = uploadFlags.String("ignore", "",
	"Path to ignore file")
var uploadRevsSet = false

var quotingReplacer = strings.NewReplacer("%", "%25",
	"?", "%3f",
	" ", "%20",
	"#", "%23")

type uploadOpType uint8

const (
	uploadFileOp = uploadOpType(iota)
	removeFileOp
	removeRecurseOp
)

func (u uploadOpType) String() string {
	switch u {
	case uploadFileOp:
		return "upload file"
	case removeFileOp:
		return "remove file"
	case removeRecurseOp:
		return "remove (recursive) file"
	}
	panic("unhandled op type")
}

type uploadReq struct {
	src        string
	dest       string
	op         uploadOpType
	remoteHash string
}

func recognizeTypeByName(n, def string) string {
	byname := mime.TypeByExtension(n)
	switch {
	case byname != "":
		return byname
	case strings.HasSuffix(n, ".js"):
		return "application/javascript"
	case strings.HasSuffix(n, ".css"):
		return "text/css"
	}
	return def
}

func processMP3Meta(src, dest string) (interface{}, error) {
	f, err := os.Open(src)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ifile := id3.Read(f)
	return ifile, nil
}

func processEXIFMeta(src, dest string) (interface{}, error) {
	f, err := os.Open(src)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return exif.Decode(f)
}

func processMeta(src, dest string) error {
	var data interface{}
	var err error

	switch filepath.Ext(strings.ToLower(src)) {
	case ".mp3":
		data, err = processMP3Meta(src, dest)
	case ".jpg", ".jpeg":
		data, err = processEXIFMeta(src, dest)
	default:
		log.Printf("No meta info for %#v",
			filepath.Ext(strings.ToLower(src)))
	}

	if err != nil || data == nil {
		return err
	}

	b, err := json.Marshal(data)
	if err != nil {
		return err
	}

	udest, err := url.Parse(dest)
	if err != nil {
		return err
	}
	udest.Path = "/.cbfs/meta" + udest.Path

	preq, err := http.NewRequest("PUT", udest.String(), bytes.NewReader(b))
	if err != nil {
		return err
	}

	preq.Header.Set("Content-Type", "application/json")

	if *uploadVerbose {
		log.Printf("Uploading meta info to %v", udest)
	}

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("HTTP Error:  %v", resp.Status)
	}

	return nil
}

func uploadFile(src, dest, localHash string) error {
	if *uploadVerbose {
		log.Printf("Uploading %v -> %v (%v)", src, dest, localHash)
	}

	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	someBytes := make([]byte, 512)
	n, err := f.Read(someBytes)
	if err != nil && err != io.EOF {
		return err
	}
	someBytes = someBytes[:n]

	length, err := f.Seek(0, 2)
	if err != nil {
		return err
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}

	preq, err := http.NewRequest("PUT", dest, f)
	if err != nil {
		return err
	}
	if uploadRevsSet {
		preq.Header.Set("X-CBFS-KeepRevs", strconv.Itoa(*uploadRevs))
	}

	ctype := http.DetectContentType(someBytes)
	if strings.HasPrefix(ctype, "text/plain") ||
		strings.HasPrefix(ctype, "application/octet-stream") {
		ctype = recognizeTypeByName(src, ctype)
	}

	preq.Header.Set("Content-Length", strconv.FormatInt(length, 10))
	preq.Header.Set("Content-Type", ctype)
	preq.Header.Set("X-CBFS-Hash", localHash)

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		r, _ := ioutil.ReadAll(resp.Body)
		return fmt.Errorf("HTTP Error:  %v: %s", resp.Status, r)
	}

	if *uploadMeta {
		err = processMeta(src, dest)
		if err != nil {
			log.Printf("Error processing meta info: %v", err)
		}
	}

	return nil
}

// This is very similar to rm's version, but uses different channel
// signaling.
func uploadRmDir(baseUrl string) ([]string, error) {
	if *uploadVerbose {
		log.Printf("Removing directory: %v", baseUrl)
	}
	r := quotingReplacer
	for strings.HasSuffix(baseUrl, "/") {
		baseUrl = baseUrl[:len(baseUrl)-1]
	}

	listing, err := listOrEmpty(baseUrl)
	for err != nil {
		return []string{}, err
	}
	for fn := range listing.Files {
		if *uploadVerbose {
			log.Printf("Removing file %v/%v", baseUrl, fn)
		}
		err = rmFile(baseUrl + "/" + r.Replace(fn))
		if err != nil {
			return []string{}, err
		}
	}
	children := make([]string, 0, len(listing.Dirs))
	for dn := range listing.Dirs {
		children = append(children, baseUrl+"/"+r.Replace(dn))
	}
	log.Printf("Children: %v", children)
	return children, nil
}

func uploadRmDashR(d string) error {
	if *uploadVerbose {
		log.Printf("Removing (recursively) %v", d)
	}

	children, err := uploadRmDir(d)
	if err == nil && len(children) > 0 {
		for _, child := range children {
			err = uploadRmDashR(child)
		}
	}
	return err
}

func localHash(fn string) string {
	f, err := os.Open(fn)
	if err != nil {
		return "unknown"
	}
	defer f.Close()

	h := sha1.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "unknown"
	}

	return hex.EncodeToString(h.Sum([]byte{}))
}

func uploadWorker(ch chan uploadReq) {
	defer uploadWg.Done()
	for req := range ch {
		retries := 0
		done := false
		for !done {
			var err error
			switch req.op {
			case uploadFileOp:
				lh := localHash(req.src)
				if req.remoteHash == "" {
					err = uploadFile(req.src, req.dest, lh)
				} else {
					if lh != req.remoteHash {
						if *uploadVerbose {
							log.Printf("%v has changed, reupping",
								req.src)
						}
						err = uploadFile(req.src, req.dest, lh)
					}
				}
			case removeFileOp:
				if *uploadVerbose {
					log.Printf("Removing file %v", req.dest)
				}
				err = rmFile(req.dest)
			case removeRecurseOp:
				err = uploadRmDashR(req.dest)
			default:
				log.Fatalf("Unhandled case")
			}
			if err != nil {
				if retries < 3 {
					retries++
					log.Printf("Error in %v: %v... retrying",
						req.op, err)
					time.Sleep(time.Duration(retries) * time.Second)
				} else {
					log.Printf("Error in %v %v: %v",
						req.op, req.src, err)
					done = true
				}
			} else {
				done = true
			}
		}
	}
}

func syncPath(path, dest string, info os.FileInfo, ch chan<- uploadReq) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	children, err := f.Readdir(0)
	if err != nil {
		return err
	}

	dest = quotingReplacer.Replace(dest)

	retries := 3
	serverListing, err := listOrEmpty(dest)
	for err != nil && retries > 0 {
		serverListing, err = listOrEmpty(dest)
		time.Sleep(time.Second)
		retries--
	}
	if err != nil {
		return err
	}

	localNames := map[string]os.FileInfo{}
	for _, c := range children {
		switch c.Mode() & os.ModeType {
		case os.ModeCharDevice, os.ModeDevice,
			os.ModeNamedPipe, os.ModeSocket, os.ModeSymlink:
			if *uploadVerbose {
				log.Printf("Ignoring special file: %v - %v",
					filepath.Join(path, c.Name()), c.Mode())
			}
		default:
			fullPath := filepath.Join(path, c.Name())
			if !isIgnored(fullPath) {
				localNames[c.Name()] = c
			} else {
				if *uploadVerbose {
					log.Printf("Ignoring %v", fullPath)
				}
			}
		}
	}

	remoteNames := map[string]bool{}
	for n := range serverListing.Files {
		if n != "" {
			remoteNames[n] = true
		}
	}
	for n := range serverListing.Dirs {
		if n != "" {
			remoteNames[n] = true
		}
	}

	// Keeping it short
	r := quotingReplacer

	missingUpstream := []string{}
	for n, fi := range localNames {
		if !(fi.IsDir() || remoteNames[n]) {
			missingUpstream = append(missingUpstream, n)
		} else if !fi.IsDir() {
			if ri, ok := serverListing.Files[n]; ok {
				ch <- uploadReq{filepath.Join(path, n),
					dest + "/" + r.Replace(n), uploadFileOp, ri.OID}
			}
		}
	}

	toRm := []string{}
	for n := range remoteNames {
		if _, ok := localNames[n]; !ok {
			toRm = append(toRm, n)
		}
	}

	if len(missingUpstream) > 0 {
		for _, m := range missingUpstream {
			ch <- uploadReq{filepath.Join(path, m),
				dest + "/" + r.Replace(m), uploadFileOp, ""}
		}
	}

	if *uploadDelete && len(toRm) > 0 {
		for _, m := range toRm {
			ch <- uploadReq{"", dest + "/" + r.Replace(m), removeFileOp, ""}
			ch <- uploadReq{"", dest + "/" + r.Replace(m), removeRecurseOp, ""}
		}
	}

	return nil
}

func syncUp(src, u string, ch chan<- uploadReq) {
	for strings.HasSuffix(u, "/") {
		u = u[:len(u)-1]
	}
	for strings.HasSuffix(src, "/") {
		src = src[:len(src)-1]
	}

	err := filepath.Walk(src,
		func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				if isIgnored(path) {
					log.Printf("Skipping directory %v",
						path)
					return filepath.SkipDir
				}
				shortPath := path[len(src):]
				err = syncPath(path, u+shortPath, info, ch)
			}
			return err
		})
	if err != nil {
		log.Fatalf("Traversal error: %v", err)
	}
}

func uploadCommand(u string, args []string) {
	uploadFlags.Parse(args)

	uploadFlags.Visit(func(f *flag.Flag) {
		if f.Name == "revs" {
			uploadRevsSet = true
		}
	})

	if uploadFlags.NArg() < 2 {
		log.Fatalf("src and dest required")
	}

	if *uploadIgnore != "" {
		err := loadIgnorePatternsFromFile(*uploadIgnore)
		if err != nil {
			log.Fatalf("Error loading ignores: %v", err)
		}
	}

	du := relativeUrl(u, uploadFlags.Arg(1))

	fi, err := os.Stat(uploadFlags.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	if fi.IsDir() {
		ch := make(chan uploadReq, 1000)

		for i := 0; i < *uploadWorkers; i++ {
			uploadWg.Add(1)
			go uploadWorker(ch)
		}

		start := time.Now()
		syncUp(uploadFlags.Arg(0), du, ch)

		close(ch)
		log.Printf("Finished traversal in %v", time.Since(start))
		uploadWg.Wait()
		log.Printf("Finished sync in %v", time.Since(start))
	} else {
		err = uploadFile(uploadFlags.Arg(0), du,
			localHash(uploadFlags.Arg(0)))
		if err != nil {
			log.Fatalf("Error uploading file: %v", err)
		}
	}
}
<commit_msg>Cleaned up some upload logging.<commit_after>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-id3"
	"github.com/dustin/goexif/exif"
)

var uploadWg = sync.WaitGroup{}

var uploadFlags = flag.NewFlagSet("upload", flag.ExitOnError)
var uploadVerbose = uploadFlags.Bool("v", false, "Verbose")
var uploadDelete = uploadFlags.Bool("delete", false,
	"Delete locally missing items")
var uploadMeta = uploadFlags.Bool("meta", false,
	"Store meta info in userData for items")
var uploadWorkers = uploadFlags.Int("workers", 4, "Number of upload workers")
var uploadRevs = uploadFlags.Int("revs", 0,
	"Number of old revisions to keep (-1 == all)")
var uploadIgnore = uploadFlags.String("ignore", "",
	"Path to ignore file")
var uploadRevsSet = false

var quotingReplacer = strings.NewReplacer("%", "%25",
	"?", "%3f",
	" ", "%20",
	"#", "%23")

type uploadOpType uint8

const (
	uploadFileOp = uploadOpType(iota)
	removeFileOp
	removeRecurseOp
)

func (u uploadOpType) String() string {
	switch u {
	case uploadFileOp:
		return "upload file"
	case removeFileOp:
		return "remove file"
	case removeRecurseOp:
		return "remove (recursive) file"
	}
	panic("unhandled op type")
}

type uploadReq struct {
	src        string
	dest       string
	op         uploadOpType
	remoteHash string
}

func recognizeTypeByName(n, def string) string {
	byname := mime.TypeByExtension(n)
	switch {
	case byname != "":
		return byname
	case strings.HasSuffix(n, ".js"):
		return "application/javascript"
	case strings.HasSuffix(n, ".css"):
		return "text/css"
	}
	return def
}

func processMP3Meta(src, dest string) (interface{}, error) {
	f, err := os.Open(src)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ifile := id3.Read(f)
	return ifile, nil
}

func processEXIFMeta(src, dest string) (interface{}, error) {
	f, err := os.Open(src)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return exif.Decode(f)
}

func processMeta(src, dest string) error {
	var data interface{}
	var err error

	switch filepath.Ext(strings.ToLower(src)) {
	case ".mp3":
		data, err = processMP3Meta(src, dest)
	case ".jpg", ".jpeg":
		data, err = processEXIFMeta(src, dest)
	default:
		log.Printf("No meta info for %#v",
			filepath.Ext(strings.ToLower(src)))
	}

	if err != nil || data == nil {
		return err
	}

	b, err := json.Marshal(data)
	if err != nil {
		return err
	}

	udest, err := url.Parse(dest)
	if err != nil {
		return err
	}
	udest.Path = "/.cbfs/meta" + udest.Path

	preq, err := http.NewRequest("PUT", udest.String(), bytes.NewReader(b))
	if err != nil {
		return err
	}

	preq.Header.Set("Content-Type", "application/json")

	if *uploadVerbose {
		log.Printf("Uploading meta info to %v", udest)
	}

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("HTTP Error:  %v", resp.Status)
	}

	return nil
}

func uploadFile(src, dest, localHash string) error {
	if *uploadVerbose {
		log.Printf("Uploading %v -> %v (%v)", src, dest, localHash)
	}

	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	someBytes := make([]byte, 512)
	n, err := f.Read(someBytes)
	if err != nil && err != io.EOF {
		return err
	}
	someBytes = someBytes[:n]

	length, err := f.Seek(0, 2)
	if err != nil {
		return err
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}

	preq, err := http.NewRequest("PUT", dest, f)
	if err != nil {
		return err
	}
	if uploadRevsSet {
		preq.Header.Set("X-CBFS-KeepRevs", strconv.Itoa(*uploadRevs))
	}

	ctype := http.DetectContentType(someBytes)
	if strings.HasPrefix(ctype, "text/plain") ||
		strings.HasPrefix(ctype, "application/octet-stream") {
		ctype = recognizeTypeByName(src, ctype)
	}

	preq.Header.Set("Content-Length", strconv.FormatInt(length, 10))
	preq.Header.Set("Content-Type", ctype)
	preq.Header.Set("X-CBFS-Hash", localHash)

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		r, _ := ioutil.ReadAll(resp.Body)
		return fmt.Errorf("HTTP Error:  %v: %s", resp.Status, r)
	}

	if *uploadMeta {
		err = processMeta(src, dest)
		if err != nil {
			log.Printf("Error processing meta info: %v", err)
		}
	}

	return nil
}

// This is very similar to rm's version, but uses different channel
// signaling.
func uploadRmDir(baseUrl string) ([]string, error) {
	if *uploadVerbose {
		log.Printf("Removing directory: %v", baseUrl)
	}
	r := quotingReplacer
	for strings.HasSuffix(baseUrl, "/") {
		baseUrl = baseUrl[:len(baseUrl)-1]
	}

	listing, err := listOrEmpty(baseUrl)
	for err != nil {
		return []string{}, err
	}
	for fn := range listing.Files {
		if *uploadVerbose {
			log.Printf("Removing file %v/%v", baseUrl, fn)
		}
		err = rmFile(baseUrl + "/" + r.Replace(fn))
		if err != nil {
			return []string{}, err
		}
	}
	children := make([]string, 0, len(listing.Dirs))
	for dn := range listing.Dirs {
		children = append(children, baseUrl+"/"+r.Replace(dn))
	}
	return children, nil
}

func uploadRmDashR(d string) error {
	if *uploadVerbose {
		log.Printf("Removing (recursively) %v", d)
	}

	children, err := uploadRmDir(d)
	if err == nil && len(children) > 0 {
		for _, child := range children {
			err = uploadRmDashR(child)
		}
	}
	return err
}

func localHash(fn string) string {
	f, err := os.Open(fn)
	if err != nil {
		return "unknown"
	}
	defer f.Close()

	h := sha1.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "unknown"
	}

	return hex.EncodeToString(h.Sum([]byte{}))
}

func uploadWorker(ch chan uploadReq) {
	defer uploadWg.Done()
	for req := range ch {
		retries := 0
		done := false
		for !done {
			var err error
			switch req.op {
			case uploadFileOp:
				lh := localHash(req.src)
				if req.remoteHash == "" {
					err = uploadFile(req.src, req.dest, lh)
				} else {
					if lh != req.remoteHash {
						if *uploadVerbose {
							log.Printf("%v has changed, reupping",
								req.src)
						}
						err = uploadFile(req.src, req.dest, lh)
					}
				}
			case removeFileOp:
				if *uploadVerbose {
					log.Printf("Removing file %v", req.dest)
				}
				err = rmFile(req.dest)
			case removeRecurseOp:
				err = uploadRmDashR(req.dest)
			default:
				log.Fatalf("Unhandled case")
			}
			if err != nil {
				if retries < 3 {
					retries++
					log.Printf("Error in %v: %v... retrying",
						req.op, err)
					time.Sleep(time.Duration(retries) * time.Second)
				} else {
					log.Printf("Error in %v %v: %v",
						req.op, req.src, err)
					done = true
				}
			} else {
				done = true
			}
		}
	}
}

func syncPath(path, dest string, info os.FileInfo, ch chan<- uploadReq) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	children, err := f.Readdir(0)
	if err != nil {
		return err
	}

	dest = quotingReplacer.Replace(dest)

	retries := 3
	serverListing, err := listOrEmpty(dest)
	for err != nil && retries > 0 {
		serverListing, err = listOrEmpty(dest)
		time.Sleep(time.Second)
		retries--
	}
	if err != nil {
		return err
	}

	localNames := map[string]os.FileInfo{}
	for _, c := range children {
		switch c.Mode() & os.ModeType {
		case os.ModeCharDevice, os.ModeDevice,
			os.ModeNamedPipe, os.ModeSocket, os.ModeSymlink:
			if *uploadVerbose {
				log.Printf("Ignoring special file: %v - %v",
					filepath.Join(path, c.Name()), c.Mode())
			}
		default:
			fullPath := filepath.Join(path, c.Name())
			if !isIgnored(fullPath) {
				localNames[c.Name()] = c
			} else {
				if *uploadVerbose {
					log.Printf("Ignoring %v", fullPath)
				}
			}
		}
	}

	remoteNames := map[string]bool{}
	for n := range serverListing.Files {
		if n != "" {
			remoteNames[n] = true
		}
	}
	for n := range serverListing.Dirs {
		if n != "" {
			remoteNames[n] = true
		}
	}

	// Keeping it short
	r := quotingReplacer

	missingUpstream := []string{}
	for n, fi := range localNames {
		if !(fi.IsDir() || remoteNames[n]) {
			missingUpstream = append(missingUpstream, n)
		} else if !fi.IsDir() {
			if ri, ok := serverListing.Files[n]; ok {
				ch <- uploadReq{filepath.Join(path, n),
					dest + "/" + r.Replace(n), uploadFileOp, ri.OID}
			}
		}
	}

	toRm := []string{}
	for n := range remoteNames {
		if _, ok := localNames[n]; !ok {
			toRm = append(toRm, n)
		}
	}

	if len(missingUpstream) > 0 {
		for _, m := range missingUpstream {
			ch <- uploadReq{filepath.Join(path, m),
				dest + "/" + r.Replace(m), uploadFileOp, ""}
		}
	}

	if *uploadDelete && len(toRm) > 0 {
		for _, m := range toRm {
			ch <- uploadReq{"", dest + "/" + r.Replace(m), removeFileOp, ""}
			ch <- uploadReq{"", dest + "/" + r.Replace(m), removeRecurseOp, ""}
		}
	}

	return nil
}

func syncUp(src, u string, ch chan<- uploadReq) {
	for strings.HasSuffix(u, "/") {
		u = u[:len(u)-1]
	}
	for strings.HasSuffix(src, "/") {
		src = src[:len(src)-1]
	}

	err := filepath.Walk(src,
		func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				if isIgnored(path) {
					if *uploadVerbose {
						log.Printf("Skipping dir %v",
							path)
					}
					return filepath.SkipDir
				}
				shortPath := path[len(src):]
				err = syncPath(path, u+shortPath, info, ch)
			}
			return err
		})
	if err != nil {
		log.Fatalf("Traversal error: %v", err)
	}
}

func uploadCommand(u string, args []string) {
	uploadFlags.Parse(args)

	uploadFlags.Visit(func(f *flag.Flag) {
		if f.Name == "revs" {
			uploadRevsSet = true
		}
	})

	if uploadFlags.NArg() < 2 {
		log.Fatalf("src and dest required")
	}

	if *uploadIgnore != "" {
		err := loadIgnorePatternsFromFile(*uploadIgnore)
		if err != nil {
			log.Fatalf("Error loading ignores: %v", err)
		}
	}

	du := relativeUrl(u, uploadFlags.Arg(1))

	fi, err := os.Stat(uploadFlags.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	if fi.IsDir() {
		ch := make(chan uploadReq, 1000)

		for i := 0; i < *uploadWorkers; i++ {
			uploadWg.Add(1)
			go uploadWorker(ch)
		}

		start := time.Now()
		syncUp(uploadFlags.Arg(0), du, ch)

		close(ch)
		log.Printf("Finished traversal in %v", time.Since(start))
		uploadWg.Wait()
		log.Printf("Finished sync in %v", time.Since(start))
	} else {
		err = uploadFile(uploadFlags.Arg(0), du,
			localHash(uploadFlags.Arg(0)))
		if err != nil {
			log.Fatalf("Error uploading file: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"time"
)

func main() {
	start := time.Now()
	ch := make(chan string)
	for _, url := range os.Args[1:] {
		go fetch(url, ch)
	}
	for range os.Args[1:] {
		fmt.Println(<-ch)
	}
	fmt.Printf("%.2fs elapsed\n", time.Since(start).Seconds())
}

func fetch(url string, ch chan<- string) {
	start := time.Now()
	resp, err := http.Get(url)
	if err != nil {
		ch <- fmt.Sprint(err)
		return
	}
	nbytes, err := io.Copy(ioutil.Discard, resp.Body)
	resp.Body.Close()
	if err != nil {
		ch <- fmt.Sprintf("while reading %s: %v", url, err)
		return
	}
	secs := time.Since(start).Seconds()
	ch <- fmt.Sprintf("%.2fs  %7d  %s", secs, nbytes, url)
}
<commit_msg>Add scheme if missing.<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"
)

func main() {
	start := time.Now()
	ch := make(chan string)
	for _, url := range os.Args[1:] {
		if !strings.HasPrefix(url, "http://") {
			url = "http://" + url
		}
		go fetch(url, ch)
	}
	for range os.Args[1:] {
		fmt.Println(<-ch)
	}
	fmt.Printf("%.2fs elapsed\n", time.Since(start).Seconds())
}

func fetch(url string, ch chan<- string) {
	start := time.Now()
	resp, err := http.Get(url)
	if err != nil {
		ch <- fmt.Sprint(err)
		return
	}
	nbytes, err := io.Copy(ioutil.Discard, resp.Body)
	resp.Body.Close()
	if err != nil {
		ch <- fmt.Sprintf("while reading %s: %v", url, err)
		return
	}
	secs := time.Since(start).Seconds()
	ch <- fmt.Sprintf("%.2fs  %7d  %s", secs, nbytes, url)
}
<|endoftext|>


<commit_before>package check

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

var (
	referenceServer = "8.8.8.8"
	testChecker     = &Checker{ReferenceServer: referenceServer}
)

func resolve(server, query string) (records stringSet, authenticated bool, err error) {
	return testChecker.resolve(server, query)
}

func TestExistent(t *testing.T) {
	assert := assert.New(t)
	result, _, err := resolve(referenceServer, "example.com")

	assert.Nil(err)
	assert.Len(result, 1)
}

func TestNotExistent(t *testing.T) {
	assert := assert.New(t)
	result, authenticated, err := resolve(referenceServer, "xxx.example.com")

	assert.Nil(err)
	assert.False(authenticated)
	assert.Len(result, 0)
}

func TestAuthenticated(t *testing.T) {
	assert := assert.New(t)
	result, authenticated, err := resolve(referenceServer, "verisignlabs.com")

	assert.Nil(err)
	assert.True(authenticated)
	assert.Len(result, 1)
}

func TestUnreachable(t *testing.T) {
	assert := assert.New(t)
	_, _, err := resolve("127.1.2.3", "example.com")

	assert.EqualError(err, "connection refused")
}

func TestPtrName(t *testing.T) {
	assert := assert.New(t)
	result := testChecker.ptrName("8.8.8.8")

	assert.Equal("dns.google.", result)
}

func TestVersion(t *testing.T) {
	assert := assert.New(t)
	result := testChecker.version("82.96.65.2")

	assert.Equal("Make my day", result)
}
<commit_msg>Fix test<commit_after>package check

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

var (
	referenceServer = "8.8.8.8"
	testChecker     = &Checker{ReferenceServer: referenceServer}
)

func resolve(server, query string) (records stringSet, authenticated bool, err error) {
	return testChecker.resolve(server, query)
}

func TestExistent(t *testing.T) {
	assert := assert.New(t)
	result, _, err := resolve(referenceServer, "example.com")

	assert.Nil(err)
	assert.Len(result, 1)
}

func TestNotExistent(t *testing.T) {
	assert := assert.New(t)
	result, authenticated, err := resolve(referenceServer, "xxx.example.com")

	assert.Nil(err)
	assert.False(authenticated)
	assert.Len(result, 0)
}

func TestAuthenticated(t *testing.T) {
	assert := assert.New(t)
	result, authenticated, err := resolve(referenceServer, "verisignlabs.com")

	assert.Nil(err)
	assert.True(authenticated)
	assert.GreaterOrEqual(len(result), 1)
}

func TestUnreachable(t *testing.T) {
	assert := assert.New(t)
	_, _, err := resolve("127.1.2.3", "example.com")

	assert.EqualError(err, "connection refused")
}

func TestPtrName(t *testing.T) {
	assert := assert.New(t)
	result := testChecker.ptrName("8.8.8.8")

	assert.Equal("dns.google.", result)
}

func TestVersion(t *testing.T) {
	assert := assert.New(t)
	result := testChecker.version("82.96.65.2")

	assert.Equal("Make my day", result)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/newrelic/go_nagios"
	. "github.com/smartystreets/goconvey/convey"
)

var devicemapperInfoJsonFromApi []byte = []byte(
	`{
		"Driver": "devicemapper",
		"DriverStatus": [
			["Data Space Used", "20.0 mb"],
			["Data Space Total", "1000.0 mb"],
			["Metadata Space Used", "15.0 mb"],
			["Metadata Space Total", "200.0 mb"]
		]
	}`,
)

var aufsInfoJsonFromApi []byte = []byte(
	`
	{
		"Containers": 0,
		"Debug": 0,
		"Driver": "aufs",
		"DriverStatus": [
			["Root Dir","/data/docker/aufs"],
			["Dirs","0"]
		],
		"ExecutionDriver": "native-0.2",
		"IPv4Forwarding": 1,
		"Images": 0,
		"IndexServerAddress": "https://index.docker.io/v1/",
		"InitPath": "/usr/bin/docker",
		"InitSha1": "",
		"KernelVersion": "3.8.0-35-generic",
		"MemoryLimit": 1,
		"NEventsListener": 0,
		"NFd": 11,
		"NGoroutines": 11,
		"Sockets": [
			"tcp://0.0.0.0:4243",
			"tcp://0.0.0.0:2375",
			"unix:///var/run/docker.sock"
		],
		"SwapLimit":1
	}`,
)

var containersJsonFromApi []byte = []byte(
	`[
	  {
	    "Command": "script/run ",
	    "Created": 1399681210,
	    "Id": "ded464bf7dfb978b6b101c289a06b59a1c64435b3b7e70c97e6876ceb2a9a159",
	    "Image": "testing:b969c9317cc60c389162cbdb2999806ef9b9666b",
	    "Names": [
	      "/insane_franklin"
	    ],
	    "Ports": [
	      {
	        "IP": "0.0.0.0",
	        "PrivatePort": 80,
	        "PublicPort": 8485,
	        "Type": "tcp"
	      }
	    ],
	    "Status": "Up 3 days"
	  },
	  {
	    "Command": "script/run ",
	    "Created": 1399681124,
	    "Id": "a64bba6cd0dbfb9b1bc1880f38d138a1c69a929853dcfca72314d1242e00017c",
	    "Image": "real:b969c9317cc60c389162cbdb2999806ef9b9666b",
	    "Names": [
	      "/sad_ptolemy"
	    ],
	    "Ports": [
	      {
	        "IP": "0.0.0.0",
	        "PrivatePort": 80,
	        "PublicPort": 80,
	        "Type": "tcp"
	      }
	    ],
	    "Status": "Exit 0"
	  },
	  {
	    "Command": "script/run ",
	    "Created": 1399681124,
	    "Id": "2938378cd0dbfb9b1bc1880f38d138a1c69a929853dcfca72314d1242e00017c",
	    "Image": "busted:b969c9317cc60c389162cbdb2999806ef9b9666b",
	    "Names": [
	      "/happy_galileo"
	    ],
	    "Ports": [
	      {
	        "IP": "0.0.0.0",
	        "PrivatePort": 80,
	        "PublicPort": 8999,
	        "Type": "tcp"
	      }
	    ],
	    "Status": "Ghost"
	  }
	]`,
)

type stubFetcher struct{}

func (fetcher stubFetcher) Fetch(url string) ([]byte, error) {
	if strings.Contains(url, "/info") {
		return devicemapperInfoJsonFromApi, nil
	}

	if strings.Contains(url, "/containers") {
		return containersJsonFromApi, nil
	}

	return nil, errors.New("Don't recognize URL: " + url)
}

func TestFloat64String(t *testing.T) {
	Convey("Converts a float to a formatted string with no decimals", t, func() {
		So(float64String(1.2), ShouldEqual, "1")
	})
}

func TestMegabytesFloat64(t *testing.T) {
	Convey("Extracts the float from a Docker megabytes measurement string", t, func() {
		result, _ := megabytesFloat64("1024.05 Mb")
		So(result, ShouldEqual, 1024.05)
	})

	Convey("Returns an error when not parseable", t, func() {
		_, err := megabytesFloat64("1024.05mb")
		So(err.Error(), ShouldContainSubstring, "invalid syntax")
	})
}

func TestFindDriverStatus(t *testing.T) {
	driverStatuses := [][]string{
		[]string{"Key", "Value"},
		[]string{"Key2", "Value2"},
	}

	Convey("Looks up values from a slice by the first element", t, func() {
		So(findDriverStatus("Key", driverStatuses), ShouldEqual, "Value")
		So(findDriverStatus("Key2", driverStatuses), ShouldEqual, "Value2")
	})

	Convey("Returns empty on failure", t, func() {
		So(findDriverStatus("KeyFoo", driverStatuses), ShouldEqual, "")
	})
}

func TestPopulateDriverInfo(t *testing.T) {
	Convey("Correctly parses devicemapper /info JSON and populates the DockerInfo", t, func() {
		var info DockerInfo
		err := populateInfo(devicemapperInfoJsonFromApi, &info)

		So(err, ShouldBeNil)
		So(info.Driver, ShouldEqual, "devicemapper")
		So(info.DataSpaceUsed, ShouldEqual, 20.0)
	})

	Convey("Correctly parses AUFS /info JSON and populates the DockerInfo", t, func() {
		var info DockerInfo
		err := populateInfo(aufsInfoJsonFromApi, &info)

		So(err, ShouldBeNil)
		So(info.Driver, ShouldEqual, "aufs")
	})
}

func TestCheckRunningContainers(t *testing.T) {
	Convey("Searches a JSON blob to find an image with a specified tag", t, func() {
		running, _, err := checkRunningContainers(containersJsonFromApi, &CliOpts{ImageId: "testing"})

		So(err, ShouldBeNil)
		So(running, ShouldBeTrue)
	})

	Convey("Correctly identifies when the tag is missing", t, func() {
		running, _, err := checkRunningContainers(containersJsonFromApi, &CliOpts{ImageId: "Shakespeare"})

		So(err, ShouldBeNil)
		So(running, ShouldBeFalse)
	})

	Convey("Bubbles up errors from the Json library", t, func() {
		running, _, err := checkRunningContainers([]byte("-"), &CliOpts{ImageId: "Shakespeare"})

		So(err, ShouldNotBeNil)
		So(running, ShouldBeFalse)
	})

	Convey("Identifies ghost containers", t, func() {
		_, ghosts, err := checkRunningContainers(containersJsonFromApi, &CliOpts{ImageId: "Shakespeare"})

		So(err, ShouldBeNil)
		So(ghosts, ShouldEqual, 1)
	})
}

func TestFetchInfo(t *testing.T) {
	Convey("Can fetch info using a Fetcher and populate a DockerInfo", t, func() {
		var info DockerInfo
		var stub stubFetcher
		err := fetchInfo(stub, CliOpts{}, &info)

		So(err, ShouldBeNil)
		So(info.DataSpaceUsed, ShouldEqual, 20.0)
	})

	Convey("Populates the ImageIsRunning field when told to by CLI flags", t, func() {
		var info DockerInfo
		var stub stubFetcher
		err := fetchInfo(stub, CliOpts{ImageId: "testing"}, &info)

		So(err, ShouldBeNil)
		So(info.ImageIsRunning, ShouldBeTrue)
	})
}

func TestMapAlertStatuses(t *testing.T) {
	opts := CliOpts{
		CritMetaSpace: 6.0,
		WarnDataSpace: 5.0,
	}

	Convey("Given AUFS /info JSON, when handed a DockerInfo, don't break", t, func() {
		var info DockerInfo
		err := populateInfo(aufsInfoJsonFromApi, &info)

		So(err, ShouldBeNil)

		results := mapAlertStatuses(&info, &opts)
		So(len(results), ShouldEqual, 0)
	})

	Convey("Given devicemapper /info JSON, when handed a DockerInfo, returns a list of check results", t, func() {
		var info DockerInfo
		populateInfo(devicemapperInfoJsonFromApi, &info)
		results := mapAlertStatuses(&info, &opts)

		So(results[0], ShouldHaveTheSameNagiosStatusAs, &nagios.NagiosStatus{"Meta Space Used: 8%", nagios.NAGIOS_CRITICAL})
		So(results[2], ShouldHaveTheSameNagiosStatusAs, &nagios.NagiosStatus{"Meta Space Used: 8%", nagios.NAGIOS_WARNING})
	})

	Convey("Produces output that can properly be aggregated by Nagios", t, func() {
		var info DockerInfo
		populateInfo(devicemapperInfoJsonFromApi, &info)
		results := mapAlertStatuses(&info, &opts)

		status := &nagios.NagiosStatus{"Chaucer", nagios.NAGIOS_UNKNOWN}
		status.Aggregate(results)
		expected := &nagios.NagiosStatus{"Chaucer - Meta Space Used: 8% - Data Space Used: 2% - Meta Space Used: 8%", nagios.NAGIOS_UNKNOWN}

		So(status, ShouldHaveTheSameNagiosStatusAs, expected)
	})

	Convey("Correctly handles the exit status when ghosts are present", t, func() {
		var info DockerInfo
		var stub stubFetcher
		opts := CliOpts{
			CritMetaSpace: 100,
			CritDataSpace: 100,
			GhostsStatus:  2,
		}
		fetchInfo(stub, opts, &info)
		results := mapAlertStatuses(&info, &opts)

		expected := &nagios.NagiosStatus{"Ghost Containers: 1", nagios.NAGIOS_CRITICAL}
		So(results[2], ShouldHaveTheSameNagiosStatusAs, expected)
	})
}

func ShouldHaveTheSameNagiosStatusAs(actual interface{}, expected ...interface{}) string {
	wanted := expected[0].(*nagios.NagiosStatus)
	got := actual.(*nagios.NagiosStatus)

	if got.Value != wanted.Value || got.Message != wanted.Message {
		return "expected:\n" + fmt.Sprintf("%#v", wanted) + "\n\ngot:\n" + fmt.Sprintf("%#v", got)
	}

	return ""
}
<commit_msg>better fixture for AUFS.<commit_after>package main

import (
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/newrelic/go_nagios"
	. "github.com/smartystreets/goconvey/convey"
)

var devicemapperInfoJsonFromApi []byte = []byte(
	`{
		"Driver": "devicemapper",
		"DriverStatus": [
			["Data Space Used", "20.0 mb"],
			["Data Space Total", "1000.0 mb"],
			["Metadata Space Used", "15.0 mb"],
			["Metadata Space Total", "200.0 mb"]
		]
	}`,
)

var aufsInfoJsonFromApi []byte = []byte(
	`
	{
		"Containers": 0,
		"Debug": 0,
		"Driver": "aufs",
		"DriverStatus": [
			["Root Dir","/usr/local/lib/docker/aufs"],
			["Dirs","0"]
		],
		"ExecutionDriver": "native-0.2",
		"IPv4Forwarding": 1,
		"Images": 0,
		"IndexServerAddress": "https://index.docker.io/v1/",
		"InitPath": "/usr/bin/docker",
		"InitSha1": "",
		"KernelVersion": "3.8.0-35-generic",
		"MemoryLimit": 1,
		"NEventsListener": 0,
		"NFd": 11,
		"NGoroutines": 11,
		"Sockets": [
			"tcp://0.0.0.0:4243",
			"tcp://0.0.0.0:2375",
			"unix:///var/run/docker.sock"
		],
		"SwapLimit":1
	}`,
)

var containersJsonFromApi []byte = []byte(
	`[
	  {
	    "Command": "script/run ",
	    "Created": 1399681210,
	    "Id": "ded464bf7dfb978b6b101c289a06b59a1c64435b3b7e70c97e6876ceb2a9a159",
	    "Image": "testing:b969c9317cc60c389162cbdb2999806ef9b9666b",
	    "Names": [
	      "/insane_franklin"
	    ],
	    "Ports": [
	      {
	        "IP": "0.0.0.0",
	        "PrivatePort": 80,
	        "PublicPort": 8485,
	        "Type": "tcp"
	      }
	    ],
	    "Status": "Up 3 days"
	  },
	  {
	    "Command": "script/run ",
	    "Created": 1399681124,
	    "Id": "a64bba6cd0dbfb9b1bc1880f38d138a1c69a929853dcfca72314d1242e00017c",
	    "Image": "real:b969c9317cc60c389162cbdb2999806ef9b9666b",
	    "Names": [
	      "/sad_ptolemy"
	    ],
	    "Ports": [
	      {
	        "IP": "0.0.0.0",
	        "PrivatePort": 80,
	        "PublicPort": 80,
	        "Type": "tcp"
	      }
	    ],
	    "Status": "Exit 0"
	  },
	  {
	    "Command": "script/run ",
	    "Created": 1399681124,
	    "Id": "2938378cd0dbfb9b1bc1880f38d138a1c69a929853dcfca72314d1242e00017c",
	    "Image": "busted:b969c9317cc60c389162cbdb2999806ef9b9666b",
	    "Names": [
	      "/happy_galileo"
	    ],
	    "Ports": [
	      {
	        "IP": "0.0.0.0",
	        "PrivatePort": 80,
	        "PublicPort": 8999,
	        "Type": "tcp"
	      }
	    ],
	    "Status": "Ghost"
	  }
	]`,
)

type stubFetcher struct{}

func (fetcher stubFetcher) Fetch(url string) ([]byte, error) {
	if strings.Contains(url, "/info") {
		return devicemapperInfoJsonFromApi, nil
	}

	if strings.Contains(url, "/containers") {
		return containersJsonFromApi, nil
	}

	return nil, errors.New("Don't recognize URL: " + url)
}

func TestFloat64String(t *testing.T) {
	Convey("Converts a float to a formatted string with no decimals", t, func() {
		So(float64String(1.2), ShouldEqual, "1")
	})
}

func TestMegabytesFloat64(t *testing.T) {
	Convey("Extracts the float from a Docker megabytes measurement string", t, func() {
		result, _ := megabytesFloat64("1024.05 Mb")
		So(result, ShouldEqual, 1024.05)
	})

	Convey("Returns an error when not parseable", t, func() {
		_, err := megabytesFloat64("1024.05mb")
		So(err.Error(), ShouldContainSubstring, "invalid syntax")
	})
}

func TestFindDriverStatus(t *testing.T) {
	driverStatuses := [][]string{
		[]string{"Key", "Value"},
		[]string{"Key2", "Value2"},
	}

	Convey("Looks up values from a slice by the first element", t, func() {
		So(findDriverStatus("Key", driverStatuses), ShouldEqual, "Value")
		So(findDriverStatus("Key2", driverStatuses), ShouldEqual, "Value2")
	})

	Convey("Returns empty on failure", t, func() {
		So(findDriverStatus("KeyFoo", driverStatuses), ShouldEqual, "")
	})
}

func TestPopulateDriverInfo(t *testing.T) {
	Convey("Correctly parses devicemapper /info JSON and populates the DockerInfo", t, func() {
		var info DockerInfo
		err := populateInfo(devicemapperInfoJsonFromApi, &info)

		So(err, ShouldBeNil)
		So(info.Driver, ShouldEqual, "devicemapper")
		So(info.DataSpaceUsed, ShouldEqual, 20.0)
	})

	Convey("Correctly parses AUFS /info JSON and populates the DockerInfo", t, func() {
		var info DockerInfo
		err := populateInfo(aufsInfoJsonFromApi, &info)

		So(err, ShouldBeNil)
		So(info.Driver, ShouldEqual, "aufs")
	})
}

func TestCheckRunningContainers(t *testing.T) {
	Convey("Searches a JSON blob to find an image with a specified tag", t, func() {
		running, _, err := checkRunningContainers(containersJsonFromApi, &CliOpts{ImageId: "testing"})

		So(err, ShouldBeNil)
		So(running, ShouldBeTrue)
	})

	Convey("Correctly identifies when the tag is missing", t, func() {
		running, _, err := checkRunningContainers(containersJsonFromApi, &CliOpts{ImageId: "Shakespeare"})

		So(err, ShouldBeNil)
		So(running, ShouldBeFalse)
	})

	Convey("Bubbles up errors from the Json library", t, func() {
		running, _, err := checkRunningContainers([]byte("-"), &CliOpts{ImageId: "Shakespeare"})

		So(err, ShouldNotBeNil)
		So(running, ShouldBeFalse)
	})

	Convey("Identifies ghost containers", t, func() {
		_, ghosts, err := checkRunningContainers(containersJsonFromApi, &CliOpts{ImageId: "Shakespeare"})

		So(err, ShouldBeNil)
		So(ghosts, ShouldEqual, 1)
	})
}

func TestFetchInfo(t *testing.T) {
	Convey("Can fetch info using a Fetcher and populate a DockerInfo", t, func() {
		var info DockerInfo
		var stub stubFetcher
		err := fetchInfo(stub, CliOpts{}, &info)

		So(err, ShouldBeNil)
		So(info.DataSpaceUsed, ShouldEqual, 20.0)
	})

	Convey("Populates the ImageIsRunning field when told to by CLI flags", t, func() {
		var info DockerInfo
		var stub stubFetcher
		err := fetchInfo(stub, CliOpts{ImageId: "testing"}, &info)

		So(err, ShouldBeNil)
		So(info.ImageIsRunning, ShouldBeTrue)
	})
}

func TestMapAlertStatuses(t *testing.T) {
	opts := CliOpts{
		CritMetaSpace: 6.0,
		WarnDataSpace: 5.0,
	}

	Convey("Given AUFS /info JSON, when handed a DockerInfo, don't break", t, func() {
		var info DockerInfo
		err := populateInfo(aufsInfoJsonFromApi, &info)

		So(err, ShouldBeNil)

		results := mapAlertStatuses(&info, &opts)
		So(len(results), ShouldEqual, 0)
	})

	Convey("Given devicemapper /info JSON, when handed a DockerInfo, returns a list of check results", t, func() {
		var info DockerInfo
		populateInfo(devicemapperInfoJsonFromApi, &info)
		results := mapAlertStatuses(&info, &opts)

		So(results[0], ShouldHaveTheSameNagiosStatusAs, &nagios.NagiosStatus{"Meta Space Used: 8%", nagios.NAGIOS_CRITICAL})
		So(results[2], ShouldHaveTheSameNagiosStatusAs, &nagios.NagiosStatus{"Meta Space Used: 8%", nagios.NAGIOS_WARNING})
	})

	Convey("Produces output that can properly be aggregated by Nagios", t, func() {
		var info DockerInfo
		populateInfo(devicemapperInfoJsonFromApi, &info)
		results := mapAlertStatuses(&info, &opts)

		status := &nagios.NagiosStatus{"Chaucer", nagios.NAGIOS_UNKNOWN}
		status.Aggregate(results)
		expected := &nagios.NagiosStatus{"Chaucer - Meta Space Used: 8% - Data Space Used: 2% - Meta Space Used: 8%", nagios.NAGIOS_UNKNOWN}

		So(status, ShouldHaveTheSameNagiosStatusAs, expected)
	})

	Convey("Correctly handles the exit status when ghosts are present", t, func() {
		var info DockerInfo
		var stub stubFetcher
		opts := CliOpts{
			CritMetaSpace: 100,
			CritDataSpace: 100,
			GhostsStatus:  2,
		}
		fetchInfo(stub, opts, &info)
		results := mapAlertStatuses(&info, &opts)

		expected := &nagios.NagiosStatus{"Ghost Containers: 1", nagios.NAGIOS_CRITICAL}
		So(results[2], ShouldHaveTheSameNagiosStatusAs, expected)
	})
}

func ShouldHaveTheSameNagiosStatusAs(actual interface{}, expected ...interface{}) string {
	wanted := expected[0].(*nagios.NagiosStatus)
	got := actual.(*nagios.NagiosStatus)

	if got.Value != wanted.Value || got.Message != wanted.Message {
		return "expected:\n" + fmt.Sprintf("%#v", wanted) + "\n\ngot:\n" + fmt.Sprintf("%#v", got)
	}

	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"net"
	"regexp"
	"time"

	"github.com/jeffail/tunny"
	"github.com/oschwald/maxminddb-golang"
	"gopkg.in/vmihailenco/msgpack.v2"
)

// AnalyticsRecord encodes the details of a request
type AnalyticsRecord struct {
	Method        string
	Path          string
	RawPath       string
	ContentLength int64
	UserAgent     string
	Day           int
	Month         time.Month
	Year          int
	Hour          int
	ResponseCode  int
	APIKey        string
	TimeStamp     time.Time
	APIVersion    string
	APIName       string
	APIID         string
	OrgID         string
	OauthID       string
	RequestTime   int64
	RawRequest    string
	RawResponse   string
	IPAddress     string
	Geo           GeoData
	Tags          []string
	Alias         string
	TrackPath     bool
	ExpireAt      time.Time `bson:"expireAt" json:"expireAt"`
}

type GeoData struct {
	Country struct {
		ISOCode string `maxminddb:"iso_code"`
	} `maxminddb:"country"`

	City struct {
		GeoNameID uint              `maxminddb:"geoname_id"`
		Names     map[string]string `maxminddb:"names"`
	} `maxminddb:"city"`

	Location struct {
		Latitude  float64 `maxminddb:"latitude"`
		Longitude float64 `maxminddb:"longitude"`
		TimeZone  string  `maxminddb:"time_zone"`
	} `maxminddb:"location"`
}

const (
	ANALYTICS_KEYNAME = "tyk-system-analytics"
)

func (a *AnalyticsRecord) GetGeo(ipStr string) {
	if !config.AnalyticsConfig.EnableGeoIP {
		return
	}

	// Not great, tightly coupled
	if analytics.GeoIPDB == nil {
		return
	}

	// Sometimes it is empty, we can't look up mpty IP addresses
	if ipStr == "" {
		return
	}

	ip := net.ParseIP(ipStr)

	var record GeoData // Or any appropriate struct
	if err := analytics.GeoIPDB.Lookup(ip, &record); err != nil {
		log.Error("GeoIP Failure (not recorded): ", err)
		return
	}

	log.Debug("ISO Code: ", record.Country.ISOCode)
	log.Debug("City: ", record.City.Names["en"])
	log.Debug("Lat: ", record.Location.Latitude)
	log.Debug("Lon: ", record.Location.Longitude)
	log.Debug("TZ: ", record.Location.TimeZone)

	a.Geo = record
}

type NormaliseURLPatterns struct {
	UUIDs  *regexp.Regexp
	IDs    *regexp.Regexp
	Custom []*regexp.Regexp
}

func InitNormalisationPatterns() NormaliseURLPatterns {
	thesePatterns := NormaliseURLPatterns{}

	uuidPat := regexp.MustCompile(`[0-9a-fA-F]{8}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{12}`)
	numPat := regexp.MustCompile(`\/(\d+)`)

	custPats := []*regexp.Regexp{}
	for _, pattern := range config.AnalyticsConfig.NormaliseUrls.Custom {
		if patRe, err := regexp.Compile(pattern); err != nil {
			log.Error("failed to compile custom pattern: ", err)
		} else {
			custPats = append(custPats, patRe)
		}
	}

	thesePatterns.UUIDs = uuidPat
	thesePatterns.IDs = numPat
	thesePatterns.Custom = custPats

	return thesePatterns
}

func (a *AnalyticsRecord) NormalisePath() {
	if config.AnalyticsConfig.NormaliseUrls.Enabled {
		if config.AnalyticsConfig.NormaliseUrls.NormaliseUUIDs {
			a.Path = config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.UUIDs.ReplaceAllString(a.Path, "{uuid}")
		}
		if config.AnalyticsConfig.NormaliseUrls.NormaliseNumbers {
			a.Path = config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.IDs.ReplaceAllString(a.Path, "/{id}")
		}
		if len(config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.Custom) > 0 {
			for _, r := range config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.Custom {
				a.Path = r.ReplaceAllString(a.Path, "{var}")
			}
		}
	}

}

func (a *AnalyticsRecord) SetExpiry(expiresInSeconds int64) {
	expiry := time.Duration(expiresInSeconds) * time.Second
	if expiresInSeconds == 0 {
		// Expiry is set to 100 years
		expiry = (24 * time.Hour) * (365 * 100)
	}

	t := time.Now()
	t2 := t.Add(expiry)
	a.ExpireAt = t2
}

var AnalyticsPool *tunny.WorkPool

// RedisAnalyticsHandler will record analytics data to a redis back end
// as defined in the Config object
type RedisAnalyticsHandler struct {
	Store   *RedisClusterStorageManager
	Clean   Purger
	GeoIPDB *maxminddb.Reader
}

func (r *RedisAnalyticsHandler) Init() {
	if config.AnalyticsConfig.EnableGeoIP {
		go r.reloadDB()
	}

	analytics.Store.Connect()
	var err error

	ps := config.AnalyticsConfig.PoolSize
	if ps == 0 {
		ps = 50
	}

	AnalyticsPool, err = tunny.CreatePoolGeneric(ps).Open()
	if err != nil {
		log.Error("Failed to init analytics pool")
	}
}

func (r *RedisAnalyticsHandler) reloadDB() {
	db, err := maxminddb.Open(config.AnalyticsConfig.GeoIPDBLocation)
	if err != nil {
		log.Error("Failed to init GeoIP Database: ", err)
	} else {
		oldDB := r.GeoIPDB
		r.GeoIPDB = db
		if oldDB != nil {
			oldDB.Close()
		}

	}
	time.Sleep(time.Hour * 1)
}

// RecordHit will store an AnalyticsRecord in Redis
func (r *RedisAnalyticsHandler) RecordHit(record AnalyticsRecord) error {

	AnalyticsPool.SendWork(func() {
		// If we are obfuscating API Keys, store the hashed representation (config check handled in hashing function)
		record.APIKey = publicHash(record.APIKey)

		if config.SlaveOptions.UseRPC {
			// Extend tag list to include this data so wecan segment by node if necessary
			record.Tags = append(record.Tags, "tyk-hybrid-rpc")
		}

		if config.DBAppConfOptions.NodeIsSegmented {
			// Extend tag list to include this data so wecan segment by node if necessary
			record.Tags = append(record.Tags, config.DBAppConfOptions.Tags...)
		}

		// Lets add some metadata
		if record.APIKey != "" {
			record.Tags = append(record.Tags, "key-"+record.APIKey)
		}

		if record.OrgID != "" {
			record.Tags = append(record.Tags, "org-"+record.OrgID)
		}

		record.Tags = append(record.Tags, "api-"+record.APIID)

		encoded, err := msgpack.Marshal(record)

		if err != nil {
			log.Error("Error encoding analytics data: ", err)
		}

		r.Store.AppendToSet(ANALYTICS_KEYNAME, string(encoded))
	})

	return nil

}
<commit_msg>Remove hour-long sleep from analytics.go<commit_after>package main

import (
	"net"
	"regexp"
	"time"

	"github.com/jeffail/tunny"
	"github.com/oschwald/maxminddb-golang"
	"gopkg.in/vmihailenco/msgpack.v2"
)

// AnalyticsRecord encodes the details of a request
type AnalyticsRecord struct {
	Method        string
	Path          string
	RawPath       string
	ContentLength int64
	UserAgent     string
	Day           int
	Month         time.Month
	Year          int
	Hour          int
	ResponseCode  int
	APIKey        string
	TimeStamp     time.Time
	APIVersion    string
	APIName       string
	APIID         string
	OrgID         string
	OauthID       string
	RequestTime   int64
	RawRequest    string
	RawResponse   string
	IPAddress     string
	Geo           GeoData
	Tags          []string
	Alias         string
	TrackPath     bool
	ExpireAt      time.Time `bson:"expireAt" json:"expireAt"`
}

type GeoData struct {
	Country struct {
		ISOCode string `maxminddb:"iso_code"`
	} `maxminddb:"country"`

	City struct {
		GeoNameID uint              `maxminddb:"geoname_id"`
		Names     map[string]string `maxminddb:"names"`
	} `maxminddb:"city"`

	Location struct {
		Latitude  float64 `maxminddb:"latitude"`
		Longitude float64 `maxminddb:"longitude"`
		TimeZone  string  `maxminddb:"time_zone"`
	} `maxminddb:"location"`
}

const (
	ANALYTICS_KEYNAME = "tyk-system-analytics"
)

func (a *AnalyticsRecord) GetGeo(ipStr string) {
	if !config.AnalyticsConfig.EnableGeoIP {
		return
	}

	// Not great, tightly coupled
	if analytics.GeoIPDB == nil {
		return
	}

	// Sometimes it is empty, we can't look up mpty IP addresses
	if ipStr == "" {
		return
	}

	ip := net.ParseIP(ipStr)

	var record GeoData // Or any appropriate struct
	if err := analytics.GeoIPDB.Lookup(ip, &record); err != nil {
		log.Error("GeoIP Failure (not recorded): ", err)
		return
	}

	log.Debug("ISO Code: ", record.Country.ISOCode)
	log.Debug("City: ", record.City.Names["en"])
	log.Debug("Lat: ", record.Location.Latitude)
	log.Debug("Lon: ", record.Location.Longitude)
	log.Debug("TZ: ", record.Location.TimeZone)

	a.Geo = record
}

type NormaliseURLPatterns struct {
	UUIDs  *regexp.Regexp
	IDs    *regexp.Regexp
	Custom []*regexp.Regexp
}

func InitNormalisationPatterns() NormaliseURLPatterns {
	thesePatterns := NormaliseURLPatterns{}

	uuidPat := regexp.MustCompile(`[0-9a-fA-F]{8}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{12}`)
	numPat := regexp.MustCompile(`\/(\d+)`)

	custPats := []*regexp.Regexp{}
	for _, pattern := range config.AnalyticsConfig.NormaliseUrls.Custom {
		if patRe, err := regexp.Compile(pattern); err != nil {
			log.Error("failed to compile custom pattern: ", err)
		} else {
			custPats = append(custPats, patRe)
		}
	}

	thesePatterns.UUIDs = uuidPat
	thesePatterns.IDs = numPat
	thesePatterns.Custom = custPats

	return thesePatterns
}

func (a *AnalyticsRecord) NormalisePath() {
	if config.AnalyticsConfig.NormaliseUrls.Enabled {
		if config.AnalyticsConfig.NormaliseUrls.NormaliseUUIDs {
			a.Path = config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.UUIDs.ReplaceAllString(a.Path, "{uuid}")
		}
		if config.AnalyticsConfig.NormaliseUrls.NormaliseNumbers {
			a.Path = config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.IDs.ReplaceAllString(a.Path, "/{id}")
		}
		if len(config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.Custom) > 0 {
			for _, r := range config.AnalyticsConfig.NormaliseUrls.compiledPatternSet.Custom {
				a.Path = r.ReplaceAllString(a.Path, "{var}")
			}
		}
	}

}

func (a *AnalyticsRecord) SetExpiry(expiresInSeconds int64) {
	expiry := time.Duration(expiresInSeconds) * time.Second
	if expiresInSeconds == 0 {
		// Expiry is set to 100 years
		expiry = (24 * time.Hour) * (365 * 100)
	}

	t := time.Now()
	t2 := t.Add(expiry)
	a.ExpireAt = t2
}

var AnalyticsPool *tunny.WorkPool

// RedisAnalyticsHandler will record analytics data to a redis back end
// as defined in the Config object
type RedisAnalyticsHandler struct {
	Store   *RedisClusterStorageManager
	Clean   Purger
	GeoIPDB *maxminddb.Reader
}

func (r *RedisAnalyticsHandler) Init() {
	if config.AnalyticsConfig.EnableGeoIP {
		go r.reloadDB()
	}

	analytics.Store.Connect()
	var err error

	ps := config.AnalyticsConfig.PoolSize
	if ps == 0 {
		ps = 50
	}

	AnalyticsPool, err = tunny.CreatePoolGeneric(ps).Open()
	if err != nil {
		log.Error("Failed to init analytics pool")
	}
}

func (r *RedisAnalyticsHandler) reloadDB() {
	db, err := maxminddb.Open(config.AnalyticsConfig.GeoIPDBLocation)
	if err != nil {
		log.Error("Failed to init GeoIP Database: ", err)
	} else {
		oldDB := r.GeoIPDB
		r.GeoIPDB = db
		if oldDB != nil {
			oldDB.Close()
		}

	}
}

// RecordHit will store an AnalyticsRecord in Redis
func (r *RedisAnalyticsHandler) RecordHit(record AnalyticsRecord) error {

	AnalyticsPool.SendWork(func() {
		// If we are obfuscating API Keys, store the hashed representation (config check handled in hashing function)
		record.APIKey = publicHash(record.APIKey)

		if config.SlaveOptions.UseRPC {
			// Extend tag list to include this data so wecan segment by node if necessary
			record.Tags = append(record.Tags, "tyk-hybrid-rpc")
		}

		if config.DBAppConfOptions.NodeIsSegmented {
			// Extend tag list to include this data so wecan segment by node if necessary
			record.Tags = append(record.Tags, config.DBAppConfOptions.Tags...)
		}

		// Lets add some metadata
		if record.APIKey != "" {
			record.Tags = append(record.Tags, "key-"+record.APIKey)
		}

		if record.OrgID != "" {
			record.Tags = append(record.Tags, "org-"+record.OrgID)
		}

		record.Tags = append(record.Tags, "api-"+record.APIID)

		encoded, err := msgpack.Marshal(record)

		if err != nil {
			log.Error("Error encoding analytics data: ", err)
		}

		r.Store.AppendToSet(ANALYTICS_KEYNAME, string(encoded))
	})

	return nil

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"regexp"
	"strconv"
	"strings"

	"github.com/sociam/xray-archiver/pipeline/db"
	"github.com/sociam/xray-archiver/pipeline/util"
)

// Err - convenience struct for marshalling errors
type Err struct {
	Code    string `json:"err"`
	Message string `json:"err_msg"`
}

var unit = util.Unit{}

var supportedMimes = map[string]util.Unit{
	"application/json":    unit,
	"application/nahmate": unit,
}

func toBytes(data interface{}) []byte {
	switch v := data.(type) {
	case Err:
		return []byte(v.Message)
	default:
		return []byte(fmt.Sprintf("%v", v))
	}
}

func writeErr(w http.ResponseWriter, mime string, status int, err, msg string, vals ...interface{}) {
	writeData(w, mime, status, Err{err, fmt.Sprintf(msg, vals...)})
}

func writeData(w http.ResponseWriter, mime string, status int, data interface{}) {
	w.WriteHeader(status)
	var err1 error
	w.Header().Set("Content-Type", mime)
	switch mime {
	case "application/nahmate":
		err1 = util.WriteDEAN(w, data)
	case "text/plain":
		_, err1 = w.Write(toBytes(data))
	default:
		w.Header().Set("Content-Type", "application/json")
		fallthrough
	case "application/json":
		err1 = util.WriteJSON(w, data)
	}
	if err1 != nil {
		fmt.Println(err1)
	}
}

func hello(w http.ResponseWriter, r *http.Request) {
	fmt.Println("Got spurious request on " + r.URL.Path)
	writeErr(w, r.Header.Get("Accept"), http.StatusNotFound, "not_found", "Nah mate!")
}

var appPrefixRe = regexp.MustCompile("^/api/apps/")
var dbIDRe = regexp.MustCompile("^\\d+$")
var appIDRe = regexp.MustCompile("^[[:alpha:]][\\w$]*(\\.[[:alpha:]][\\w$]*)*$")

func parseNumCheck(num string) (val int, oops string, err error) {
	val, err = strconv.Atoi(num)

	if err != nil {
		return 0, "num value must be a number", nil
	}

	if val < 0 {
		return 0, "num can not be a value less than 0...", nil
	}

	return val, "", nil
}

func parseLimit(num string) (val string, oops string, err error) {

	if len(val) > 1 {
		return "", "num must have a single value", nil
	}
	realNum := 0

	realNum, oops, err = parseNumCheck(num)

	if oops != "" {
		return num, oops, nil
	}

	if realNum > 1000000 {
		return num, "Limit to high. Please slow down. Chunk the request using the offset", nil
	}

	return num, "", err
}

func parseOffset(num string) (val string, oops string, err error) {
	realNum := 0

	realNum, oops, err = parseNumCheck(num)

	if oops != "" {
		return num, oops, nil
	}

	if err != nil {
		return num, "offset value must be a number", nil
	}
	if realNum < 0 {
		return num, "offset value must positive", nil
	}

	return num, "", err
}

func appsEndpoint(w http.ResponseWriter, r *http.Request) {
	mime := r.Header.Get("Accept")
	//Check input
	if r.Method == "POST" || r.Method == "GET" {
		if _, ok := supportedMimes[mime]; !ok {
			writeErr(w, mime, http.StatusNotAcceptable, "not_acceptable", "This API only supports JSON at the moment.")
			return
		}

		//XXX:Assuming all endpoints have a form to process...
		err := r.ParseForm()
		if err != nil {
			writeErr(w, mime, http.StatusBadRequest, "bad_form", "Error parsing form input: %s", err.Error())
			return
		}

		mime := r.Header.Get("Accept")
		//Default apps

		limit := "10"

		offset := "0"
		isFull := false

		titles := []string{""}
		developers := []string{""}
		genres := []string{""}
		permissions := []string{""}
		appIDs := []string{""}

		fmt.Printf("Parsing app form parameters, params size %s", fmt.Sprint(len(r.Form)))
		//Should not complain if form is 0...

		for name, val := range r.Form {
			oops := ""

			switch name {
			case "limit":
				fmt.Println("Got range of limits", val)
				limit, oops, _ = parseLimit(val[0])
				fmt.Println("Limit Value = ", limit)
				if oops != "" {
					writeErr(w, mime, http.StatusBadRequest, "bad_form", oops)
					return
				}

			case "offset":
				offset, oops, _ = parseOffset(val[0])
				if oops != "" {
					writeErr(w, mime, http.StatusBadRequest, "bad_form", oops)
					return
				}

			case "isFull":
				var err error
				isFull, err = strconv.ParseBool(val[0])
				if err != nil {
					writeErr(w, mime, http.StatusBadRequest, "bad_form", "isFull needs to be a boolean value, true or false")
					return
				}

			case "title":
				fmt.Println("titles:", len(val))
				titles = val

			case "developer":
				developers = val

			case "genre":
				genres = val
				//Valid genre constant check

			case "appId":
				appIDs = val

			default:
				writeErr(w, mime, http.StatusBadRequest, "bad_form", "passed form values did not match params", name)
				return
			}
		}

		if len(titles) == 0 {
			developers = []string{}
		}

		fmt.Println("Gathering full details")

		results, err := db.QuickQuery(isFull, "playstore_apps", limit, offset, developers, genres, permissions, appIDs, titles)

		if err != nil {
			fmt.Println("Error querying database: ", err.Error())
			writeErr(w, mime, http.StatusInternalServerError, "internal_error", "An internal error occurred")
			return
		}

		if !isFull {
			stubs := make([]db.AppStub, len(results), len(results))
			for i, result := range results {
				fmt.Println(result.App)
				stubs[i].Title = result.StoreInfo.(db.PlayStoreInfo).Title
				stubs[i].App = result.App
			}

			writeData(w, mime, http.StatusOK, stubs)
		} else {
			writeData(w, mime, http.StatusOK, results)
		}

	} else {
		writeErr(w, mime, http.StatusBadRequest, "bad_method", "You must POST or GET this endpoint!")
	}

}

// altAppsEndpoint allows for external entities to query for alternative apps based on app ID.
func altAppsEndpoint(w http.ResponseWriter, r *http.Request) {
	mime := r.Header.Get("Accept")
	if r.Method == "POST" || r.Method == "GET" {
		if _, ok := supportedMimes[mime]; !ok {
			writeErr(w, mime, http.StatusNotAcceptable, "not_acceptable", "Yo Dawg, we deal with json only son.")
			return
		}

		split := strings.Split(r.URL.Path, "/")

		if len(split) < 3 {
			writeErr(w, mime, http.StatusBadRequest, "bad_app", "Bad app slashes specified")
			return
		}

		appID := split[3]

		alts, err := db.GetAltApps(appID)

		if err != nil {
			writeErr(w, mime, http.StatusBadRequest, "bad_app", "Seems like we couldn't find your app... Probs means that we don't have any alts")
			return
		}

		writeData(w, mime, http.StatusOK, alts)
	}
}

var cfgFile = flag.String("cfg", "/etc/xray/config.json", "config file location")
var port = flag.Uint("port", 8118, "Port to serve on.")

func init() {
	util.LoadCfg(*cfgFile, util.APIServ)
	db.Open(util.Cfg, true)
}

func main() {
	http.Handle("/", http.FileServer(http.Dir(util.Cfg.AppDir)))

	http.HandleFunc("/api/apps/", appsEndpoint)
	http.HandleFunc("/api/alt/", altAppsEndpoint)

	panic(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
}
<commit_msg>Allow cross origin again (#38)<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"regexp"
	"strconv"
	"strings"

	"github.com/sociam/xray-archiver/pipeline/db"
	"github.com/sociam/xray-archiver/pipeline/util"
)

// Err - convenience struct for marshalling errors
type Err struct {
	Code    string `json:"err"`
	Message string `json:"err_msg"`
}

var unit = util.Unit{}

var supportedMimes = map[string]util.Unit{
	"application/json":    unit,
	"application/nahmate": unit,
}

func toBytes(data interface{}) []byte {
	switch v := data.(type) {
	case Err:
		return []byte(v.Message)
	default:
		return []byte(fmt.Sprintf("%v", v))
	}
}

func writeErr(w http.ResponseWriter, mime string, status int, err, msg string, vals ...interface{}) {
	writeData(w, mime, status, Err{err, fmt.Sprintf(msg, vals...)})
}

func writeData(w http.ResponseWriter, mime string, status int, data interface{}) {
	w.WriteHeader(status)
	var err1 error
	w.Header().Set("Content-Type", mime)
	switch mime {
	case "application/nahmate":
		err1 = util.WriteDEAN(w, data)
	case "text/plain":
		_, err1 = w.Write(toBytes(data))
	default:
		w.Header().Set("Content-Type", "application/json")
		fallthrough
	case "application/json":
		err1 = util.WriteJSON(w, data)
	}
	if err1 != nil {
		fmt.Println(err1)
	}
}

func hello(w http.ResponseWriter, r *http.Request) {
	fmt.Println("Got spurious request on " + r.URL.Path)
	writeErr(w, r.Header.Get("Accept"), http.StatusNotFound, "not_found", "Nah mate!")
}

var appPrefixRe = regexp.MustCompile("^/api/apps/")
var dbIDRe = regexp.MustCompile("^\\d+$")
var appIDRe = regexp.MustCompile("^[[:alpha:]][\\w$]*(\\.[[:alpha:]][\\w$]*)*$")

func parseNumCheck(num string) (val int, oops string, err error) {
	val, err = strconv.Atoi(num)

	if err != nil {
		return 0, "num value must be a number", nil
	}

	if val < 0 {
		return 0, "num can not be a value less than 0...", nil
	}

	return val, "", nil
}

func parseLimit(num string) (val string, oops string, err error) {

	if len(val) > 1 {
		return "", "num must have a single value", nil
	}
	realNum := 0

	realNum, oops, err = parseNumCheck(num)

	if oops != "" {
		return num, oops, nil
	}

	if realNum > 1000000 {
		return num, "Limit to high. Please slow down. Chunk the request using the offset", nil
	}

	return num, "", err
}

func parseOffset(num string) (val string, oops string, err error) {
	realNum := 0

	realNum, oops, err = parseNumCheck(num)

	if oops != "" {
		return num, oops, nil
	}

	if err != nil {
		return num, "offset value must be a number", nil
	}
	if realNum < 0 {
		return num, "offset value must positive", nil
	}

	return num, "", err
}

func appsEndpoint(w http.ResponseWriter, r *http.Request) {
	mime := r.Header.Get("Accept")

	// DONT DELETE DEAN.
	w.Header().Set("Access-Control-Allow-Origin", "*")
	// DEAN... DONT DELETE.

	//Check input
	if r.Method == "POST" || r.Method == "GET" {
		if _, ok := supportedMimes[mime]; !ok {
			writeErr(w, mime, http.StatusNotAcceptable, "not_acceptable", "This API only supports JSON at the moment.")
			return
		}

		//XXX:Assuming all endpoints have a form to process...
		err := r.ParseForm()
		if err != nil {
			writeErr(w, mime, http.StatusBadRequest, "bad_form", "Error parsing form input: %s", err.Error())
			return
		}

		mime := r.Header.Get("Accept")
		//Default apps

		limit := "10"

		offset := "0"
		isFull := false

		titles := []string{""}
		developers := []string{""}
		genres := []string{""}
		permissions := []string{""}
		appIDs := []string{""}

		fmt.Printf("Parsing app form parameters, params size %s", fmt.Sprint(len(r.Form)))
		//Should not complain if form is 0...

		for name, val := range r.Form {
			oops := ""

			switch name {
			case "limit":
				fmt.Println("Got range of limits", val)
				limit, oops, _ = parseLimit(val[0])
				fmt.Println("Limit Value = ", limit)
				if oops != "" {
					writeErr(w, mime, http.StatusBadRequest, "bad_form", oops)
					return
				}

			case "offset":
				offset, oops, _ = parseOffset(val[0])
				if oops != "" {
					writeErr(w, mime, http.StatusBadRequest, "bad_form", oops)
					return
				}

			case "isFull":
				var err error
				isFull, err = strconv.ParseBool(val[0])
				if err != nil {
					writeErr(w, mime, http.StatusBadRequest, "bad_form", "isFull needs to be a boolean value, true or false")
					return
				}

			case "title":
				fmt.Println("titles:", len(val))
				titles = val

			case "developer":
				developers = val

			case "genre":
				genres = val
				//Valid genre constant check

			case "appId":
				appIDs = val

			default:
				writeErr(w, mime, http.StatusBadRequest, "bad_form", "passed form values did not match params", name)
				return
			}
		}

		if len(titles) == 0 {
			developers = []string{}
		}

		fmt.Println("Gathering full details")

		results, err := db.QuickQuery(isFull, "playstore_apps", limit, offset, developers, genres, permissions, appIDs, titles)

		if err != nil {
			fmt.Println("Error querying database: ", err.Error())
			writeErr(w, mime, http.StatusInternalServerError, "internal_error", "An internal error occurred")
			return
		}

		if !isFull {
			stubs := make([]db.AppStub, len(results), len(results))
			for i, result := range results {
				fmt.Println(result.App)
				stubs[i].Title = result.StoreInfo.(db.PlayStoreInfo).Title
				stubs[i].App = result.App
			}

			writeData(w, mime, http.StatusOK, stubs)
		} else {
			writeData(w, mime, http.StatusOK, results)
		}

	} else {
		writeErr(w, mime, http.StatusBadRequest, "bad_method", "You must POST or GET this endpoint!")
	}

}

// altAppsEndpoint allows for external entities to query for alternative apps based on app ID.
func altAppsEndpoint(w http.ResponseWriter, r *http.Request) {
	mime := r.Header.Get("Accept")
	// DEAN DON'T DELETE THIS EITHER
	w.Header().Set("Access-Control-Allow-Origin", "*")
	// DEAN
	if r.Method == "POST" || r.Method == "GET" {
		if _, ok := supportedMimes[mime]; !ok {
			writeErr(w, mime, http.StatusNotAcceptable, "not_acceptable", "Yo Dawg, we deal with json only son.")
			return
		}

		split := strings.Split(r.URL.Path, "/")

		if len(split) < 3 {
			writeErr(w, mime, http.StatusBadRequest, "bad_app", "Bad app slashes specified")
			return
		}

		appID := split[3]

		alts, err := db.GetAltApps(appID)

		if err != nil {
			writeErr(w, mime, http.StatusBadRequest, "bad_app", "Seems like we couldn't find your app... Probs means that we don't have any alts")
			return
		}

		writeData(w, mime, http.StatusOK, alts)
	}
}

var cfgFile = flag.String("cfg", "/etc/xray/config.json", "config file location")
var port = flag.Uint("port", 8118, "Port to serve on.")

func init() {
	util.LoadCfg(*cfgFile, util.APIServ)
	db.Open(util.Cfg, true)
}

func main() {
	http.Handle("/", http.FileServer(http.Dir(util.Cfg.AppDir)))

	http.HandleFunc("/api/apps/", appsEndpoint)
	http.HandleFunc("/api/alt/", altAppsEndpoint)

	panic(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package pcap2log

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/kr/pretty"
	"io"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
)

var textFrameSeparator []byte = []byte("\nFrame ")
var textFrameSeparator1 []byte = []byte("Frame ")

func splitTextFrames(data []byte, atEOF bool) (advance int, token []byte, err error) {
	separatorIndex := bytes.Index(data, textFrameSeparator)
	if separatorIndex == -1 {
		if atEOF {
			log.Println("WARNING skipping before EOF:", string(data))
			return len(data), nil, nil
		} else {
			return 0, nil, nil
		}
	}
	if separatorIndex != 0 {
		if bytes.HasPrefix(data, textFrameSeparator1) {
			separatorIndex = 0
		} else {
			log.Println("WARNING skipping prefix:", string(data[:separatorIndex]))
			return separatorIndex, nil, nil
		}
	}
	// find start of the next frame
	const skip1 = 5
	separatorIndex = bytes.Index(data[skip1:], textFrameSeparator)
	if separatorIndex == -1 {
		if atEOF {
			return len(data), data, nil
		} else {
			return 0, nil, nil
		}
	}
	separatorIndex += skip1

	return separatorIndex, data[:separatorIndex], nil
}

type translator struct {
	r   io.Reader
	w   io.Writer
	sim simulator
	// current message data
	kvStr       map[string]string
	kvInt       map[string]uint
	msgType     byte
	refNumDelta []uint // for Block Single Side Delete Message
	qom         QOMessage
}

func NewTranslator(r io.Reader, w io.Writer) translator {
	return translator{
		r:   r,
		w:   w,
		sim: NewSimulator(w),
	}
}

type MarketSide byte

const (
	MasketSideUnknown MarketSide = 0
	MarketSideBuy                = 'B'
	MarketSideSell               = 'S'
)

type MessageType byte

const (
	MessageTypeUnknown MessageType = iota
	MessageTypeQuoteAdd
	MessageTypeQuoteReplace
	MessageTypeQuoteDelete
	MessageTypeOrderAdd
	MessageTypeOrderExecute
	MessageTypeOrderExecuteWPrice
	MessageTypeOrderCancel
	MessageTypeOrderUpdate
	MessageTypeOrderReplace
	MessageTypeOrderDelete
	MessageTypeBlockOrderDelete
)

type OrderSide struct {
	refNumDelta     uint
	origRefNumDelta uint
	price           uint
	size            uint
	side            MarketSide
}
type QOMessage struct {
	typ          MessageType
	timestamp    uint
	optionId     uint
	side1        OrderSide
	side2        OrderSide
	sseCrossNum  uint
	sseMatchNum  uint
	ssePrintable bool
	ssuReason    byte
	bssdNum      uint
	bssdRefs     []uint
}

var charToMessageType = []MessageType{
	'j': MessageTypeQuoteAdd,
	'J': MessageTypeQuoteAdd,
	'k': MessageTypeQuoteReplace,
	'K': MessageTypeQuoteReplace,
	'Y': MessageTypeQuoteDelete,
	'a': MessageTypeOrderAdd,
	'A': MessageTypeOrderAdd,
	'E': MessageTypeOrderExecute,
	'C': MessageTypeOrderExecuteWPrice,
	'X': MessageTypeOrderCancel,
	'G': MessageTypeOrderUpdate,
	'u': MessageTypeOrderReplace,
	'U': MessageTypeOrderReplace,
	'D': MessageTypeOrderDelete,
	'Z': MessageTypeBlockOrderDelete,
}

func (t *translator) translateQOMessage() {
	t.qom = QOMessage{
		typ:       charToMessageType[t.msgType],
		optionId:  t.kvInt["Option ID"],
		timestamp: t.kvInt["Timestamp"],
	}
	switch t.msgType {
	case 'T', 'L', 'S', 'H', 'O', 'Q', 'I': // ignore Seconds, Base Reference, System,  Options Trading Action, Option Open, Cross Trade, NOII
	case 'j': // Add Quote
		t.qom.side1 = OrderSide{
			side:        MarketSideBuy,
			refNumDelta: t.kvInt["Bid Reference Number Delta"],
			size:        t.kvInt["Bid Size"],
			price:       t.kvInt["Bid Price"],
		}
		t.qom.side2 = OrderSide{
			side:        MarketSideSell,
			refNumDelta: t.kvInt["Ask Reference Number Delta"],
			size:        t.kvInt["Ask Size"],
			price:       t.kvInt["Ask Price"],
		}
	case 'J': // Add Quote
		t.qom.side1 = OrderSide{
			side:        MarketSideBuy,
			refNumDelta: t.kvInt["Bid Reference Number Delta"],
			size:        t.kvInt["Bid Size"],
			price:       t.kvInt["Bid"],
		}
		t.qom.side2 = OrderSide{
			side:        MarketSideSell,
			refNumDelta: t.kvInt["Ask Reference Number Delta"],
			size:        t.kvInt["Ask Size"],
			price:       t.kvInt["Ask"],
		}
	case 'k', 'K': // Quote Replace
		t.qom.side1 = OrderSide{
			side:            MarketSideBuy,
			refNumDelta:     t.kvInt["Bid Reference Number Delta"],
			origRefNumDelta: t.kvInt["Original Bid Reference Number Delta"],
			size:            t.kvInt["Bid Size"],
			price:           t.kvInt["Bid Price"],
		}
		t.qom.side2 = OrderSide{
			side:            MarketSideSell,
			refNumDelta:     t.kvInt["Ask Reference Delta Number"],
			origRefNumDelta: t.kvInt["Original Ask Reference Number Delta"],
			size:            t.kvInt["Ask Size"],
			price:           t.kvInt["Ask Price"],
		}
	case 'Y': // Quote Delete
		t.qom.side1 = OrderSide{
			side:            MarketSideBuy,
			origRefNumDelta: t.kvInt["Bid Reference Number Delta"],
		}
		t.qom.side2 = OrderSide{
			side:            MarketSideSell,
			origRefNumDelta: t.kvInt["Ask Reference Number Delta"],
		}
	case 'a', 'A': // Add Order
		t.qom.side1 = OrderSide{
			side:        MarketSide(t.kvInt["Market Side"]),
			refNumDelta: t.kvInt["Order Reference Number Delta"],
			size:        t.kvInt["Volume"],
			price:       t.kvInt["Price"],
		}
	case 'E': // Single Side Executed
		t.qom.side1 = OrderSide{
			origRefNumDelta: t.kvInt["Reference Number Delta"],
			size:            t.kvInt["Executed Contracts"],
		}
		t.qom.sseCrossNum = t.kvInt["Cross Number"]
		t.qom.sseMatchNum = t.kvInt["Match Number"]
	case 'C': // Single Side Executed with Price
		t.qom.side1 = OrderSide{
			origRefNumDelta: t.kvInt["Reference Number Delta"],
			size:            t.kvInt["Volume"],
			price:           t.kvInt["Price"],
		}
		t.qom.sseCrossNum = t.kvInt["Cross Number"]
		t.qom.sseMatchNum = t.kvInt["Match Number"]
		t.qom.ssePrintable = t.kvStr["Printable"] == "Y"
	case 'X': //  Order Cancel
		t.qom.side1 = OrderSide{
			origRefNumDelta: t.kvInt["Order Reference Number Delta"],
			size:            t.kvInt["Cancelled Contracts"],
		}
	case 'G': // Single Side Update
		t.qom.side1 = OrderSide{
			origRefNumDelta: t.kvInt["Reference Number Delta"],
			price:           t.kvInt["Price"],
			size:            t.kvInt["Volume"],
		}
	case 'u', 'U': // Single Side Replace
		t.qom.side1 = OrderSide{
			refNumDelta:     t.kvInt["New Reference Number Delta"],
			origRefNumDelta: t.kvInt["Original Reference Number Delta"],
			price:           t.kvInt["Price"],
			size:            t.kvInt["Volume"],
		}
	case 'D': // Single Side Delete
		t.qom.side1 = OrderSide{
			origRefNumDelta: t.kvInt["Reference Number Delta"],
		}
	case 'Z': // Block Single Side Delete
		t.qom.bssdNum = t.kvInt["Total Number of Reference Number Deltas."]
		if uint(len(t.refNumDelta)) != t.qom.bssdNum {
			pretty.Println(t.kvInt)
			log.Fatalf("Unexpected number of refs in Z message (%d != %d)\n", t.qom.bssdNum, len(t.refNumDelta))
		}
		t.qom.bssdRefs = append([]uint(nil), t.refNumDelta...)
	default:
		s := pretty.Sprintf("%v", t)
		//log.Fatalf("Unknown message type %d (%c)\n%s\n", t.msgType, t.msgType, s)
		log.Printf("Unknown message type %d (%c)\n%s\n", t.msgType, t.msgType, s)
	}
}

func (t *translator) translate() {
	kvRegexp := regexp.MustCompile("(?m)^            ([^:]*): (.*)$")
	parValueRegexp := regexp.MustCompile(".*\\((\\d+)\\)")
	scanner := bufio.NewScanner(t.r)
	scanner.Split(splitTextFrames)
	for scanner.Scan() {
		//fmt.Println("=====================")
		//fmt.Println(scanner.Text())
		ittoMessages := strings.Split(scanner.Text(), "        ITTO ")
		if len(ittoMessages) == 1 {
			continue
		}
		for _, ittoMessage := range ittoMessages[1:] {
			matches := kvRegexp.FindAllStringSubmatch(ittoMessage, -1)
			t.kvStr = make(map[string]string)
			t.kvInt = make(map[string]uint)
			t.refNumDelta = nil
			t.msgType = 0
			for _, m := range matches {
				k := m[1]
				v := m[2]
				if t.msgType == 'Z' && k == "Reference Number Delta" {
					vInt, err := strconv.ParseUint(v, 0, 32)
					if err != nil {
						log.Fatal("Can't parse", v)
					}
					t.refNumDelta = append(t.refNumDelta, uint(vInt))
				} else {
					if _, ok := t.kvStr[k]; ok {
						pretty.Println(ittoMessage)
						pretty.Println(matches)
						pretty.Println(m)
						log.Fatal("Duplicate key ", k)
					}
					t.kvStr[k] = v
					vInt, err := strconv.ParseUint(v, 0, 32)
					if err == nil {
						t.kvInt[k] = uint(vInt)
					} else if matches := parValueRegexp.FindStringSubmatch(v); matches != nil {
						vInt, err := strconv.ParseUint(matches[1], 0, 32)
						t.kvInt[k] = uint(vInt)
						if err != nil {
							log.Fatal("Can't parse", v)
						}
						if k == "Message Type" {
							t.msgType = byte(vInt)
						}
					}
				}
			}
			t.translateQOMessage()
			t.sim.addMessage(&t.qom, t.msgType)
		}
	}
}

func getTsharkDump(fileName string, args []string) (reader io.Reader, finisher func()) {
	//pretty.Println(fileName, args)
	cmdArgs := []string{
		"-d", "udp.port==18000:10,moldudp64",
		"-V",
		"-r",
		fileName,
	}
	cmdArgs = append(cmdArgs, args...)
	cmd := exec.Command("tshark", cmdArgs...)
	reader, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	finisher = func() {
		if err := cmd.Wait(); err != nil {
			log.Fatal(err)
		}
	}
	return
}

type pcap2log struct {
	InputFileName  string                        `long:"input" short:"i" required:"y" value-name:"PCAP_FILE" description:"input pcap file to read"`
	OutputFileName string                        `long:"output" short:"o" value-name:"FILE" default:"/dev/stdout" default-mask:"stdout" description:"output file"`
	Args           struct{ TsharkArgs []string } `positional-args:"y"`
}

func (p *pcap2log) Execute(args []string) error {
	//fmt.Println("pcap2log Executed", p, args)
	//pretty.Println(p)
	//pretty.Println(args)
	dumpReader, finisher := getTsharkDump(p.InputFileName, p.Args.TsharkArgs)
	defer finisher()
	outFile, err := os.OpenFile(p.OutputFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		log.Fatal(err)
	}
	defer outFile.Close()
	t := NewTranslator(dumpReader, outFile)
	t.translate()
	return nil
}

func InitArgv(parser *flags.Parser) {
	var p2l pcap2log
	parser.AddCommand("pcap2log",
		"convert pcap file to simulator output",
		"",
		&p2l)
}

/*****************************************************************************/
// experiments and debugging

func main() {
	t := NewTranslator(os.Stdin, os.Stdout)
	t.translate()
	_ = pretty.Print
	_ = fmt.Print

}
<commit_msg>pcap2log: make a designated type for option id<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package pcap2log

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/kr/pretty"
	"io"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
)

var textFrameSeparator []byte = []byte("\nFrame ")
var textFrameSeparator1 []byte = []byte("Frame ")

func splitTextFrames(data []byte, atEOF bool) (advance int, token []byte, err error) {
	separatorIndex := bytes.Index(data, textFrameSeparator)
	if separatorIndex == -1 {
		if atEOF {
			log.Println("WARNING skipping before EOF:", string(data))
			return len(data), nil, nil
		} else {
			return 0, nil, nil
		}
	}
	if separatorIndex != 0 {
		if bytes.HasPrefix(data, textFrameSeparator1) {
			separatorIndex = 0
		} else {
			log.Println("WARNING skipping prefix:", string(data[:separatorIndex]))
			return separatorIndex, nil, nil
		}
	}
	// find start of the next frame
	const skip1 = 5
	separatorIndex = bytes.Index(data[skip1:], textFrameSeparator)
	if separatorIndex == -1 {
		if atEOF {
			return len(data), data, nil
		} else {
			return 0, nil, nil
		}
	}
	separatorIndex += skip1

	return separatorIndex, data[:separatorIndex], nil
}

type translator struct {
	r   io.Reader
	w   io.Writer
	sim simulator
	// current message data
	kvStr       map[string]string
	kvInt       map[string]uint
	msgType     byte
	refNumDelta []uint // for Block Single Side Delete Message
	qom         QOMessage
}

func NewTranslator(r io.Reader, w io.Writer) translator {
	return translator{
		r:   r,
		w:   w,
		sim: NewSimulator(w),
	}
}

type MarketSide byte

const (
	MasketSideUnknown MarketSide = 0
	MarketSideBuy                = 'B'
	MarketSideSell               = 'S'
)

type MessageType byte

const (
	MessageTypeUnknown MessageType = iota
	MessageTypeQuoteAdd
	MessageTypeQuoteReplace
	MessageTypeQuoteDelete
	MessageTypeOrderAdd
	MessageTypeOrderExecute
	MessageTypeOrderExecuteWPrice
	MessageTypeOrderCancel
	MessageTypeOrderUpdate
	MessageTypeOrderReplace
	MessageTypeOrderDelete
	MessageTypeBlockOrderDelete
)

type OptionId uint

const OptionIdUnknown OptionId = 0

type OrderSide struct {
	refNumDelta     uint
	origRefNumDelta uint
	price           uint
	size            uint
	side            MarketSide
}
type QOMessage struct {
	typ          MessageType
	timestamp    uint
	optionId     OptionId
	side1        OrderSide
	side2        OrderSide
	sseCrossNum  uint
	sseMatchNum  uint
	ssePrintable bool
	ssuReason    byte
	bssdNum      uint
	bssdRefs     []uint
}

var charToMessageType = []MessageType{
	'j': MessageTypeQuoteAdd,
	'J': MessageTypeQuoteAdd,
	'k': MessageTypeQuoteReplace,
	'K': MessageTypeQuoteReplace,
	'Y': MessageTypeQuoteDelete,
	'a': MessageTypeOrderAdd,
	'A': MessageTypeOrderAdd,
	'E': MessageTypeOrderExecute,
	'C': MessageTypeOrderExecuteWPrice,
	'X': MessageTypeOrderCancel,
	'G': MessageTypeOrderUpdate,
	'u': MessageTypeOrderReplace,
	'U': MessageTypeOrderReplace,
	'D': MessageTypeOrderDelete,
	'Z': MessageTypeBlockOrderDelete,
}

func (t *translator) translateQOMessage() {
	t.qom = QOMessage{
		typ:       charToMessageType[t.msgType],
		timestamp: t.kvInt["Timestamp"],
	}
	if oid := t.kvInt["Option ID"]; oid != 0 {
		t.qom.optionId = OptionId(oid)
	} else {
		t.qom.optionId = OptionIdUnknown
	}
	switch t.msgType {
	case 'T', 'L', 'S', 'H', 'O', 'Q', 'I': // ignore Seconds, Base Reference, System,  Options Trading Action, Option Open, Cross Trade, NOII
	case 'j': // Add Quote
		t.qom.side1 = OrderSide{
			side:        MarketSideBuy,
			refNumDelta: t.kvInt["Bid Reference Number Delta"],
			size:        t.kvInt["Bid Size"],
			price:       t.kvInt["Bid Price"],
		}
		t.qom.side2 = OrderSide{
			side:        MarketSideSell,
			refNumDelta: t.kvInt["Ask Reference Number Delta"],
			size:        t.kvInt["Ask Size"],
			price:       t.kvInt["Ask Price"],
		}
	case 'J': // Add Quote
		t.qom.side1 = OrderSide{
			side:        MarketSideBuy,
			refNumDelta: t.kvInt["Bid Reference Number Delta"],
			size:        t.kvInt["Bid Size"],
			price:       t.kvInt["Bid"],
		}
		t.qom.side2 = OrderSide{
			side:        MarketSideSell,
			refNumDelta: t.kvInt["Ask Reference Number Delta"],
			size:        t.kvInt["Ask Size"],
			price:       t.kvInt["Ask"],
		}
	case 'k', 'K': // Quote Replace
		t.qom.side1 = OrderSide{
			side:            MarketSideBuy,
			refNumDelta:     t.kvInt["Bid Reference Number Delta"],
			origRefNumDelta: t.kvInt["Original Bid Reference Number Delta"],
			size:            t.kvInt["Bid Size"],
			price:           t.kvInt["Bid Price"],
		}
		t.qom.side2 = OrderSide{
			side:            MarketSideSell,
			refNumDelta:     t.kvInt["Ask Reference Delta Number"],
			origRefNumDelta: t.kvInt["Original Ask Reference Number Delta"],
			size:            t.kvInt["Ask Size"],
			price:           t.kvInt["Ask Price"],
		}
	case 'Y': // Quote Delete
		t.qom.side1 = OrderSide{
			side:            MarketSideBuy,
			origRefNumDelta: t.kvInt["Bid Reference Number Delta"],
		}
		t.qom.side2 = OrderSide{
			side:            MarketSideSell,
			origRefNumDelta: t.kvInt["Ask Reference Number Delta"],
		}
	case 'a', 'A': // Add Order
		t.qom.side1 = OrderSide{
			side:        MarketSide(t.kvInt["Market Side"]),
			refNumDelta: t.kvInt["Order Reference Number Delta"],
			size:        t.kvInt["Volume"],
			price:       t.kvInt["Price"],
		}
	case 'E': // Single Side Executed
		t.qom.side1 = OrderSide{
			origRefNumDelta: t.kvInt["Reference Number Delta"],
			size:            t.kvInt["Executed Contracts"],
		}
		t.qom.sseCrossNum = t.kvInt["Cross Number"]
		t.qom.sseMatchNum = t.kvInt["Match Number"]
	case 'C': // Single Side Executed with Price
		t.qom.side1 = OrderSide{
			origRefNumDelta: t.kvInt["Reference Number Delta"],
			size:            t.kvInt["Volume"],
			price:           t.kvInt["Price"],
		}
		t.qom.sseCrossNum = t.kvInt["Cross Number"]
		t.qom.sseMatchNum = t.kvInt["Match Number"]
		t.qom.ssePrintable = t.kvStr["Printable"] == "Y"
	case 'X': //  Order Cancel
		t.qom.side1 = OrderSide{
			origRefNumDelta: t.kvInt["Order Reference Number Delta"],
			size:            t.kvInt["Cancelled Contracts"],
		}
	case 'G': // Single Side Update
		t.qom.side1 = OrderSide{
			origRefNumDelta: t.kvInt["Reference Number Delta"],
			price:           t.kvInt["Price"],
			size:            t.kvInt["Volume"],
		}
	case 'u', 'U': // Single Side Replace
		t.qom.side1 = OrderSide{
			refNumDelta:     t.kvInt["New Reference Number Delta"],
			origRefNumDelta: t.kvInt["Original Reference Number Delta"],
			price:           t.kvInt["Price"],
			size:            t.kvInt["Volume"],
		}
	case 'D': // Single Side Delete
		t.qom.side1 = OrderSide{
			origRefNumDelta: t.kvInt["Reference Number Delta"],
		}
	case 'Z': // Block Single Side Delete
		t.qom.bssdNum = t.kvInt["Total Number of Reference Number Deltas."]
		if uint(len(t.refNumDelta)) != t.qom.bssdNum {
			pretty.Println(t.kvInt)
			log.Fatalf("Unexpected number of refs in Z message (%d != %d)\n", t.qom.bssdNum, len(t.refNumDelta))
		}
		t.qom.bssdRefs = append([]uint(nil), t.refNumDelta...)
	default:
		s := pretty.Sprintf("%v", t)
		//log.Fatalf("Unknown message type %d (%c)\n%s\n", t.msgType, t.msgType, s)
		log.Printf("Unknown message type %d (%c)\n%s\n", t.msgType, t.msgType, s)
	}
}

func (t *translator) translate() {
	kvRegexp := regexp.MustCompile("(?m)^            ([^:]*): (.*)$")
	parValueRegexp := regexp.MustCompile(".*\\((\\d+)\\)")
	scanner := bufio.NewScanner(t.r)
	scanner.Split(splitTextFrames)
	for scanner.Scan() {
		//fmt.Println("=====================")
		//fmt.Println(scanner.Text())
		ittoMessages := strings.Split(scanner.Text(), "        ITTO ")
		if len(ittoMessages) == 1 {
			continue
		}
		for _, ittoMessage := range ittoMessages[1:] {
			matches := kvRegexp.FindAllStringSubmatch(ittoMessage, -1)
			t.kvStr = make(map[string]string)
			t.kvInt = make(map[string]uint)
			t.refNumDelta = nil
			t.msgType = 0
			for _, m := range matches {
				k := m[1]
				v := m[2]
				if t.msgType == 'Z' && k == "Reference Number Delta" {
					vInt, err := strconv.ParseUint(v, 0, 32)
					if err != nil {
						log.Fatal("Can't parse", v)
					}
					t.refNumDelta = append(t.refNumDelta, uint(vInt))
				} else {
					if _, ok := t.kvStr[k]; ok {
						pretty.Println(ittoMessage)
						pretty.Println(matches)
						pretty.Println(m)
						log.Fatal("Duplicate key ", k)
					}
					t.kvStr[k] = v
					vInt, err := strconv.ParseUint(v, 0, 32)
					if err == nil {
						t.kvInt[k] = uint(vInt)
					} else if matches := parValueRegexp.FindStringSubmatch(v); matches != nil {
						vInt, err := strconv.ParseUint(matches[1], 0, 32)
						t.kvInt[k] = uint(vInt)
						if err != nil {
							log.Fatal("Can't parse", v)
						}
						if k == "Message Type" {
							t.msgType = byte(vInt)
						}
					}
				}
			}
			t.translateQOMessage()
			t.sim.addMessage(&t.qom, t.msgType)
		}
	}
}

func getTsharkDump(fileName string, args []string) (reader io.Reader, finisher func()) {
	//pretty.Println(fileName, args)
	cmdArgs := []string{
		"-d", "udp.port==18000:10,moldudp64",
		"-V",
		"-r",
		fileName,
	}
	cmdArgs = append(cmdArgs, args...)
	cmd := exec.Command("tshark", cmdArgs...)
	reader, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	finisher = func() {
		if err := cmd.Wait(); err != nil {
			log.Fatal(err)
		}
	}
	return
}

type pcap2log struct {
	InputFileName  string                        `long:"input" short:"i" required:"y" value-name:"PCAP_FILE" description:"input pcap file to read"`
	OutputFileName string                        `long:"output" short:"o" value-name:"FILE" default:"/dev/stdout" default-mask:"stdout" description:"output file"`
	Args           struct{ TsharkArgs []string } `positional-args:"y"`
}

func (p *pcap2log) Execute(args []string) error {
	//fmt.Println("pcap2log Executed", p, args)
	//pretty.Println(p)
	//pretty.Println(args)
	dumpReader, finisher := getTsharkDump(p.InputFileName, p.Args.TsharkArgs)
	defer finisher()
	outFile, err := os.OpenFile(p.OutputFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		log.Fatal(err)
	}
	defer outFile.Close()
	t := NewTranslator(dumpReader, outFile)
	t.translate()
	return nil
}

func InitArgv(parser *flags.Parser) {
	var p2l pcap2log
	parser.AddCommand("pcap2log",
		"convert pcap file to simulator output",
		"",
		&p2l)
}

/*****************************************************************************/
// experiments and debugging

func main() {
	t := NewTranslator(os.Stdin, os.Stdout)
	t.translate()
	_ = pretty.Print
	_ = fmt.Print

}
<|endoftext|>


<commit_before>package accounts

import (
	"net/url"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
)

// Account holds configuration information for an account
type Account struct {
	DocID       string
	DocRev      string
	AccountType string                 `json:"account_type"`
	Basic       *url.Userinfo          `json:"basic,omitempty"`
	Oauth       *OauthInfo             `json:"oauth,omitempty"`
	Extras      map[string]interface{} `json:"oauth_callback_results"`
}

// OauthInfo holds configuration information for an oauth account
type OauthInfo struct {
	AccessToken  string    `json:"access_token,omitempty"`
	TokenType    string    `json:"token_type,omitempty"`
	ExpiresAt    time.Time `json:"expires_at,omitempty"`
	RefreshToken string    `json:"refresh_token,omitempty"`
}

// ID is used to implement the couchdb.Doc interface
func (ac *Account) ID() string { return ac.DocID }

// Rev is used to implement the couchdb.Doc interface
func (ac *Account) Rev() string { return ac.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (ac *Account) SetID(id string) { ac.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (ac *Account) SetRev(rev string) { ac.DocRev = rev }

// DocType implements couchdb.Doc
func (ac *Account) DocType() string { return consts.Accounts }

// Clone implements couchdb.Doc
func (ac *Account) Clone() couchdb.Doc { cloned := *ac; return &cloned }

// Valid implements permissions.Validable
func (ac *Account) Valid(field, expected string) bool {
	return field == "account_type" && expected == ac.AccountType
}
<commit_msg>fix account serialization<commit_after>package accounts

import (
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
)

// Account holds configuration information for an account
type Account struct {
	DocID       string                 `json:"_id,omitempty"`
	DocRev      string                 `json:"_rev,omitempty"`
	Name        string                 `json:"name"`
	AccountType string                 `json:"account_type"`
	Basic       *BasicInfo             `json:"auth,omitempty"`
	Oauth       *OauthInfo             `json:"oauth,omitempty"`
	Extras      map[string]interface{} `json:"oauth_callback_results,omitempty"`
}

// OauthInfo holds configuration information for an oauth account
type OauthInfo struct {
	AccessToken  string    `json:"access_token,omitempty"`
	TokenType    string    `json:"token_type,omitempty"`
	ExpiresAt    time.Time `json:"expires_at,omitempty"`
	RefreshToken string    `json:"refresh_token,omitempty"`
}

// BasicInfo holds configuration information for an user/pass account
type BasicInfo struct {
	Login    string `json:"login,omitempty"`
	Password string `json:"password,omitempty"`
}

// ID is used to implement the couchdb.Doc interface
func (ac *Account) ID() string { return ac.DocID }

// Rev is used to implement the couchdb.Doc interface
func (ac *Account) Rev() string { return ac.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (ac *Account) SetID(id string) { ac.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (ac *Account) SetRev(rev string) { ac.DocRev = rev }

// DocType implements couchdb.Doc
func (ac *Account) DocType() string { return consts.Accounts }

// Clone implements couchdb.Doc
func (ac *Account) Clone() couchdb.Doc { cloned := *ac; return &cloned }

// Valid implements permissions.Validable
func (ac *Account) Valid(field, expected string) bool {
	return field == "account_type" && expected == ac.AccountType
}
<|endoftext|>