<commit_before>package gamehack

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/subosito/twilio"

	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
)

type Notification struct {
	UserID           int64             `json:"userId"`
	StorylineUpdates []StorylineUpdate `json:"storylineUpdates"`
}

type StorylineUpdate struct {
	// TODO: Change to equivalent of enum
	Reason          string `json:"reason"`
	LastSegmentType string `json:"lastSegmentType"`
}

type Location struct {
	Lat float64 `json:"lat"`
	Lon float64 `json:"lon"`
}

type Place struct {
	Id       int      `json:"id"`
	Type     string   `json:"type"`
	Location Location `json:"location"`
}

type Segment struct {
	Place      Place  `json:"place"`
	LastUpdate string `json:"lastUpdate"`
}

type DailySegments struct {
	Segments []Segment `json:"segments"`
}

func init() {
	http.HandleFunc("/", root)
	http.HandleFunc("/login", login)
	http.HandleFunc("/logout", logout)
	http.HandleFunc("/phone", phone)
	http.HandleFunc("/addphone", addPhone)
	http.HandleFunc("/delphone", delPhone)
	http.HandleFunc("/authorize", authorize)
	http.HandleFunc("/oauth2callback", oauthCallback)
	http.HandleFunc("/notification", handleNotification)
}

func handleNotification(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Invalid method.", http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, "Error reading request body.", http.StatusBadRequest)
		return
	}

	var notification Notification
	err = json.Unmarshal(body, &notification)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	hasDataUpload := false
	for _, update := range notification.StorylineUpdates {
		if update.Reason == "DataUpload" {
			hasDataUpload = true
			break
		}
	}

	if hasDataUpload {
	}
	/*fmt.Fprintf(w, "%v", notification)
	if err != nil {
		http.Error(w, "Error writing response body.", http.StatusInternalServerError)
		return
	}*/
	/*var place Place
	err = json.Unmarshal(body, &place)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	sendText(place, "+15555555555", w, r)*/
}

// TODO: this function is untested
func updateDailySegments(dailySegmentsList []DailySegments, userID string, w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	var phoneEntries []PhoneEntry
	_, err := datastore.NewQuery("PhoneEntry").Ancestor(datastore.NewKey(c, "User", userID, 0, nil)).GetAll(c, &phoneEntries)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	for _, dailySegments := range dailySegmentsList {
		for _, segment := range dailySegments.Segments {
			for _, phone := range phoneEntries {
				sendText(segment.Place, phone.Phone, w, r)
			}
			break // first item in slice is latest one
		}
		break // first item in slice is latest one
	}
}

func sendText(place Place, phone string, w http.ResponseWriter, r *http.Request) {
	a := appengine.NewContext(r)
	f := urlfetch.Client(a)
	c := twilio.NewClient(twilioSid, twilioAuthToken, f)

	params := twilio.MessageParams{
		Body: fmt.Sprintf("Your child is now at lat %f lon %f", place.Location.Lat, place.Location.Lon),
	}
	_, _, err := c.Messages.Send("+15555555555", phone, params)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
	}
}
<commit_msg>pass in user key instead of user id<commit_after>package gamehack

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/subosito/twilio"

	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
)

type Notification struct {
	UserID           int64             `json:"userId"`
	StorylineUpdates []StorylineUpdate `json:"storylineUpdates"`
}

type StorylineUpdate struct {
	// TODO: Change to equivalent of enum
	Reason          string `json:"reason"`
	LastSegmentType string `json:"lastSegmentType"`
}

type Location struct {
	Lat float64 `json:"lat"`
	Lon float64 `json:"lon"`
}

type Place struct {
	Id       int      `json:"id"`
	Type     string   `json:"type"`
	Location Location `json:"location"`
}

type Segment struct {
	Place      Place  `json:"place"`
	LastUpdate string `json:"lastUpdate"`
}

type DailySegments struct {
	Segments []Segment `json:"segments"`
}

func init() {
	http.HandleFunc("/", root)
	http.HandleFunc("/login", login)
	http.HandleFunc("/logout", logout)
	http.HandleFunc("/phone", phone)
	http.HandleFunc("/addphone", addPhone)
	http.HandleFunc("/delphone", delPhone)
	http.HandleFunc("/authorize", authorize)
	http.HandleFunc("/oauth2callback", oauthCallback)
	http.HandleFunc("/notification", handleNotification)
}

func handleNotification(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Invalid method.", http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, "Error reading request body.", http.StatusBadRequest)
		return
	}

	var notification Notification
	err = json.Unmarshal(body, &notification)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	hasDataUpload := false
	for _, update := range notification.StorylineUpdates {
		if update.Reason == "DataUpload" {
			hasDataUpload = true
			break
		}
	}

	if hasDataUpload {
	}
	/*fmt.Fprintf(w, "%v", notification)
	if err != nil {
		http.Error(w, "Error writing response body.", http.StatusInternalServerError)
		return
	}*/
	/*var place Place
	err = json.Unmarshal(body, &place)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	sendText(place, "+15555555555", w, r)*/
}

// TODO: this function is untested
func updateDailySegments(dailySegmentsList []DailySegments, userKey *datastore.Key, w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	var phoneEntries []PhoneEntry
	_, err := datastore.NewQuery("PhoneEntry").Ancestor(userKey).GetAll(c, &phoneEntries)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	for _, dailySegments := range dailySegmentsList {
		for _, segment := range dailySegments.Segments {
			for _, phone := range phoneEntries {
				sendText(segment.Place, phone.Phone, w, r)
			}
			break // first item in slice is latest one
		}
		break // first item in slice is latest one
	}
}

func sendText(place Place, phone string, w http.ResponseWriter, r *http.Request) {
	a := appengine.NewContext(r)
	f := urlfetch.Client(a)
	c := twilio.NewClient(twilioSid, twilioAuthToken, f)

	params := twilio.MessageParams{
		Body: fmt.Sprintf("Your child is now at lat %f lon %f", place.Location.Lat, place.Location.Lon),
	}
	_, _, err := c.Messages.Send("+15555555555", phone, params)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/BenPhegan/vagrantshadow/Godeps/_workspace/src/github.com/mcuadros/go-version"
	"log"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
)

type BoxHandler struct {
	Boxes          map[string]map[string]Box
	Directories    []string
	TemplateString string
	Hostname       string
	Port           int
}

type BoxMetadata struct {
	Provider string `json:"provider"`
}

type SimpleBox struct {
	Username string
	Boxname  string
	Location string
	Provider string
	Version  string
}

type Box struct {
	Created             string    `json:"created_at"`
	Updated             string    `json:"updated_at"`
	Tag                 string    `json:"tag"`
	Name                string    `json:"name"`
	ShortDescription    string    `json:"short_description"`
	DescriptionHtml     string    `json:"description_html"`
	DescriptionMarkdown string    `json:"description_markdown"`
	Username            string    `json:"username"`
	Private             bool      `json:"private"`
	CurrentVersion      *Version  `json:"current_version"`
	Versions            []Version `json:"versions"`
}

type Version struct {
	Version             string     `json:"version"`
	Status              string     `json:"status"`
	DescriptionHtml     string     `json:"description_html"`
	DescriptionMarkdown string     `json:"description_markdown"`
	Created             string     `json:"created_at"`
	Updated             string     `json:"updated_at"`
	Number              int        `json:"number"`
	Downloads           int        `json:"downloads"`
	ReleaseUrl          string     `json:"release_url"`
	RevokeUrl           string     `json:"revoke_url"`
	Providers           []Provider `json:"providers"`
}

type Provider struct {
	Name         string `json:"name"`
	Hosted       string `json:"hosted"`
	HostedToken  string `json:"hosted_token"`
	OriginalUrl  string `json:"original_url"`
	UploadUrl    string `json:"upload_url"`
	Created      string `json:"created_at"`
	Updated      string `json:"updated_at"`
	DownloadUrl  string `json:"download_url"`
	Url          string `json:"url"`
	LocalBoxFile string `json:"-"`
}

func (bh *BoxHandler) BoxRegex() string {
	return `(?P<owner>\w*)-VAGRANTSLASH-(?P<boxname>[a-zA-Z0-9]*)__(?P<version>[a-zA-Z0-9\.-]*)__(?P<provider>[a-zA-Z0-9]*).box`
}

func (bh *BoxHandler) BoxAvailable(username string, boxname string) bool {
	return (bh.Boxes[username][boxname].Username != "")
}

func (bh *BoxHandler) GetBoxFileLocation(username string, boxName string, provider string, version string) string {
	boxList := bh.Boxes[username][boxName]
	for _, box := range boxList.Versions {
		if box.Version == version {
			for _, boxprovider := range box.Providers {
				if boxprovider.Name == provider {
					return boxprovider.LocalBoxFile
				}
			}
		}
	}
	return ""
}

func (bh *BoxHandler) GetBox(user string, boxName string) Box {
	return bh.Boxes[user][boxName]
}

func (bh *BoxHandler) PopulateBoxes(directories []string, port *int, hostname *string) {
	log.Println("Populating boxes..")
	absolutedirectories := []string{}
	for _, d := range directories {
		if !path.IsAbs(d) {
			wd, _ := os.Getwd()
			absolute := path.Clean(path.Join(wd, d))
			absolutedirectories = append(absolutedirectories, absolute)
		}
	}
	bh.Directories = absolutedirectories
	boxfiles := getBoxList(absolutedirectories)
	boxdata := bh.getBoxData(boxfiles)
	bh.createBoxes(boxdata, *port, hostname)

	for _, boxinfo := range bh.Boxes {
		for boxname, box := range boxinfo {
			for _, version := range box.Versions {
				for _, provider := range version.Providers {
					log.Println("Found " + box.Username + "/" + boxname + "/" + version.Version + "/" + provider.Name)
				}
			}
		}
	}
}

// getBoxList returns a list of .box files in the directories provided.
// Returns full path
func getBoxList(directories []string) []string {
	boxes := []string{}
	for _, d := range directories {
		log.Println("Checking for files in: " + d)
		directoryglob := path.Join(d, "*.box")
		files, _ := filepath.Glob(directoryglob)
		boxes = append(boxes, files...)
	}
	return boxes
}

//getBoxData returns an array of SimpleBox objects based on Vagrant box files
func (bh *BoxHandler) getBoxData(boxfiles []string) []SimpleBox {
	results := []SimpleBox{}
	var myExp = regexp.MustCompile(bh.BoxRegex())
	for _, b := range boxfiles {
		matches := myExp.FindStringSubmatch(filepath.Base(b))
		if matches == nil || len(matches) != 5 {
			log.Println("Could not match metadata from filename: " + filepath.Base(b))
			return results
		}
		newbox := SimpleBox{Username: matches[1], Boxname: matches[2], Location: b, Provider: matches[4], Version: matches[3]}
		results = append(results, newbox)
	}

	return results
}

//Creates the data structure used to provide box data to Vagrant
func (bh *BoxHandler) createBoxes(sb []SimpleBox, port int, hostname *string) {
	boxes := make(map[string]map[string]Box)
	for _, b := range sb {

		box := boxes[b.Username][b.Boxname]
		if box.Name == "" {
			box = Box{}
			box.Name = b.Username + "/" + b.Boxname
			box.Username = b.Username
			box.Private = false
		}

		provider := Provider{}
		provider.Name = b.Provider
		provider.Hosted = "true"
		provider.DownloadUrl = "http://" + *hostname + ":" + strconv.Itoa(port) + "/" + b.Username + "/" + b.Boxname + "/" + b.Version + "/" + b.Provider + "/" + b.Provider + ".box"
		provider.Url = provider.DownloadUrl
		provider.LocalBoxFile = b.Location

		if len(box.Versions) > 0 {
			providerAppended := false
			for i, v := range box.Versions {
				if v.Version == b.Version {
					box.Versions[i].Providers = append(box.Versions[i].Providers, provider)
					providerAppended = true
				}
			}

			if providerAppended == false {
				newversion := Version{}
				newversion.Status = "active"
				newversion.Version = b.Version
				newversion.Providers = []Provider{provider}
				box.Versions = append(box.Versions, newversion)
			}
		} else {
			newversion := Version{}
			newversion.Status = "active"
			newversion.Version = b.Version
			newversion.Providers = []Provider{provider}
			box.Versions = []Version{newversion}
		}

		if boxes[b.Username] == nil {
			boxes[b.Username] = make(map[string]Box)
		}

		box.CurrentVersion = nil
		for _, v := range box.Versions {
			if box.CurrentVersion == nil {
				box.CurrentVersion = &v
			}
			if version.Compare(box.CurrentVersion.Version, v.Version, "<") {
				box.CurrentVersion = &v
			}
		}

		boxes[b.Username][b.Boxname] = box
	}

	bh.Boxes = boxes
}
<commit_msg>Fix issue where absolute directories were ignored.<commit_after>package main

import (
	"github.com/BenPhegan/vagrantshadow/Godeps/_workspace/src/github.com/mcuadros/go-version"
	"log"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
)

type BoxHandler struct {
	Boxes          map[string]map[string]Box
	Directories    []string
	TemplateString string
	Hostname       string
	Port           int
}

type BoxMetadata struct {
	Provider string `json:"provider"`
}

type SimpleBox struct {
	Username string
	Boxname  string
	Location string
	Provider string
	Version  string
}

type Box struct {
	Created             string    `json:"created_at"`
	Updated             string    `json:"updated_at"`
	Tag                 string    `json:"tag"`
	Name                string    `json:"name"`
	ShortDescription    string    `json:"short_description"`
	DescriptionHtml     string    `json:"description_html"`
	DescriptionMarkdown string    `json:"description_markdown"`
	Username            string    `json:"username"`
	Private             bool      `json:"private"`
	CurrentVersion      *Version  `json:"current_version"`
	Versions            []Version `json:"versions"`
}

type Version struct {
	Version             string     `json:"version"`
	Status              string     `json:"status"`
	DescriptionHtml     string     `json:"description_html"`
	DescriptionMarkdown string     `json:"description_markdown"`
	Created             string     `json:"created_at"`
	Updated             string     `json:"updated_at"`
	Number              int        `json:"number"`
	Downloads           int        `json:"downloads"`
	ReleaseUrl          string     `json:"release_url"`
	RevokeUrl           string     `json:"revoke_url"`
	Providers           []Provider `json:"providers"`
}

type Provider struct {
	Name         string `json:"name"`
	Hosted       string `json:"hosted"`
	HostedToken  string `json:"hosted_token"`
	OriginalUrl  string `json:"original_url"`
	UploadUrl    string `json:"upload_url"`
	Created      string `json:"created_at"`
	Updated      string `json:"updated_at"`
	DownloadUrl  string `json:"download_url"`
	Url          string `json:"url"`
	LocalBoxFile string `json:"-"`
}

func (bh *BoxHandler) BoxRegex() string {
	return `(?P<owner>\w*)-VAGRANTSLASH-(?P<boxname>[a-zA-Z0-9]*)__(?P<version>[a-zA-Z0-9\.-]*)__(?P<provider>[a-zA-Z0-9]*).box`
}

func (bh *BoxHandler) BoxAvailable(username string, boxname string) bool {
	return (bh.Boxes[username][boxname].Username != "")
}

func (bh *BoxHandler) GetBoxFileLocation(username string, boxName string, provider string, version string) string {
	boxList := bh.Boxes[username][boxName]
	for _, box := range boxList.Versions {
		if box.Version == version {
			for _, boxprovider := range box.Providers {
				if boxprovider.Name == provider {
					return boxprovider.LocalBoxFile
				}
			}
		}
	}
	return ""
}

func (bh *BoxHandler) GetBox(user string, boxName string) Box {
	return bh.Boxes[user][boxName]
}

func (bh *BoxHandler) PopulateBoxes(directories []string, port *int, hostname *string) {
	log.Println("Populating boxes..")
	absolutedirectories := []string{}
	for _, d := range directories {
		if !path.IsAbs(d) {
			wd, _ := os.Getwd()
			absolute := path.Clean(path.Join(wd, d))
			absolutedirectories = append(absolutedirectories, absolute)
		} else {
			absolutedirectories = append(absolutedirectories, d)
		}
	}
	bh.Directories = absolutedirectories
	boxfiles := getBoxList(absolutedirectories)
	boxdata := bh.getBoxData(boxfiles)
	bh.createBoxes(boxdata, *port, hostname)

	for _, boxinfo := range bh.Boxes {
		for boxname, box := range boxinfo {
			for _, version := range box.Versions {
				for _, provider := range version.Providers {
					log.Println("Found " + box.Username + "/" + boxname + "/" + version.Version + "/" + provider.Name)
				}
			}
		}
	}
}

// getBoxList returns a list of .box files in the directories provided.
// Returns full path
func getBoxList(directories []string) []string {
	boxes := []string{}
	for _, d := range directories {
		log.Println("Checking for files in: " + d)
		directoryglob := path.Join(d, "*.box")
		files, _ := filepath.Glob(directoryglob)
		boxes = append(boxes, files...)
	}
	return boxes
}

//getBoxData returns an array of SimpleBox objects based on Vagrant box files
func (bh *BoxHandler) getBoxData(boxfiles []string) []SimpleBox {
	results := []SimpleBox{}
	var myExp = regexp.MustCompile(bh.BoxRegex())
	for _, b := range boxfiles {
		matches := myExp.FindStringSubmatch(filepath.Base(b))
		if matches == nil || len(matches) != 5 {
			log.Println("Could not match metadata from filename: " + filepath.Base(b))
			return results
		}
		newbox := SimpleBox{Username: matches[1], Boxname: matches[2], Location: b, Provider: matches[4], Version: matches[3]}
		results = append(results, newbox)
	}

	return results
}

//Creates the data structure used to provide box data to Vagrant
func (bh *BoxHandler) createBoxes(sb []SimpleBox, port int, hostname *string) {
	boxes := make(map[string]map[string]Box)
	for _, b := range sb {

		box := boxes[b.Username][b.Boxname]
		if box.Name == "" {
			box = Box{}
			box.Name = b.Username + "/" + b.Boxname
			box.Username = b.Username
			box.Private = false
		}

		provider := Provider{}
		provider.Name = b.Provider
		provider.Hosted = "true"
		provider.DownloadUrl = "http://" + *hostname + ":" + strconv.Itoa(port) + "/" + b.Username + "/" + b.Boxname + "/" + b.Version + "/" + b.Provider + "/" + b.Provider + ".box"
		provider.Url = provider.DownloadUrl
		provider.LocalBoxFile = b.Location

		if len(box.Versions) > 0 {
			providerAppended := false
			for i, v := range box.Versions {
				if v.Version == b.Version {
					box.Versions[i].Providers = append(box.Versions[i].Providers, provider)
					providerAppended = true
				}
			}

			if providerAppended == false {
				newversion := Version{}
				newversion.Status = "active"
				newversion.Version = b.Version
				newversion.Providers = []Provider{provider}
				box.Versions = append(box.Versions, newversion)
			}
		} else {
			newversion := Version{}
			newversion.Status = "active"
			newversion.Version = b.Version
			newversion.Providers = []Provider{provider}
			box.Versions = []Version{newversion}
		}

		if boxes[b.Username] == nil {
			boxes[b.Username] = make(map[string]Box)
		}

		box.CurrentVersion = nil
		for _, v := range box.Versions {
			if box.CurrentVersion == nil {
				box.CurrentVersion = &v
			}
			if version.Compare(box.CurrentVersion.Version, v.Version, "<") {
				box.CurrentVersion = &v
			}
		}

		boxes[b.Username][b.Boxname] = box
	}

	bh.Boxes = boxes
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"testing"
)

func testcmd(cmd string) (string, error) {
	fmt.Println("(T) " + cmd)
	switch {
	case cmd == "sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir":
		return currenttest.vs.ls(), nil
	case cmd == "docker ps -aq --no-trunc":
		res := ""
		for _, _ = range currenttest.cs {
			res = res + "x\n"
		}
		return res, nil
	case strings.HasPrefix(cmd, "docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' "):
		return currenttest.inspectVolumes(), nil
	case strings.HasPrefix(cmd, "sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):
		deleted := cmd[len("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):]
		deletions = append(deletions, deleted)
		return "", nil
	case strings.HasPrefix(cmd, "sudo readlink /mnt/sda1/var/lib/docker/vfs/dir/"):
		if strings.Contains(cmd, ",nonexistent") {
			return "", errors.New("non-existent linked folder")
		}
		r := regexp.MustCompile(`.*\$([^#]+)###.*`)
		ss := r.FindStringSubmatch(cmd)
		if len(ss) == 2 {
			folder := ss[1]
			folder = folder + strings.Repeat("1", 64-len(folder))
			return folder, nil
		}
		return "", nil
	case strings.HasPrefix(cmd, "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/"):
		if cmd == "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/" {
			return "", errors.New("non-ls linked folder")
		}
		return "", nil
	default:
		currentT.Fatalf("test '%s': unknown command!\n", cmd)
		return fmt.Sprintf("test '%s'", cmd), errors.New("unknown command")
	}
}

type volspecs []string
type contspecs []string
type Test struct {
	title string
	vs    volspecs
	cs    contspecs
	res   []int
	strs  []string
	ci    int
}

func newTest(title string) *Test {
	return &Test{title: title, res: []int{0, 0, 0, 0, 0}}
}
func (t *Test) setContainersPs(cs contspecs) *Test {
	t.cs = cs
	return t
}
func (t *Test) setVolumesLs(vs volspecs) *Test {
	t.vs = vs
	return t
}

type setterRes interface {
	setResAt(index int) *Test
}

type result struct {
	res int
	t   *Test
}

type resultOne struct {
	t *Test
}

func (t *Test) expects(number int) *result {
	return &result{t: t, res: number}
}

func (t *Test) expectOne() *resultOne {
	return &resultOne{t: t}
}

func (r *result) setResAt(index int) *Test {
	r.t.res[index] = r.res
	return r.t
}
func (r *resultOne) setResAt(index int) *Test {
	r.t.res[index] = 1
	return r.t
}

func (r *result) containers() *Test {
	return r.setResAt(0)
}
func (ro resultOne) container() *Test {
	return ro.setResAt(0)
}

func (r *result) volumes() *Test {
	return r.setResAt(2)
}
func (r *result) orphanedVolumes() *Test {
	return r.setResAt(3)
}

func (r *result) markers() *Test {
	return r.setResAt(4)
}

func (t *Test) mustProduce(strs []string) *Test {
	t.strs = strs
	return t
}

func (vs volspecs) ls() string {
	if len(vs) == 0 {
		return ""
	}
	res := ""
	for i, spec := range vs {
		switch {
		case strings.HasSuffix(spec, "/"):
			spec = spec[:len(spec)-1]
			res = res + spec + strings.Repeat(fmt.Sprintf("%d", i), 64-len(spec)) + "/\n"
		case strings.HasSuffix(spec, "@"):
			mp := "." + strings.Replace(spec, ";", "###", -1)
			mp = strings.Replace(mp, "/", ",#,", -1)
			res = res + mp + "\n"

		default:
			res = res + spec + "\n"
		}
	}
	return res
}

func (t *Test) inspectVolumes() string {
	if len(t.cs) == 0 {
		return ""
	}
	res := t.cs[t.ci]
	t.ci = t.ci + 1
	return res
}

var deletions = []string{}
var tests = []*Test{
	newTest("empty vfs"),
	newTest("2 valid containers without any volume").
		setContainersPs([]string{"/contA,", "/contB,"}).
		expects(2).containers().
		mustProduce([]string{"cnt 'contA' (x)[false] - 0 vol", "cnt 'contB' (x)[false] - 0 vol"}),
	newTest("2 valid volumes").
		setVolumesLs([]string{"fa/", "fb/"}).
		expects(2).volumes().
		expects(2).orphanedVolumes().
		mustProduce([]string{"vol 'fa00000'<<nil>>", "vol 'fb11111'<<nil>>"}),
	newTest("Invalid (ill-formed) markers must be deleted").
		setVolumesLs([]string{"cainv/path/a@"}).
		expects(-1).markers(),
	newTest("Invalid (no readlink) markers must be deleted").
		setVolumesLs([]string{"ca;/path/nonexistenta@", "cb;/path/nonexistentb@"}).
		expects(-2).markers(),
	newTest("Invalid (no ls) markers must be deleted").
		setVolumesLs([]string{"ca;/path/nolsa@", "cb;/path/nolsb@"}).
		expects(-2).markers(),
	newTest("Invalid (no vdir) markers must be deleted").
		setVolumesLs([]string{"ca$novdira;/path/nolsa@", "cb$novdirb;/path/nolsb@"}).
		expects(-2).markers(),
	newTest("two valid markers").
		setVolumesLs([]string{"ca$fa;/path/vola@", "cb$fb;/path/volb@"}).
		expects(2).markers().
		mustProduce([]string{"marker 'fa11111'<ca$fa->/path/vola>", "marker 'fb11111'<cb$fb->/path/volb>"}),
	/*
		Test{"Invalid (bad name) volume", []string{"inva/"}, []int{0, 0, -1, 0, 0}, []string{}},
		Test{"Invalid file in volume vfs dir", []string{"invf"}, []int{0, 0, -1, 0, 0}, []string{}},
	*/
}
var currenttest *Test
var currentT *testing.T

// TestContainers test different vfs scenarios
func TestContainers(t *testing.T) {
	cmd = testcmd
	currentT = t
	for i, test := range tests {
		currenttest = test
		deletions = []string{}
		fmt.Println("------ vvv " + test.title + " vvv ------")
		main()
		tc := Containers()
		toc := OrphanedContainers()
		tv := Volumes()
		tov := OrphanedVolumes()
		tm := Markers()
		if len(tc) != test.res[0] {
			t.Errorf("Test %d: '%s' expected '%d' containers, got '%d'", i+1, test.title, test.res[0], len(tc))
		}
		if len(toc) != test.res[1] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned containers, got '%d'", i+1, test.title, test.res[1], len(toc))
		}
		if nbvolumes(tv) != test.res[2] {
			t.Errorf("Test %d: '%s' expected '%d' volumes, got '%d'", i+1, test.title, test.res[2], nbvolumes(tv))
		}
		if len(tov) != test.res[3] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned volumes, got '%d'", i+1, test.title, test.res[3], len(tov))
		}
		if nbmarkers(tm) != test.res[4] {
			t.Errorf("Test %d: '%s' expected '%d' markers, got '%d'", i+1, test.title, test.res[4], nbmarkers(tm))
		}

		for _, c := range tc {
			cs := c.String()
			check(cs, "container", test, t, i)
		}
		for _, v := range tv {
			vs := v.String()
			check(vs, "volume", test, t, i)
		}
		for _, m := range tm {
			ms := m.String()
			check(ms, "marker", test, t, i)
		}
		fmt.Println("------ ^^^ " + test.title + " ^^^ ------")
		fmt.Println("----------")
	}
}

func check(s string, tmsg string, test *Test, t *testing.T, i int) {
	found := false
	for _, tms := range test.strs {
		if s == tms {
			found = true
			break
		}
	}
	if !found {
		t.Errorf("Test %d: '%s' expected %s >%s<, not found", i+1, test.title, tmsg, s)
	}

}

func nbmarkers(tm markers) int {
	res := len(tm)
	for _, d := range deletions {
		if strings.HasPrefix(d, ".") {
			res = res - 1
		}
	}
	return res
}

func nbvolumes(vm volumes) int {
	res := len(vm)
	for _, d := range deletions {
		if !strings.HasPrefix(d, ".") {
			res = res - 1
		}
	}
	return res
}
<commit_msg>gcl_test.go:restore invalid volumes tests<commit_after>package main

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"testing"
)

func testcmd(cmd string) (string, error) {
	fmt.Println("(T) " + cmd)
	switch {
	case cmd == "sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir":
		return currenttest.vs.ls(), nil
	case cmd == "docker ps -aq --no-trunc":
		res := ""
		for _, _ = range currenttest.cs {
			res = res + "x\n"
		}
		return res, nil
	case strings.HasPrefix(cmd, "docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' "):
		return currenttest.inspectVolumes(), nil
	case strings.HasPrefix(cmd, "sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):
		deleted := cmd[len("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):]
		deletions = append(deletions, deleted)
		return "", nil
	case strings.HasPrefix(cmd, "sudo readlink /mnt/sda1/var/lib/docker/vfs/dir/"):
		if strings.Contains(cmd, ",nonexistent") {
			return "", errors.New("non-existent linked folder")
		}
		r := regexp.MustCompile(`.*\$([^#]+)###.*`)
		ss := r.FindStringSubmatch(cmd)
		if len(ss) == 2 {
			folder := ss[1]
			folder = folder + strings.Repeat("1", 64-len(folder))
			return folder, nil
		}
		return "", nil
	case strings.HasPrefix(cmd, "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/"):
		if cmd == "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/" {
			return "", errors.New("non-ls linked folder")
		}
		return "", nil
	default:
		currentT.Fatalf("test '%s': unknown command!\n", cmd)
		return fmt.Sprintf("test '%s'", cmd), errors.New("unknown command")
	}
}

type volspecs []string
type contspecs []string
type Test struct {
	title string
	vs    volspecs
	cs    contspecs
	res   []int
	strs  []string
	ci    int
}

func newTest(title string) *Test {
	return &Test{title: title, res: []int{0, 0, 0, 0, 0}}
}
func (t *Test) setContainersPs(cs contspecs) *Test {
	t.cs = cs
	return t
}
func (t *Test) setVolumesLs(vs volspecs) *Test {
	t.vs = vs
	return t
}

type setterRes interface {
	setResAt(index int) *Test
}

type result struct {
	res int
	t   *Test
}

type resultOne struct {
	t *Test
}

func (t *Test) expects(number int) *result {
	return &result{t: t, res: number}
}

func (t *Test) expectOne() *resultOne {
	return &resultOne{t: t}
}

func (r *result) setResAt(index int) *Test {
	r.t.res[index] = r.res
	return r.t
}
func (r *resultOne) setResAt(index int) *Test {
	r.t.res[index] = 1
	return r.t
}

func (r *result) containers() *Test {
	return r.setResAt(0)
}
func (ro resultOne) container() *Test {
	return ro.setResAt(0)
}

func (r *result) volumes() *Test {
	return r.setResAt(2)
}
func (r *result) orphanedVolumes() *Test {
	return r.setResAt(3)
}

func (r *result) markers() *Test {
	return r.setResAt(4)
}

func (t *Test) mustProduce(strs []string) *Test {
	t.strs = strs
	return t
}

func (vs volspecs) ls() string {
	if len(vs) == 0 {
		return ""
	}
	res := ""
	for i, spec := range vs {
		switch {
		case strings.HasSuffix(spec, "/"):
			spec = spec[:len(spec)-1]
			res = res + spec + strings.Repeat(fmt.Sprintf("%d", i), 64-len(spec)) + "/\n"
		case strings.HasSuffix(spec, "@"):
			mp := "." + strings.Replace(spec, ";", "###", -1)
			mp = strings.Replace(mp, "/", ",#,", -1)
			res = res + mp + "\n"

		default:
			res = res + spec + "\n"
		}
	}
	return res
}

func (t *Test) inspectVolumes() string {
	if len(t.cs) == 0 {
		return ""
	}
	res := t.cs[t.ci]
	t.ci = t.ci + 1
	return res
}

var deletions = []string{}
var tests = []*Test{
	newTest("empty vfs"),
	newTest("2 valid containers without any volume").
		setContainersPs([]string{"/contA,", "/contB,"}).
		expects(2).containers().
		mustProduce([]string{"cnt 'contA' (x)[false] - 0 vol", "cnt 'contB' (x)[false] - 0 vol"}),
	newTest("2 valid volumes").
		setVolumesLs([]string{"fa/", "fb/"}).
		expects(2).volumes().
		expects(2).orphanedVolumes().
		mustProduce([]string{"vol 'fa00000'<<nil>>", "vol 'fb11111'<<nil>>"}),
	newTest("Invalid (ill-formed) markers must be deleted").
		setVolumesLs([]string{"cainv/path/a@"}).
		expects(-1).markers(),
	newTest("Invalid (no readlink) markers must be deleted").
		setVolumesLs([]string{"ca;/path/nonexistenta@", "cb;/path/nonexistentb@"}).
		expects(-2).markers(),
	newTest("Invalid (no ls) markers must be deleted").
		setVolumesLs([]string{"ca;/path/nolsa@", "cb;/path/nolsb@"}).
		expects(-2).markers(),
	newTest("Invalid (no vdir) markers must be deleted").
		setVolumesLs([]string{"ca$novdira;/path/nolsa@", "cb$novdirb;/path/nolsb@"}).
		expects(-2).markers(),
	newTest("two valid markers").
		setVolumesLs([]string{"ca$fa;/path/vola@", "cb$fb;/path/volb@"}).
		expects(2).markers().
		mustProduce([]string{"marker 'fa11111'<ca$fa->/path/vola>", "marker 'fb11111'<cb$fb->/path/volb>"}),
	newTest("Invalid (bad name) volume").
		setVolumesLs([]string{"inva/"}).
		expects(-1).volumes(),
	newTest("Invalid file in volume vfs dir").
		setVolumesLs([]string{"invf"}).
		expects(-1).volumes(),
}
var currenttest *Test
var currentT *testing.T

// TestContainers test different vfs scenarios
func TestContainers(t *testing.T) {
	cmd = testcmd
	currentT = t
	for i, test := range tests {
		currenttest = test
		deletions = []string{}
		fmt.Println("------ vvv " + test.title + " vvv ------")
		main()
		tc := Containers()
		toc := OrphanedContainers()
		tv := Volumes()
		tov := OrphanedVolumes()
		tm := Markers()
		if len(tc) != test.res[0] {
			t.Errorf("Test %d: '%s' expected '%d' containers, got '%d'", i+1, test.title, test.res[0], len(tc))
		}
		if len(toc) != test.res[1] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned containers, got '%d'", i+1, test.title, test.res[1], len(toc))
		}
		if nbvolumes(tv) != test.res[2] {
			t.Errorf("Test %d: '%s' expected '%d' volumes, got '%d'", i+1, test.title, test.res[2], nbvolumes(tv))
		}
		if len(tov) != test.res[3] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned volumes, got '%d'", i+1, test.title, test.res[3], len(tov))
		}
		if nbmarkers(tm) != test.res[4] {
			t.Errorf("Test %d: '%s' expected '%d' markers, got '%d'", i+1, test.title, test.res[4], nbmarkers(tm))
		}

		for _, c := range tc {
			cs := c.String()
			check(cs, "container", test, t, i)
		}
		for _, v := range tv {
			vs := v.String()
			check(vs, "volume", test, t, i)
		}
		for _, m := range tm {
			ms := m.String()
			check(ms, "marker", test, t, i)
		}
		fmt.Println("------ ^^^ " + test.title + " ^^^ ------")
		fmt.Println("----------")
	}
}

func check(s string, tmsg string, test *Test, t *testing.T, i int) {
	found := false
	for _, tms := range test.strs {
		if s == tms {
			found = true
			break
		}
	}
	if !found {
		t.Errorf("Test %d: '%s' expected %s >%s<, not found", i+1, test.title, tmsg, s)
	}

}

func nbmarkers(tm markers) int {
	res := len(tm)
	for _, d := range deletions {
		if strings.HasPrefix(d, ".") {
			res = res - 1
		}
	}
	return res
}

func nbvolumes(vm volumes) int {
	res := len(vm)
	for _, d := range deletions {
		if !strings.HasPrefix(d, ".") {
			res = res - 1
		}
	}
	return res
}
<|endoftext|>


<commit_before>package ccrdt

import (
	"strconv"

	"github.com/koding/redis"
)

// GCounter is a grow-only counter (inspired by vector clocks) in which only
// increment and merge are possible. Divergent histories are resolved by taking
// the maximum count for the counter.  The value of the counter is the sum of
// all counts.
//
// TODO implement merge!!
type GCounter struct {
	ccrdt *CCRDT
	key   string
}

// NewGCounter creates a new GCounter
func (c *CCRDT) NewGCounter(key string) *GCounter {
	return &GCounter{
		ccrdt: c,
		key:   key,
	}
}

// Add adds item to the GCounter with a given delta
func (g *GCounter) Add(delta int64) error {
	_, err := g.ccrdt.sessions.One().IncrBy(g.key, delta)
	return err
}

// Sum returns the sum of all the actors
func (g *GCounter) Sum() (int64, error) {
	var res int64
	for _, c := range g.ccrdt.sessions.All() {
		val, err := c.Get(g.key)
		if err != nil && err != redis.ErrNil {
			return 0, err
		}

		if val == "" {
			val = "0"
		}
		i, err := strconv.ParseInt(val, 10, 64)
		if err != nil {
			return 0, err
		}

		res += i
	}

	return res, nil
}
<commit_msg>CCRDT: implement merge according to the paper<commit_after>package ccrdt

import (
	"strconv"

	"github.com/koding/redis"
)

// GCounter is a grow-only counter (inspired by vector clocks) in which only
// increment and merge are possible. Divergent histories are resolved by taking
// the maximum count for the counter.  The value of the counter is the sum of
// all counts.
//
// TODO implement merge!!
type GCounter struct {
	ccrdt *CCRDT
	key   string
}

// NewGCounter creates a new GCounter
func (c *CCRDT) NewGCounter(key string) *GCounter {
	return &GCounter{
		ccrdt: c,
		key:   key,
	}
}

// Add adds item to the GCounter with a given delta
func (g *GCounter) Add(delta int64) error {

	errCount := 0
	var lastErr error

	// add goroutine support
	for _, c := range g.ccrdt.sessions.All() {

		// TODO we can do read-repair here
		// redis returns lastest value
		_, err := c.IncrBy(g.key, delta)
		if err != nil {
			lastErr = err
			errCount++
		}
	}

	// at least we have one success
	if errCount != g.ccrdt.sessions.Count() {
		return nil
	}

	return lastErr
}

// Merge returns the sum of all the actors
func (g *GCounter) Merge() (int64, error) {
	var res int64
	var repairNeeded bool
	for i, c := range g.ccrdt.sessions.All() {
		val, err := c.Get(g.key)
		if err != nil && err != redis.ErrNil {
			// ignore the errors
			// return 0, err
		}

		if val == "" {
			val = "0"
		}

		d, err := strconv.ParseInt(val, 10, 64)
		if err != nil {
			// ignore the errors
			// return 0, err
		}

		// if the `res`is smaller than the current value, previous ones should
		// be repaired
		if res < d {
			// if this is the first operation, ignore the case
			if i != 0 {
				repairNeeded = true
			}

			res = d
		}
	}

	if repairNeeded {
		go g.repair()
	}

	return res, nil
}

func (g *GCounter) repair() {
	// do repairing
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"net/http"
	"time"

	"golang.org/x/oauth2"

	"github.com/jacobsa/reqtrace"

	storagev1 "google.golang.org/api/storage/v1"
)

// OAuth scopes for GCS. For use with e.g. google.DefaultTokenSource.
const (
	Scope_FullControl = storagev1.DevstorageFullControlScope
	Scope_ReadOnly    = storagev1.DevstorageReadOnlyScope
	Scope_ReadWrite   = storagev1.DevstorageReadWriteScope
)

// Conn represents a connection to GCS, pre-bound with a project ID and
// information required for authorization.
type Conn interface {
	// Return a Bucket object representing the GCS bucket with the given name. No
	// immediate validation is performed.
	GetBucket(name string) Bucket
}

// Configuration accepted by NewConn.
type ConnConfig struct {
	// An oauth2 token source to use for authenticating to GCS.
	//
	// You probably want this one:
	//     http://godoc.org/golang.org/x/oauth2/google#DefaultTokenSource
	TokenSource oauth2.TokenSource

	// The value to set in User-Agent headers for outgoing HTTP requests. If
	// empty, a default will be used.
	UserAgent string

	// The maximum amount of time to spend sleeping in a retry loop with
	// exponential backoff for failed requests. The default of zero disables
	// automatic retries.
	//
	// If you enable automatic retries, beware of the following:
	//
	//  *  Bucket.CreateObject will buffer the entire object contents in memory,
	//     so your object contents must not be too large to fit.
	//
	//  *  Bucket.NewReader needs to perform an additional round trip to GCS in
	//     order to find the latest object generation if you don't specify a
	//     particular generation.
	//
	//  *  Make sure your operations are idempotent, or that your application can
	//     tolerate it if not.
	//
	MaxBackoffSleep time.Duration
}

// Open a connection to GCS.
func NewConn(cfg *ConnConfig) (c Conn, err error) {
	// Fix the user agent if there is none.
	userAgent := cfg.UserAgent
	if userAgent == "" {
		const defaultUserAgent = "github.com-jacobsa-gloud-gcs"
		userAgent = defaultUserAgent
	}

	c = &conn{
		client:          cfg.HTTPClient,
		userAgent:       userAgent,
		maxBackoffSleep: cfg.MaxBackoffSleep,
	}

	return
}

type conn struct {
	client          *http.Client
	userAgent       string
	maxBackoffSleep time.Duration
}

func (c *conn) GetBucket(name string) (b Bucket) {
	b = newBucket(c.client, c.userAgent, name)

	// Enable retry loops if requested.
	if c.maxBackoffSleep > 0 {
		// TODO(jacobsa): Show the retries as distinct spans in the trace.
		b = newRetryBucket(c.maxBackoffSleep, b)
	}

	// Enable tracing if appropriate.
	if reqtrace.Enabled() {
		b = &reqtraceBucket{
			Wrapped: b,
		}
	}

	// Print debug output when enabled.
	b = newDebugBucket(b)

	return
}
<commit_msg>Return an error from Conn.GetBucket.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"net/http"
	"time"

	"golang.org/x/oauth2"

	"github.com/jacobsa/reqtrace"

	storagev1 "google.golang.org/api/storage/v1"
)

// OAuth scopes for GCS. For use with e.g. google.DefaultTokenSource.
const (
	Scope_FullControl = storagev1.DevstorageFullControlScope
	Scope_ReadOnly    = storagev1.DevstorageReadOnlyScope
	Scope_ReadWrite   = storagev1.DevstorageReadWriteScope
)

// Conn represents a connection to GCS, pre-bound with a project ID and
// information required for authorization.
type Conn interface {
	// Return a Bucket object representing the GCS bucket with the given name.
	GetBucket(name string) (b Bucket, err error)
}

// Configuration accepted by NewConn.
type ConnConfig struct {
	// An oauth2 token source to use for authenticating to GCS.
	//
	// You probably want this one:
	//     http://godoc.org/golang.org/x/oauth2/google#DefaultTokenSource
	TokenSource oauth2.TokenSource

	// The value to set in User-Agent headers for outgoing HTTP requests. If
	// empty, a default will be used.
	UserAgent string

	// The maximum amount of time to spend sleeping in a retry loop with
	// exponential backoff for failed requests. The default of zero disables
	// automatic retries.
	//
	// If you enable automatic retries, beware of the following:
	//
	//  *  Bucket.CreateObject will buffer the entire object contents in memory,
	//     so your object contents must not be too large to fit.
	//
	//  *  Bucket.NewReader needs to perform an additional round trip to GCS in
	//     order to find the latest object generation if you don't specify a
	//     particular generation.
	//
	//  *  Make sure your operations are idempotent, or that your application can
	//     tolerate it if not.
	//
	MaxBackoffSleep time.Duration
}

// Open a connection to GCS.
func NewConn(cfg *ConnConfig) (c Conn, err error) {
	// Fix the user agent if there is none.
	userAgent := cfg.UserAgent
	if userAgent == "" {
		const defaultUserAgent = "github.com-jacobsa-gloud-gcs"
		userAgent = defaultUserAgent
	}

	c = &conn{
		client:          cfg.HTTPClient,
		userAgent:       userAgent,
		maxBackoffSleep: cfg.MaxBackoffSleep,
	}

	return
}

type conn struct {
	client          *http.Client
	userAgent       string
	maxBackoffSleep time.Duration
}

func (c *conn) GetBucket(name string) (b Bucket) {
	b = newBucket(c.client, c.userAgent, name)

	// Enable retry loops if requested.
	if c.maxBackoffSleep > 0 {
		// TODO(jacobsa): Show the retries as distinct spans in the trace.
		b = newRetryBucket(c.maxBackoffSleep, b)
	}

	// Enable tracing if appropriate.
	if reqtrace.Enabled() {
		b = &reqtraceBucket{
			Wrapped: b,
		}
	}

	// Print debug output when enabled.
	b = newDebugBucket(b)

	return
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	postByDay = `select DATE(Time) as date, count(RID) as count from messages` +
		` where channel = '#geekhack' group by date order by count` +
		` desc limit 10;`
	postByDayAll = `select DATE(Time) as date, count(RID) as count from messages` +
		` where channel = '#geekhack' group by date order by date`
	totalPosts = `select Nick, COUNT(Nick) as Posts from messages where channel` +
		` = '#geekhack' group by nick order by Posts desc limit 10;`
	postsByMinute = `select count from (select HOUR(Time)*60+MINUTE(Time) as` +
		` date, count(RID)/(SELECT DATEDIFF(NOW(), (SELECT MIN(Time)` +
		` from messages where channel = '#geekhack'))) as count from` +
		` messages where channel ='#geekhack' group by date order by date) as subquery;`
	topTenWords = `select Nick, ` + "`" + `%[1]s` + "`" + ` from Words order by ` + "`" + `%[1]s` + "`" + ` desc limit 10;`
)

type Geekhack struct {
	db     *sql.DB
	config Config

	mutex                 sync.RWMutex // Protects:
	PostsByDay            []Tuple
	CurseWords            map[string][]Tuple
	TotalPosts            []Tuple
	PostsByMinute         []float64
	PostsByMinuteSmoothed []float64
	PostByDayAll          [][]int64
	PostByDayAllSmoothed  [][]int64
	age                   time.Time
}

type Tuple struct {
	Name  string
	Count int
}

func NewGeekhack(config Config) (*Geekhack, error) {
	db, err := sql.Open("mysql", config.DBConn)
	if err != nil {
		return nil, err
	}

	//err = db.Ping()
	if err != nil {
		return nil, err
	}

	geekhack := &Geekhack{
		CurseWords: make(map[string][]Tuple),
		db:         db,
		config:     config,
	}

	go geekhack.Update()
	go geekhack.Updater()

	return geekhack, nil
}

func (g *Geekhack) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	switch r.URL.Path {
	case "/geekhack/":
		g.Main(w, r)
	case "/geekhack/postsbyminute":
		g.pbmHandler(w, r)
	case "/geekhack/postsbydayall":
		g.pbdaHandler(w, r)
	default:
		http.NotFound(w, r)
	}
}

func (g *Geekhack) Main(w http.ResponseWriter, r *http.Request) {
	g.mutex.RLock()
	defer g.mutex.RUnlock()
	if err := templates.ExecuteTemplate(w, "geekhack.tmpl", g); err != nil {
		log.Println(err)
	}
}

func (g *Geekhack) pbmHandler(w http.ResponseWriter, r *http.Request) {
	g.mutex.RLock()
	defer g.mutex.RUnlock()
	type returnStruct struct {
		Type string    `json:"type,omitempty"`
		Name string    `json:"name"`
		Data []float64 `json:"data"`
	}

	jsonSource := []returnStruct{
		returnStruct{
			"",
			"Posts Per Minute",
			g.PostsByMinute,
		},
		returnStruct{
			"spline",
			"Posts Per Minute Smoothed",
			g.PostsByMinuteSmoothed,
		},
	}
	jsonData, err := json.Marshal(jsonSource)
	if err != nil {
		log.Println(err)
		http.Error(w, "Error generating minute data", 500)
	}
	w.Header().Set("Content-Type", "application/json")
	written, err := w.Write(jsonData)
	if written < len(jsonData) || err != nil {
		log.Println("Error writing response to client")
	}
}

func (g *Geekhack) pbdaHandler(w http.ResponseWriter, r *http.Request) {
	g.mutex.RLock()
	defer g.mutex.RUnlock()
	type returnStruct struct {
		Type string    `json:"type,omitempty"`
		Name string    `json:"name"`
		Data [][]int64 `json:"data"`
	}
	jsonSource := []returnStruct{
		returnStruct{
			Type: "",
			Name: "Posts Per Day All",
			Data: g.PostByDayAll,
		},
		returnStruct{
			Type: "spline",
			Name: "Posts Per Day All Smoothed",
			Data: g.PostByDayAllSmoothed,
		},
	}
	jsonData, err := json.Marshal(jsonSource)
	if err != nil {
		log.Println(err)
		http.Error(w, "Error generating day data", 500)
	}
	w.Header().Set("Content-Type", "application/json")
	written, err := w.Write(jsonData)
	if written < len(jsonData) || err != nil {
		log.Println("Error writing response to client")
	}
}

func (g *Geekhack) runQuery(query string) ([]Tuple, error) {
	var nick string
	var posts int
	var tuple []Tuple
	rows, err := g.db.Query(query)
	if err != nil {
		log.Println(err)
		return nil, err
	}
	for rows.Next() {
		if err := rows.Scan(&nick, &posts); err != nil {
			return nil, err
		}
		tuple = append(tuple, Tuple{nick, posts})
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	return tuple, nil
}

func (g *Geekhack) UpdateCurseWords() (map[string][]Tuple, error) {
	CurseWords := make(map[string][]Tuple)
	query, err := g.db.Query(`SELECT * FROM Words LIMIT 1`)
	if err != nil {
		return nil, err
	}
	defer query.Close()
	words, err := query.Columns()
	if err != nil {
		return nil, err
	}
	for _, word := range words {
		if word == "Nick" {
			continue
		}
		tuple, err := g.runQuery(fmt.Sprintf(topTenWords, word))
		if err != nil {
			return nil, err
		}
		CurseWords[word] = tuple
	}
	return CurseWords, nil
}

func (g *Geekhack) UpdatePostByDayAll() ([][]int64, error) {
	var date string
	var posts int
	var returnValue [][]int64
	rows, err := g.db.Query(postByDayAll)
	if err != nil {
		return nil, err
	}
	for rows.Next() {
		if err := rows.Scan(&date, &posts); err != nil {
			return nil, err
		}
		convTime, err := time.Parse("2006-01-02", date)
		if err != nil {
			return nil, err
		}
		returnValue = append(returnValue, []int64{convTime.Unix() * 1000, int64(posts)})
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	return returnValue, nil
}

func (g *Geekhack) Update() {
	start := time.Now()
	PostsByDay, err := g.runQuery(postByDay)
	if err != nil {
		log.Println(err)
		return
	}

	TotalPosts, err := g.runQuery(totalPosts)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Posts updated in:", time.Since(start))

	start = time.Now()
	CurseWords, err := g.UpdateCurseWords()
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("CurseWords updated in:", time.Since(start))

	start = time.Now()
	PostsByMinute := []float64{}
	rows, err := g.db.Query(postsByMinute)
	if err != nil {
		log.Println(err)
		return
	}
	for rows.Next() {
		var posts float64
		if err := rows.Scan(&posts); err != nil {
			log.Println(err)
			return
		}
		PostsByMinute = append(PostsByMinute, posts)
	}
	if err := rows.Err(); err != nil {
		log.Println(err)
		return
	}
	PostsByMinuteSmoothed := movingAverage(movingAverage(movingAverage(PostsByMinute, 20), 10), 5)
	log.Println("PostsByMinute updated in:", time.Since(start))

	start = time.Now()
	PostsByDayAll, err := g.UpdatePostByDayAll()
	if err != nil {
		log.Println(err)
		return
	}
	PostsByDayAllSmoothed := averageWithTime(PostsByDayAll)
	log.Println("PostsByDayAll updated in:", time.Since(start))

	// Update the struct
	g.mutex.Lock()
	g.PostsByDay = PostsByDay
	g.TotalPosts = TotalPosts
	g.CurseWords = CurseWords
	g.PostsByMinute = PostsByMinute
	g.PostsByMinuteSmoothed = PostsByMinuteSmoothed
	g.PostByDayAll = PostsByDayAll
	g.PostByDayAllSmoothed = PostsByDayAllSmoothed
	g.age = time.Now()
	g.mutex.Unlock()
	// Finish update, need to unlock it
}

func averageWithTime(original [][]int64) [][]int64 {
	first := make([]int64, len(original))
	second := make([]float64, len(original))
	for key, value := range original {
		first[key] = value[0]
		second[key] = float64(value[1])
	}
	second = movingAverage(movingAverage(movingAverage(second, 20), 10), 5)
	result := make([][]int64, len(original))
	for i := 0; i < len(original); i++ {
		result[i] = []int64{first[i], int64(second[i])}
	}
	return result
}

func movingAverage(input []float64, size int) []float64 {
	var start, end int
	result := make([]float64, len(input))
	for i := 0; i < len(input); i++ {
		if i < size {
			start = 0
		} else {
			start = i - size
		}
		if size+i+1 > len(input) {
			end = len(input)
		} else {
			end = size + i + 1
		}
		for _, value := range input[start:end] {
			result[i] += value
		}
		result[i] /= float64(len(input[start:end]))
	}
	return result
}

func (g *Geekhack) Updater() {
	ticker := time.Tick(2 * time.Minute)
	for _ = range ticker {
		g.Update()
	}
}
<commit_msg>renamed words table<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	postByDay = `select DATE(Time) as date, count(RID) as count from messages` +
		` where channel = '#geekhack' group by date order by count` +
		` desc limit 10;`
	postByDayAll = `select DATE(Time) as date, count(RID) as count from messages` +
		` where channel = '#geekhack' group by date order by date`
	totalPosts = `select Nick, COUNT(Nick) as Posts from messages where channel` +
		` = '#geekhack' group by nick order by Posts desc limit 10;`
	postsByMinute = `select count from (select HOUR(Time)*60+MINUTE(Time) as` +
		` date, count(RID)/(SELECT DATEDIFF(NOW(), (SELECT MIN(Time)` +
		` from messages where channel = '#geekhack'))) as count from` +
		` messages where channel ='#geekhack' group by date order by date) as subquery;`
	topTenWords = `select Nick, ` + "`" + `%[1]s` + "`" + ` from words order by ` + "`" + `%[1]s` + "`" + ` desc limit 10;`
)

type Geekhack struct {
	db     *sql.DB
	config Config

	mutex                 sync.RWMutex // Protects:
	PostsByDay            []Tuple
	CurseWords            map[string][]Tuple
	TotalPosts            []Tuple
	PostsByMinute         []float64
	PostsByMinuteSmoothed []float64
	PostByDayAll          [][]int64
	PostByDayAllSmoothed  [][]int64
	age                   time.Time
}

type Tuple struct {
	Name  string
	Count int
}

func NewGeekhack(config Config) (*Geekhack, error) {
	db, err := sql.Open("mysql", config.DBConn)
	if err != nil {
		return nil, err
	}

	//err = db.Ping()
	if err != nil {
		return nil, err
	}

	geekhack := &Geekhack{
		CurseWords: make(map[string][]Tuple),
		db:         db,
		config:     config,
	}

	go geekhack.Update()
	go geekhack.Updater()

	return geekhack, nil
}

func (g *Geekhack) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	switch r.URL.Path {
	case "/geekhack/":
		g.Main(w, r)
	case "/geekhack/postsbyminute":
		g.pbmHandler(w, r)
	case "/geekhack/postsbydayall":
		g.pbdaHandler(w, r)
	default:
		http.NotFound(w, r)
	}
}

func (g *Geekhack) Main(w http.ResponseWriter, r *http.Request) {
	g.mutex.RLock()
	defer g.mutex.RUnlock()
	if err := templates.ExecuteTemplate(w, "geekhack.tmpl", g); err != nil {
		log.Println(err)
	}
}

func (g *Geekhack) pbmHandler(w http.ResponseWriter, r *http.Request) {
	g.mutex.RLock()
	defer g.mutex.RUnlock()
	type returnStruct struct {
		Type string    `json:"type,omitempty"`
		Name string    `json:"name"`
		Data []float64 `json:"data"`
	}

	jsonSource := []returnStruct{
		returnStruct{
			"",
			"Posts Per Minute",
			g.PostsByMinute,
		},
		returnStruct{
			"spline",
			"Posts Per Minute Smoothed",
			g.PostsByMinuteSmoothed,
		},
	}
	jsonData, err := json.Marshal(jsonSource)
	if err != nil {
		log.Println(err)
		http.Error(w, "Error generating minute data", 500)
	}
	w.Header().Set("Content-Type", "application/json")
	written, err := w.Write(jsonData)
	if written < len(jsonData) || err != nil {
		log.Println("Error writing response to client")
	}
}

func (g *Geekhack) pbdaHandler(w http.ResponseWriter, r *http.Request) {
	g.mutex.RLock()
	defer g.mutex.RUnlock()
	type returnStruct struct {
		Type string    `json:"type,omitempty"`
		Name string    `json:"name"`
		Data [][]int64 `json:"data"`
	}
	jsonSource := []returnStruct{
		returnStruct{
			Type: "",
			Name: "Posts Per Day All",
			Data: g.PostByDayAll,
		},
		returnStruct{
			Type: "spline",
			Name: "Posts Per Day All Smoothed",
			Data: g.PostByDayAllSmoothed,
		},
	}
	jsonData, err := json.Marshal(jsonSource)
	if err != nil {
		log.Println(err)
		http.Error(w, "Error generating day data", 500)
	}
	w.Header().Set("Content-Type", "application/json")
	written, err := w.Write(jsonData)
	if written < len(jsonData) || err != nil {
		log.Println("Error writing response to client")
	}
}

func (g *Geekhack) runQuery(query string) ([]Tuple, error) {
	var nick string
	var posts int
	var tuple []Tuple
	rows, err := g.db.Query(query)
	if err != nil {
		log.Println(err)
		return nil, err
	}
	for rows.Next() {
		if err := rows.Scan(&nick, &posts); err != nil {
			return nil, err
		}
		tuple = append(tuple, Tuple{nick, posts})
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	return tuple, nil
}

func (g *Geekhack) UpdateCurseWords() (map[string][]Tuple, error) {
	CurseWords := make(map[string][]Tuple)
	query, err := g.db.Query(`SELECT * FROM words LIMIT 1`)
	if err != nil {
		return nil, err
	}
	defer query.Close()
	words, err := query.Columns()
	if err != nil {
		return nil, err
	}
	for _, word := range words {
		if word == "Nick" {
			continue
		}
		tuple, err := g.runQuery(fmt.Sprintf(topTenWords, word))
		if err != nil {
			return nil, err
		}
		CurseWords[word] = tuple
	}
	return CurseWords, nil
}

func (g *Geekhack) UpdatePostByDayAll() ([][]int64, error) {
	var date string
	var posts int
	var returnValue [][]int64
	rows, err := g.db.Query(postByDayAll)
	if err != nil {
		return nil, err
	}
	for rows.Next() {
		if err := rows.Scan(&date, &posts); err != nil {
			return nil, err
		}
		convTime, err := time.Parse("2006-01-02", date)
		if err != nil {
			return nil, err
		}
		returnValue = append(returnValue, []int64{convTime.Unix() * 1000, int64(posts)})
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	return returnValue, nil
}

func (g *Geekhack) Update() {
	start := time.Now()
	PostsByDay, err := g.runQuery(postByDay)
	if err != nil {
		log.Println(err)
		return
	}

	TotalPosts, err := g.runQuery(totalPosts)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Posts updated in:", time.Since(start))

	start = time.Now()
	CurseWords, err := g.UpdateCurseWords()
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("CurseWords updated in:", time.Since(start))

	start = time.Now()
	PostsByMinute := []float64{}
	rows, err := g.db.Query(postsByMinute)
	if err != nil {
		log.Println(err)
		return
	}
	for rows.Next() {
		var posts float64
		if err := rows.Scan(&posts); err != nil {
			log.Println(err)
			return
		}
		PostsByMinute = append(PostsByMinute, posts)
	}
	if err := rows.Err(); err != nil {
		log.Println(err)
		return
	}
	PostsByMinuteSmoothed := movingAverage(movingAverage(movingAverage(PostsByMinute, 20), 10), 5)
	log.Println("PostsByMinute updated in:", time.Since(start))

	start = time.Now()
	PostsByDayAll, err := g.UpdatePostByDayAll()
	if err != nil {
		log.Println(err)
		return
	}
	PostsByDayAllSmoothed := averageWithTime(PostsByDayAll)
	log.Println("PostsByDayAll updated in:", time.Since(start))

	// Update the struct
	g.mutex.Lock()
	g.PostsByDay = PostsByDay
	g.TotalPosts = TotalPosts
	g.CurseWords = CurseWords
	g.PostsByMinute = PostsByMinute
	g.PostsByMinuteSmoothed = PostsByMinuteSmoothed
	g.PostByDayAll = PostsByDayAll
	g.PostByDayAllSmoothed = PostsByDayAllSmoothed
	g.age = time.Now()
	g.mutex.Unlock()
	// Finish update, need to unlock it
}

func averageWithTime(original [][]int64) [][]int64 {
	first := make([]int64, len(original))
	second := make([]float64, len(original))
	for key, value := range original {
		first[key] = value[0]
		second[key] = float64(value[1])
	}
	second = movingAverage(movingAverage(movingAverage(second, 20), 10), 5)
	result := make([][]int64, len(original))
	for i := 0; i < len(original); i++ {
		result[i] = []int64{first[i], int64(second[i])}
	}
	return result
}

func movingAverage(input []float64, size int) []float64 {
	var start, end int
	result := make([]float64, len(input))
	for i := 0; i < len(input); i++ {
		if i < size {
			start = 0
		} else {
			start = i - size
		}
		if size+i+1 > len(input) {
			end = len(input)
		} else {
			end = size + i + 1
		}
		for _, value := range input[start:end] {
			result[i] += value
		}
		result[i] /= float64(len(input[start:end]))
	}
	return result
}

func (g *Geekhack) Updater() {
	ticker := time.Tick(2 * time.Minute)
	for _ = range ticker {
		g.Update()
	}
}
<|endoftext|>


<commit_before>package yarn

import (
	"bufio"
	"fmt"
	"log"
	"math/rand"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// If there is a open ticket, does nothing, else kinits a new session
func kCheck(username, keytab string) error {
	if err := exec.Command("klist", "-s").Run(); err != nil { // returns no err if there's an open ticket
		if err = kInit(username, keytab); err != nil {
			return err
		}
	}

	return nil
}

func kInit(username, keytab string) error {

	if len(username) < 1 {
		return fmt.Errorf("A username is required to authenticate with Kerberos.")
	}
	if len(keytab) < 0 {
		return fmt.Errorf("A keytab is required to authenticate with Kerberos.")
	}

	cmd := exec.Command("kinit", username, "-k", "-t", keytab)
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return err
	}

	// Scanning for errors while cmd is running
	go func() {
		in := bufio.NewScanner(stderr)
		for in.Scan() {
			log.Printf(in.Text())
		}
	}()

	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("Failed to authenticate: %v", err)
	}

	return nil
}

func randStr(strlen int) string {
	rand.Seed(time.Now().UTC().UnixNano())
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	r := make([]byte, strlen)
	for i := 0; i < strlen; i++ {
		r[i] = chars[rand.Intn(len(chars))]
	}
	return string(r)
}

func cleanDir(dir string) error {
	log.Printf("Removing empty directory %s", dir)
	cmdClean := exec.Command("hadoop", "fs", "-rmdir", dir)
	if out, err := cmdClean.Output(); err != nil {
		log.Printf("Failed to remove outdir.\n%s", string(out))
		return err
	}
	return nil
}

// StartCloud starts a yarn cloud by shelling out to hadoop
//
// This process needs to store the job-ID to kill the process in the future
func StartCloud(size int, kerberos bool, mem, name, enginePath, username, keytab string) (string, string, string, error) {
	if kerberos {
		if err := kCheck(username, keytab); err != nil {
			return "", "", "", err
		}
	}

	out := "steam/" + name + "_" + randStr(5) + "_out"

	cmdArgs := []string{
		"jar",              //
		enginePath,         //
		"-jobname",         //
		"STEAM_" + name,    //
		"-n",               //
		strconv.Itoa(size), //
		"-mapperXmx",       //
		mem,                //
		"-output",          //
		out,                //
		"-disown",          //
	}

	log.Println("Attempting to start cloud...")

	cmd := exec.Command("hadoop", cmdArgs...)

	stdOut, err := cmd.StdoutPipe()
	if err != nil {
		return "", "", "", err
	}
	stdErr, err := cmd.StderrPipe()
	if err != nil {
		return "", "", "", err
	}

	if err := cmd.Start(); err != nil {
		return "", "", "", err
	}

	reNode := regexp.MustCompile(`H2O node (\d+\.\d+\.\d+\.\d+:\d+)`)
	reApID := regexp.MustCompile(`application_(\d+_\d+)`)
	var cmdErr, apID, addr string
	go func() {
		in := bufio.NewScanner(stdOut)
		for in.Scan() {
			log.Printf("--YARN-- %s\n", in.Text())
			if s := reNode.FindSubmatch([]byte(in.Text())); s != nil {
				addr = string(s[1])
			}
			if s := reApID.FindSubmatch([]byte(in.Text())); s != nil {
				apID = string(s[1])
			}
			if strings.Contains(in.Text(), "ERROR") {
				cmdErr += in.Text() + "\n"
			}
		}
	}()
	go func() {
		in := bufio.NewScanner(stdErr)
		for in.Scan() {
			cmdErr += in.Text() + "\n"
		}
	}()

	// TODO should be a ticket system, not halting
	if err := cmd.Wait(); err != nil {
		cleanDir(name + "_out")
		return "", "", "", fmt.Errorf("Failed to launch hadoop.\n%s", cmdErr)
	}
	return apID, addr, out, nil
}

// StopCloud kills a hadoop cloud by shelling out a command based on the job-ID
//
// In the future this
func StopCloud(kerberos bool, name, id, outdir, username, keytab string) error {
	if kerberos {
		if err := kCheck(username, keytab); err != nil {
			return err
		}
	}

	log.Println("Attempting to stop cloud...")
	cmd := exec.Command("hadoop", "job", "-kill", "job_"+id)
	stdOut, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stdErr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	e := make(chan error)
	var cmdOut, cmdErr string
	go func() {
		in := bufio.NewScanner(stdOut)
		for in.Scan() {
			log.Printf("--YARN-- %s\n", in.Text())
			cmdOut += in.Text() + "\n"
		}
	}()
	go func() {
		in := bufio.NewScanner(stdErr)
		for in.Scan() {
			cmdErr = cmdErr + in.Text() + "\n"
			if strings.Contains(in.Text(), "Exception") {
				e <- fmt.Errorf("Failed to shutdown hadoop.\n%s", in.Text())
			}
		}
	}()
	go func() {
		err := cmd.Wait()
		if err != nil {
			log.Printf("Failed to shutdown hadoop.\n%s", cmdErr)
			e <- fmt.Errorf("Failed to shutdown hadoop.\n%s", cmdErr)
		}
		e <- nil
	}()
	if err := <-e; err != nil {
		return err
	}

	if err := cleanDir(outdir); err != nil {
		return err
	}

	return nil
}
<commit_msg>STEAM-203: first pass yarn log cleanup<commit_after>package yarn

import (
	"bufio"
	"fmt"
	"log"
	"math/rand"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// If there is a open ticket, does nothing, else kinits a new session
func kCheck(username, keytab string) error {
	if err := exec.Command("klist", "-s").Run(); err != nil { // returns no err if there's an open ticket
		if err = kInit(username, keytab); err != nil {
			return err
		}
	}

	return nil
}

func kInit(username, keytab string) error {

	if len(username) < 1 {
		return fmt.Errorf("A username is required to authenticate with Kerberos.")
	}
	if len(keytab) < 0 {
		return fmt.Errorf("A keytab is required to authenticate with Kerberos.")
	}

	cmd := exec.Command("kinit", username, "-k", "-t", keytab)
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return err
	}

	// Scanning for errors while cmd is running
	go func() {
		in := bufio.NewScanner(stderr)
		for in.Scan() {
			log.Printf(in.Text())
		}
	}()

	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("Failed to authenticate: %v", err)
	}

	return nil
}

func randStr(strlen int) string {
	rand.Seed(time.Now().UTC().UnixNano())
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	r := make([]byte, strlen)
	for i := 0; i < strlen; i++ {
		r[i] = chars[rand.Intn(len(chars))]
	}
	return string(r)
}

func cleanDir(dir string) error {
	log.Printf("Removing empty directory %s", dir)
	cmdClean := exec.Command("hadoop", "fs", "-rmdir", dir)
	if out, err := cmdClean.Output(); err != nil {
		log.Printf("Failed to remove outdir.\n%s", string(out))
		return err
	}
	return nil
}

// StartCloud starts a yarn cloud by shelling out to hadoop
//
// This process needs to store the job-ID to kill the process in the future
func StartCloud(size int, kerberos bool, mem, name, enginePath, username, keytab string) (string, string, string, error) {
	if kerberos {
		if err := kCheck(username, keytab); err != nil {
			return "", "", "", err
		}
	}

	out := "steam/" + name + "_" + randStr(5) + "_out"

	cmdArgs := []string{
		"jar",              //
		enginePath,         //
		"-jobname",         //
		"STEAM_" + name,    //
		"-n",               //
		strconv.Itoa(size), //
		"-mapperXmx",       //
		mem,                //
		"-output",          //
		out,                //
		"-disown",          //
	}

	log.Println("Attempting to start cloud...")

	cmd := exec.Command("hadoop", cmdArgs...)

	stdOut, err := cmd.StdoutPipe()
	if err != nil {
		return "", "", "", err
	}
	// stdErr, err := cmd.StderrPipe()
	// if err != nil {
	// 	return "", "", "", err
	// }

	if err := cmd.Start(); err != nil {
		return "", "", "", err
	}

	reNode := regexp.MustCompile(`H2O node (\d+\.\d+\.\d+\.\d+:\d+)`)
	reApID := regexp.MustCompile(`application_(\d+_\d+)`)
	var cmdErr, apID, addr string
	go func() {
		in := bufio.NewScanner(stdOut)
		for in.Scan() {
			log.Printf("--YARN-- %s\n", in.Text())
			if s := reNode.FindSubmatch([]byte(in.Text())); s != nil {
				addr = string(s[1])
			}
			if s := reApID.FindSubmatch([]byte(in.Text())); s != nil {
				apID = string(s[1])
			}
			if strings.Contains(in.Text(), "ERROR") {
				cmdErr += in.Text() + "\n"
			}
		}
	}()
	// go func() {
	// 	in := bufio.NewScanner(stdErr)
	// 	for in.Scan() {
	// 		cmdErr += in.Text() + "\n"
	// 	}
	// }()

	// TODO should be a ticket system, not halting
	if err := cmd.Wait(); err != nil {
		cleanDir(name + "_out")
		return "", "", "", fmt.Errorf("Failed to launch hadoop.\n%s", cmdErr)
	}
	return apID, addr, out, nil
}

// StopCloud kills a hadoop cloud by shelling out a command based on the job-ID
//
// In the future this
func StopCloud(kerberos bool, name, id, outdir, username, keytab string) error {
	if kerberos {
		if err := kCheck(username, keytab); err != nil {
			return err
		}
	}

	log.Println("Attempting to stop cloud...")
	cmd := exec.Command("hadoop", "job", "-kill", "job_"+id)
	stdOut, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stdErr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	e := make(chan error)
	var cmdOut, cmdErr string
	go func() {
		in := bufio.NewScanner(stdOut)
		for in.Scan() {
			log.Printf("--YARN-- %s\n", in.Text())
			cmdOut += in.Text() + "\n"
		}
	}()
	go func() {
		in := bufio.NewScanner(stdErr)
		for in.Scan() {
			cmdErr = cmdErr + in.Text() + "\n"
			if strings.Contains(in.Text(), "Exception") {
				e <- fmt.Errorf("Failed to shutdown hadoop.\n%s", in.Text())
			}
		}
	}()
	go func() {
		err := cmd.Wait()
		if err != nil {
			log.Printf("Failed to shutdown hadoop.\n%s", cmdErr)
			e <- fmt.Errorf("Failed to shutdown hadoop.\n%s", cmdErr)
		}
		e <- nil
	}()
	if err := <-e; err != nil {
		return err
	}

	if err := cleanDir(outdir); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package tradingdays

import (
	"fmt"
	"testing"
	"time"

	"github.com/toomore/gogrs/utils"
)

func TestIsOpen(t *testing.T) {
	fmt.Println(exceptDays)
	if IsOpen(2015, 4, 17, time.Local) != true {
		t.Error("Should be `true`")
	}
	if IsOpen(2015, 4, 18, time.Local) != false {
		t.Error("Should be `false`")
	}
	if IsOpen(2015, 4, 20, time.Local) != true {
		t.Error("Should be `true`")
	}
	if IsOpen(2015, 5, 1, time.Local) != false {
		t.Error("Should be `false`")
	}
}

func BenchmarkIsOpen(t *testing.B) {
	for i := 0; i < t.N; i++ {
		IsOpen(2015, 4, 19, utils.TaipeiTimeZone)
	}
}
<commit_msg>Fixed testing.<commit_after>package tradingdays

import (
	"fmt"
	"testing"

	"github.com/toomore/gogrs/utils"
)

func TestIsOpen(t *testing.T) {
	fmt.Println(exceptDays)
	if IsOpen(2015, 4, 17, utils.TaipeiTimeZone) != true {
		t.Error("Should be `true`")
	}
	if IsOpen(2015, 4, 18, utils.TaipeiTimeZone) != false {
		t.Error("Should be `false`")
	}
	if IsOpen(2015, 4, 20, utils.TaipeiTimeZone) != true {
		t.Error("Should be `true`")
	}
	if IsOpen(2015, 5, 1, utils.TaipeiTimeZone) != false {
		t.Error("Should be `false`")
	}
}

func BenchmarkIsOpen(t *testing.B) {
	for i := 0; i < t.N; i++ {
		IsOpen(2015, 4, 19, utils.TaipeiTimeZone)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main_test

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"testing"
)

const (
	// Data directory, also the package directory for the test.
	testdata = "testdata"

	// Binaries we compile.
	testcover = "./testcover.exe"
)

var (
	// Files we use.
	testMain     = filepath.Join(testdata, "main.go")
	testTest     = filepath.Join(testdata, "test.go")
	coverInput   = filepath.Join(testdata, "test_line.go")
	coverOutput  = filepath.Join(testdata, "test_cover.go")
	coverProfile = filepath.Join(testdata, "profile.cov")

	// The HTML test files are in a separate directory
	// so they are a complete package.
	htmlProfile = filepath.Join(testdata, "html", "html.cov")
	htmlHTML    = filepath.Join(testdata, "html", "html.html")
	htmlGolden  = filepath.Join(testdata, "html", "html.golden")
)

var debug = flag.Bool("debug", false, "keep rewritten files for debugging")

// Run this shell script, but do it in Go so it can be run by "go test".
//
//	replace the word LINE with the line number < testdata/test.go > testdata/test_line.go
// 	go build -o ./testcover
// 	./testcover -mode=count -var=CoverTest -o ./testdata/test_cover.go testdata/test_line.go
//	go run ./testdata/main.go ./testdata/test.go
//
func TestCover(t *testing.T) {
	testenv.MustHaveGoBuild(t)

	// Read in the test file (testTest) and write it, with LINEs specified, to coverInput.
	file, err := ioutil.ReadFile(testTest)
	if err != nil {
		t.Fatal(err)
	}
	lines := bytes.Split(file, []byte("\n"))
	for i, line := range lines {
		lines[i] = bytes.Replace(line, []byte("LINE"), []byte(fmt.Sprint(i+1)), -1)
	}

	// Add a function that is not gofmt'ed. This used to cause a crash.
	// We don't put it in test.go because then we would have to gofmt it.
	// Issue 23927.
	lines = append(lines, []byte("func unFormatted() {"),
		[]byte("\tif true {"),
		[]byte("\t}else{"),
		[]byte("\t}"),
		[]byte("}"))
	lines = append(lines, []byte("func unFormatted2(b bool) {if b{}else{}}"))

	if err := ioutil.WriteFile(coverInput, bytes.Join(lines, []byte("\n")), 0666); err != nil {
		t.Fatal(err)
	}

	// defer removal of test_line.go
	if !*debug {
		defer os.Remove(coverInput)
	}

	// go build -o testcover
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", testcover)
	run(cmd, t)

	// defer removal of testcover
	defer os.Remove(testcover)

	// ./testcover -mode=count -var=thisNameMustBeVeryLongToCauseOverflowOfCounterIncrementStatementOntoNextLineForTest -o ./testdata/test_cover.go testdata/test_line.go
	cmd = exec.Command(testcover, "-mode=count", "-var=thisNameMustBeVeryLongToCauseOverflowOfCounterIncrementStatementOntoNextLineForTest", "-o", coverOutput, coverInput)
	run(cmd, t)

	// defer removal of ./testdata/test_cover.go
	if !*debug {
		defer os.Remove(coverOutput)
	}

	// go run ./testdata/main.go ./testdata/test.go
	cmd = exec.Command(testenv.GoToolPath(t), "run", testMain, coverOutput)
	run(cmd, t)

	file, err = ioutil.ReadFile(coverOutput)
	if err != nil {
		t.Fatal(err)
	}
	// compiler directive must appear right next to function declaration.
	if got, err := regexp.MatchString(".*\n//go:nosplit\nfunc someFunction().*", string(file)); err != nil || !got {
		t.Error("misplaced compiler directive")
	}
	// "go:linkname" compiler directive should be present.
	if got, err := regexp.MatchString(`.*go\:linkname some\_name some\_name.*`, string(file)); err != nil || !got {
		t.Error("'go:linkname' compiler directive not found")
	}

	// Other comments should be preserved too.
	c := ".*// This comment didn't appear in generated go code.*"
	if got, err := regexp.MatchString(c, string(file)); err != nil || !got {
		t.Errorf("non compiler directive comment %q not found", c)
	}
}

// TestDirectives checks that compiler directives are preserved and positioned
// correctly. Directives that occur before top-level declarations should remain
// above those declarations, even if they are not part of the block of
// documentation comments.
func TestDirectives(t *testing.T) {
	// Read the source file and find all the directives. We'll keep
	// track of whether each one has been seen in the output.
	testDirectives := filepath.Join(testdata, "directives.go")
	source, err := ioutil.ReadFile(testDirectives)
	if err != nil {
		t.Fatal(err)
	}
	sourceDirectives := findDirectives(source)

	// go tool cover -mode=atomic ./testdata/directives.go
	cmd := exec.Command(testenv.GoToolPath(t), "tool", "cover", "-mode=atomic", testDirectives)
	cmd.Stderr = os.Stderr
	output, err := cmd.Output()
	if err != nil {
		t.Fatal(err)
	}

	// Check that all directives are present in the output.
	outputDirectives := findDirectives(output)
	foundDirective := make(map[string]bool)
	for _, p := range sourceDirectives {
		foundDirective[p.name] = false
	}
	for _, p := range outputDirectives {
		if found, ok := foundDirective[p.name]; !ok {
			t.Errorf("unexpected directive in output: %s", p.text)
		} else if found {
			t.Errorf("directive found multiple times in output: %s", p.text)
		}
		foundDirective[p.name] = true
	}
	for name, found := range foundDirective {
		if !found {
			t.Errorf("missing directive: %s", name)
		}
	}

	// Check that directives that start with the name of top-level declarations
	// come before the beginning of the named declaration and after the end
	// of the previous declaration.
	fset := token.NewFileSet()
	astFile, err := parser.ParseFile(fset, testDirectives, output, 0)
	if err != nil {
		t.Fatal(err)
	}

	prevEnd := 0
	for _, decl := range astFile.Decls {
		var name string
		switch d := decl.(type) {
		case *ast.FuncDecl:
			name = d.Name.Name
		case *ast.GenDecl:
			if len(d.Specs) == 0 {
				// An empty group declaration. We still want to check that
				// directives can be associated with it, so we make up a name
				// to match directives in the test data.
				name = "_empty"
			} else if spec, ok := d.Specs[0].(*ast.TypeSpec); ok {
				name = spec.Name.Name
			}
		}
		pos := fset.Position(decl.Pos()).Offset
		end := fset.Position(decl.End()).Offset
		if name == "" {
			prevEnd = end
			continue
		}
		for _, p := range outputDirectives {
			if !strings.HasPrefix(p.name, name) {
				continue
			}
			if p.offset < prevEnd || pos < p.offset {
				t.Errorf("directive %s does not appear before definition %s", p.text, name)
			}
		}
		prevEnd = end
	}
}

type directiveInfo struct {
	text   string // full text of the comment, not including newline
	name   string // text after //go:
	offset int    // byte offset of first slash in comment
}

func findDirectives(source []byte) []directiveInfo {
	var directives []directiveInfo
	directivePrefix := []byte("\n//go:")
	offset := 0
	for {
		i := bytes.Index(source[offset:], directivePrefix)
		if i < 0 {
			break
		}
		i++ // skip newline
		p := source[offset+i:]
		j := bytes.IndexByte(p, '\n')
		if j < 0 {
			// reached EOF
			j = len(p)
		}
		directive := directiveInfo{
			text:   string(p[:j]),
			name:   string(p[len(directivePrefix)-1 : j]),
			offset: offset + i,
		}
		directives = append(directives, directive)
		offset += i + j
	}
	return directives
}

// Makes sure that `cover -func=profile.cov` reports accurate coverage.
// Issue #20515.
func TestCoverFunc(t *testing.T) {
	// go tool cover -func ./testdata/profile.cov
	cmd := exec.Command(testenv.GoToolPath(t), "tool", "cover", "-func", coverProfile)
	out, err := cmd.Output()
	if err != nil {
		if ee, ok := err.(*exec.ExitError); ok {
			t.Logf("%s", ee.Stderr)
		}
		t.Fatal(err)
	}

	if got, err := regexp.Match(".*total:.*100.0.*", out); err != nil || !got {
		t.Logf("%s", out)
		t.Errorf("invalid coverage counts. got=(%v, %v); want=(true; nil)", got, err)
	}
}

// Check that cover produces correct HTML.
// Issue #25767.
func TestCoverHTML(t *testing.T) {
	if _, err := exec.LookPath("diff"); err != nil {
		t.Skipf("skip test on %s: diff command is required", runtime.GOOS)
	}
	testenv.MustHaveGoBuild(t)
	if !*debug {
		defer os.Remove(testcover)
		defer os.Remove(htmlProfile)
		defer os.Remove(htmlHTML)
	}
	// go build -o testcover
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", testcover)
	run(cmd, t)
	// go test -coverprofile testdata/html/html.cov cmd/cover/testdata/html
	cmd = exec.Command(testenv.GoToolPath(t), "test", "-coverprofile", htmlProfile, "cmd/cover/testdata/html")
	run(cmd, t)
	// ./testcover -html testdata/html/html.cov -o testdata/html/html.html
	cmd = exec.Command(testcover, "-html", htmlProfile, "-o", htmlHTML)
	run(cmd, t)

	// Extract the parts of the HTML with comment markers,
	// and compare against a golden file.
	entireHTML, err := ioutil.ReadFile(htmlHTML)
	if err != nil {
		t.Fatal(err)
	}
	var out bytes.Buffer
	scan := bufio.NewScanner(bytes.NewReader(entireHTML))
	in := false
	for scan.Scan() {
		line := scan.Text()
		if strings.Contains(line, "// START") {
			in = true
		}
		if in {
			fmt.Fprintln(&out, line)
		}
		if strings.Contains(line, "// END") {
			in = false
		}
	}
	if err := ioutil.WriteFile(htmlHTML, out.Bytes(), 0644); err != nil {
		t.Fatal(err)
	}
	// diff -uw testdata/html/html.html testdata/html/html.golden
	cmd = exec.Command("diff", "-u", "-w", htmlHTML, htmlGolden)
	run(cmd, t)
}

func run(c *exec.Cmd, t *testing.T) {
	t.Helper()
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	err := c.Run()
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>cmd/cover: fix TestCoverHTML on Plan 9<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main_test

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"testing"
)

const (
	// Data directory, also the package directory for the test.
	testdata = "testdata"

	// Binaries we compile.
	testcover = "./testcover.exe"
)

var (
	// Files we use.
	testMain     = filepath.Join(testdata, "main.go")
	testTest     = filepath.Join(testdata, "test.go")
	coverInput   = filepath.Join(testdata, "test_line.go")
	coverOutput  = filepath.Join(testdata, "test_cover.go")
	coverProfile = filepath.Join(testdata, "profile.cov")

	// The HTML test files are in a separate directory
	// so they are a complete package.
	htmlProfile = filepath.Join(testdata, "html", "html.cov")
	htmlHTML    = filepath.Join(testdata, "html", "html.html")
	htmlGolden  = filepath.Join(testdata, "html", "html.golden")
)

var debug = flag.Bool("debug", false, "keep rewritten files for debugging")

// Run this shell script, but do it in Go so it can be run by "go test".
//
//	replace the word LINE with the line number < testdata/test.go > testdata/test_line.go
// 	go build -o ./testcover
// 	./testcover -mode=count -var=CoverTest -o ./testdata/test_cover.go testdata/test_line.go
//	go run ./testdata/main.go ./testdata/test.go
//
func TestCover(t *testing.T) {
	testenv.MustHaveGoBuild(t)

	// Read in the test file (testTest) and write it, with LINEs specified, to coverInput.
	file, err := ioutil.ReadFile(testTest)
	if err != nil {
		t.Fatal(err)
	}
	lines := bytes.Split(file, []byte("\n"))
	for i, line := range lines {
		lines[i] = bytes.Replace(line, []byte("LINE"), []byte(fmt.Sprint(i+1)), -1)
	}

	// Add a function that is not gofmt'ed. This used to cause a crash.
	// We don't put it in test.go because then we would have to gofmt it.
	// Issue 23927.
	lines = append(lines, []byte("func unFormatted() {"),
		[]byte("\tif true {"),
		[]byte("\t}else{"),
		[]byte("\t}"),
		[]byte("}"))
	lines = append(lines, []byte("func unFormatted2(b bool) {if b{}else{}}"))

	if err := ioutil.WriteFile(coverInput, bytes.Join(lines, []byte("\n")), 0666); err != nil {
		t.Fatal(err)
	}

	// defer removal of test_line.go
	if !*debug {
		defer os.Remove(coverInput)
	}

	// go build -o testcover
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", testcover)
	run(cmd, t)

	// defer removal of testcover
	defer os.Remove(testcover)

	// ./testcover -mode=count -var=thisNameMustBeVeryLongToCauseOverflowOfCounterIncrementStatementOntoNextLineForTest -o ./testdata/test_cover.go testdata/test_line.go
	cmd = exec.Command(testcover, "-mode=count", "-var=thisNameMustBeVeryLongToCauseOverflowOfCounterIncrementStatementOntoNextLineForTest", "-o", coverOutput, coverInput)
	run(cmd, t)

	// defer removal of ./testdata/test_cover.go
	if !*debug {
		defer os.Remove(coverOutput)
	}

	// go run ./testdata/main.go ./testdata/test.go
	cmd = exec.Command(testenv.GoToolPath(t), "run", testMain, coverOutput)
	run(cmd, t)

	file, err = ioutil.ReadFile(coverOutput)
	if err != nil {
		t.Fatal(err)
	}
	// compiler directive must appear right next to function declaration.
	if got, err := regexp.MatchString(".*\n//go:nosplit\nfunc someFunction().*", string(file)); err != nil || !got {
		t.Error("misplaced compiler directive")
	}
	// "go:linkname" compiler directive should be present.
	if got, err := regexp.MatchString(`.*go\:linkname some\_name some\_name.*`, string(file)); err != nil || !got {
		t.Error("'go:linkname' compiler directive not found")
	}

	// Other comments should be preserved too.
	c := ".*// This comment didn't appear in generated go code.*"
	if got, err := regexp.MatchString(c, string(file)); err != nil || !got {
		t.Errorf("non compiler directive comment %q not found", c)
	}
}

// TestDirectives checks that compiler directives are preserved and positioned
// correctly. Directives that occur before top-level declarations should remain
// above those declarations, even if they are not part of the block of
// documentation comments.
func TestDirectives(t *testing.T) {
	// Read the source file and find all the directives. We'll keep
	// track of whether each one has been seen in the output.
	testDirectives := filepath.Join(testdata, "directives.go")
	source, err := ioutil.ReadFile(testDirectives)
	if err != nil {
		t.Fatal(err)
	}
	sourceDirectives := findDirectives(source)

	// go tool cover -mode=atomic ./testdata/directives.go
	cmd := exec.Command(testenv.GoToolPath(t), "tool", "cover", "-mode=atomic", testDirectives)
	cmd.Stderr = os.Stderr
	output, err := cmd.Output()
	if err != nil {
		t.Fatal(err)
	}

	// Check that all directives are present in the output.
	outputDirectives := findDirectives(output)
	foundDirective := make(map[string]bool)
	for _, p := range sourceDirectives {
		foundDirective[p.name] = false
	}
	for _, p := range outputDirectives {
		if found, ok := foundDirective[p.name]; !ok {
			t.Errorf("unexpected directive in output: %s", p.text)
		} else if found {
			t.Errorf("directive found multiple times in output: %s", p.text)
		}
		foundDirective[p.name] = true
	}
	for name, found := range foundDirective {
		if !found {
			t.Errorf("missing directive: %s", name)
		}
	}

	// Check that directives that start with the name of top-level declarations
	// come before the beginning of the named declaration and after the end
	// of the previous declaration.
	fset := token.NewFileSet()
	astFile, err := parser.ParseFile(fset, testDirectives, output, 0)
	if err != nil {
		t.Fatal(err)
	}

	prevEnd := 0
	for _, decl := range astFile.Decls {
		var name string
		switch d := decl.(type) {
		case *ast.FuncDecl:
			name = d.Name.Name
		case *ast.GenDecl:
			if len(d.Specs) == 0 {
				// An empty group declaration. We still want to check that
				// directives can be associated with it, so we make up a name
				// to match directives in the test data.
				name = "_empty"
			} else if spec, ok := d.Specs[0].(*ast.TypeSpec); ok {
				name = spec.Name.Name
			}
		}
		pos := fset.Position(decl.Pos()).Offset
		end := fset.Position(decl.End()).Offset
		if name == "" {
			prevEnd = end
			continue
		}
		for _, p := range outputDirectives {
			if !strings.HasPrefix(p.name, name) {
				continue
			}
			if p.offset < prevEnd || pos < p.offset {
				t.Errorf("directive %s does not appear before definition %s", p.text, name)
			}
		}
		prevEnd = end
	}
}

type directiveInfo struct {
	text   string // full text of the comment, not including newline
	name   string // text after //go:
	offset int    // byte offset of first slash in comment
}

func findDirectives(source []byte) []directiveInfo {
	var directives []directiveInfo
	directivePrefix := []byte("\n//go:")
	offset := 0
	for {
		i := bytes.Index(source[offset:], directivePrefix)
		if i < 0 {
			break
		}
		i++ // skip newline
		p := source[offset+i:]
		j := bytes.IndexByte(p, '\n')
		if j < 0 {
			// reached EOF
			j = len(p)
		}
		directive := directiveInfo{
			text:   string(p[:j]),
			name:   string(p[len(directivePrefix)-1 : j]),
			offset: offset + i,
		}
		directives = append(directives, directive)
		offset += i + j
	}
	return directives
}

// Makes sure that `cover -func=profile.cov` reports accurate coverage.
// Issue #20515.
func TestCoverFunc(t *testing.T) {
	// go tool cover -func ./testdata/profile.cov
	cmd := exec.Command(testenv.GoToolPath(t), "tool", "cover", "-func", coverProfile)
	out, err := cmd.Output()
	if err != nil {
		if ee, ok := err.(*exec.ExitError); ok {
			t.Logf("%s", ee.Stderr)
		}
		t.Fatal(err)
	}

	if got, err := regexp.Match(".*total:.*100.0.*", out); err != nil || !got {
		t.Logf("%s", out)
		t.Errorf("invalid coverage counts. got=(%v, %v); want=(true; nil)", got, err)
	}
}

// Check that cover produces correct HTML.
// Issue #25767.
func TestCoverHTML(t *testing.T) {
	if _, err := exec.LookPath("diff"); err != nil {
		t.Skipf("skip test on %s: diff command is required", runtime.GOOS)
	}
	testenv.MustHaveGoBuild(t)
	if !*debug {
		defer os.Remove(testcover)
		defer os.Remove(htmlProfile)
		defer os.Remove(htmlHTML)
	}
	// go build -o testcover
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", testcover)
	run(cmd, t)
	// go test -coverprofile testdata/html/html.cov cmd/cover/testdata/html
	cmd = exec.Command(testenv.GoToolPath(t), "test", "-coverprofile", htmlProfile, "cmd/cover/testdata/html")
	run(cmd, t)
	// ./testcover -html testdata/html/html.cov -o testdata/html/html.html
	cmd = exec.Command(testcover, "-html", htmlProfile, "-o", htmlHTML)
	run(cmd, t)

	// Extract the parts of the HTML with comment markers,
	// and compare against a golden file.
	entireHTML, err := ioutil.ReadFile(htmlHTML)
	if err != nil {
		t.Fatal(err)
	}
	var out bytes.Buffer
	scan := bufio.NewScanner(bytes.NewReader(entireHTML))
	in := false
	for scan.Scan() {
		line := scan.Text()
		if strings.Contains(line, "// START") {
			in = true
		}
		if in {
			fmt.Fprintln(&out, line)
		}
		if strings.Contains(line, "// END") {
			in = false
		}
	}
	if err := ioutil.WriteFile(htmlHTML, out.Bytes(), 0644); err != nil {
		t.Fatal(err)
	}
	diff := "diff"
	if runtime.GOOS == "plan9" {
		diff = "/bin/ape/diff"
	}
	// diff -uw testdata/html/html.html testdata/html/html.golden
	cmd = exec.Command(diff, "-u", "-w", htmlHTML, htmlGolden)
	run(cmd, t)
}

func run(c *exec.Cmd, t *testing.T) {
	t.Helper()
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	err := c.Run()
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>