<commit_before>package main

import (
	"fmt"
	"io"
	"text/template"

	humanize "github.com/dustin/go-humanize"
	"github.com/vbatts/imgsrv/types"
)

var emptyInterface interface{}

var headTemplate = template.Must(template.New("head").Parse(headTemplateHTML))
var headTemplateHTML = `

  <link href="/assets/bootstrap.css" media="screen" rel="stylesheet" type="text/css" />

  <title>{{.title}}</title>

`

var navbarTemplate = template.Must(template.New("navbar").Parse(navbarTemplateHTML))
var navbarTemplateHTML = `

        filesrv

Home

Upload

URLie

All

            Other

Keywords

File ext

MD5s

`
var containerBeginTemplate = template.Must(template.New("containerBegin").Parse(containerBeginTemplateHTML))
var containerBeginTemplateHTML = `

`

var tailTemplate = template.Must(template.New("tail").Parse(tailTemplateHTML))
var tailTemplateHTML = `

{{if .}}
{{.footer}}
{{end}}

`

var formDeleteFileTemplate = template.Must(template.New("formDeleteFile").Parse(formDeleteFileTemplateHTML))
var formDeleteFileTemplateHTML = `

Get file from URL

{{if .}}

Are you sure?

no!

yes! delete!

{{else}}

ERROR: No File provided!

{{end}}

{{/* hero-unit */}}

{{/* span9 */}}
`

var formGetUrlTemplate = template.Must(template.New("formGetUrl").Parse(formGetUrlTemplateHTML))
var formGetUrlTemplateHTML = `

Get file from URL

<form enctype="multipart/form-data" action="/urlie" method="POST">

      <input type="text" name="url" placeholder="file URL">

      <input type="text" name="keywords" placeholder="keywords">(comma seperatated, no spaces)

      <input type="checkbox" name="rand" value="true">Randomize filename

      <input type="submit" value="Fetch File">

</form>

{{/* hero-unit */}}

{{/* span9 */}}
`

var formFileUploadTemplate = template.Must(template.New("formFileUpload").Parse(formFileUploadTemplateHTML))
var formFileUploadTemplateHTML = `

Upload a File

<form enctype="multipart/form-data" action="/upload" method="POST">

      <input type="file" name="filename" placeholder="filename">

      <input type="text" name="keywords" placeholder="keywords">(comma seperatated, no spaces)

      <input type="checkbox" name="rand" value="true">Randomize filename

      <input type="submit" value="Upload File">

</form>

{{/* hero-unit */}}

{{/* span9 */}}
`

var listTemplate = template.Must(template.New("list").Parse(listTemplateHTML))
var listTemplateHTML = `
{{if .}}

{{range .}}

{{.Filename}}
[keywords:{{range $key := .Metadata.Keywords}} {{$key}}{{end}}]
[md5: {{.Md5 | printf "%8.8s"}}...]

{{end}}

{{end}}
`

var tagcloudTemplate = template.Must(template.New("tagcloud").Parse(tagcloudTemplateHTML))
var tagcloudTemplateHTML = `
{{if .}}

{{range .}}
{{.Id}}
{{end}}

{{end}}

`

var fileViewImageTemplate = template.Must(template.New("file").Parse(fileViewImageTemplateHTML))
var fileViewImageTemplateHTML = `
{{if .}}

{{end}}
`

var fileViewAudioTemplate = template.Must(template.New("file").Parse(fileViewAudioTemplateHTML))
var fileViewAudioTemplateHTML = `
{{if .}}

<audio controls>
  <source src="/f/{{.Filename}}" />
  Your browser does not support the video tag.
</audio>

{{end}}
`

var fileViewVideoTemplate = template.Must(template.New("file").Parse(fileViewVideoTemplateHTML))
var fileViewVideoTemplateHTML = `
{{if .}}

<video width="320" height="240" controls>
  <source src="/f/{{.Filename}}" />
  Your browser does not support the video tag.
</video>

{{end}}
`

var fileViewTemplate = template.Must(template.New("file").Parse(fileViewTemplateHTML))
var fileViewTemplateHTML = `
{{if .}}
{{.Filename}}
{{end}}
`

var funcs = template.FuncMap{
	"humanBytes": humanize.Bytes,
	"humanTime":  humanize.Time,
}

var fileViewInfoTemplate = template.Must(template.New("file").Funcs(funcs).Parse(fileViewInfoTemplateHTML))
var fileViewInfoTemplateHTML = `
{{if .}}

[keywords:{{range $key := .Metadata.Keywords}} {{$key}}{{end}}]

[md5: {{.Md5}}]

[size: {{humanBytes .Length}}]

[UploadDate: {{.Metadata.TimeStamp}} ({{humanTime .Metadata.TimeStamp}})]

[Delete]
{{end}}
`

func DeleteFilePage(w io.Writer, filename string) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv :: delete"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	err = formDeleteFileTemplate.Execute(w, &filename)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func UrliePage(w io.Writer) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv :: URLie"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = formGetUrlTemplate.Execute(w, &emptyInterface)
	if err != nil {
		return err
	}
	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func UploadPage(w io.Writer) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv :: Upload"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	// main context of this page
	err = formFileUploadTemplate.Execute(w, &emptyInterface)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func ImageViewPage(w io.Writer, file types.File) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	if file.IsImage() {
		err = fileViewImageTemplate.Execute(w, file)
	} else if file.IsAudio() {
		err = fileViewAudioTemplate.Execute(w, file)
	} else if file.IsVideo() {
		err = fileViewVideoTemplate.Execute(w, file)
	} else {
		err = fileViewTemplate.Execute(w, file)
	}
	if err != nil {
		return err
	}
	err = fileViewInfoTemplate.Execute(w, file)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func ListFilesPage(w io.Writer, files []types.File) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	// main context of this page
	err = listTemplate.Execute(w, files)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func ListTagCloudPage(w io.Writer, ic []types.IdCount) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	// main context of this page
	err = tagcloudTemplate.Execute(w, ic)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func ErrorPage(w io.Writer, e error) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv :: ERROR"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	// main context of this page
	err = template.Must(template.New("serverError").Parse(`
  {{if .}}

ERROR!

      {{.Error()}}

 {{/* hero-unit */}}

 {{/* span9 */}}
  {{end}}
  `)).Execute(w, e)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}
<commit_msg>layouts: autoplay music and video<commit_after>package main

import (
	"fmt"
	"io"
	"text/template"

	humanize "github.com/dustin/go-humanize"
	"github.com/vbatts/imgsrv/types"
)

var emptyInterface interface{}

var headTemplate = template.Must(template.New("head").Parse(headTemplateHTML))
var headTemplateHTML = `

  <link href="/assets/bootstrap.css" media="screen" rel="stylesheet" type="text/css" />

  <title>{{.title}}</title>

`

var navbarTemplate = template.Must(template.New("navbar").Parse(navbarTemplateHTML))
var navbarTemplateHTML = `

        filesrv

Home

Upload

URLie

All

            Other

Keywords

File ext

MD5s

`
var containerBeginTemplate = template.Must(template.New("containerBegin").Parse(containerBeginTemplateHTML))
var containerBeginTemplateHTML = `

`

var tailTemplate = template.Must(template.New("tail").Parse(tailTemplateHTML))
var tailTemplateHTML = `

{{if .}}
{{.footer}}
{{end}}

`

var formDeleteFileTemplate = template.Must(template.New("formDeleteFile").Parse(formDeleteFileTemplateHTML))
var formDeleteFileTemplateHTML = `

Get file from URL

{{if .}}

Are you sure?

no!

yes! delete!

{{else}}

ERROR: No File provided!

{{end}}

{{/* hero-unit */}}

{{/* span9 */}}
`

var formGetUrlTemplate = template.Must(template.New("formGetUrl").Parse(formGetUrlTemplateHTML))
var formGetUrlTemplateHTML = `

Get file from URL

<form enctype="multipart/form-data" action="/urlie" method="POST">

      <input type="text" name="url" placeholder="file URL">

      <input type="text" name="keywords" placeholder="keywords">(comma seperatated, no spaces)

      <input type="checkbox" name="rand" value="true">Randomize filename

      <input type="submit" value="Fetch File">

</form>

{{/* hero-unit */}}

{{/* span9 */}}
`

var formFileUploadTemplate = template.Must(template.New("formFileUpload").Parse(formFileUploadTemplateHTML))
var formFileUploadTemplateHTML = `

Upload a File

<form enctype="multipart/form-data" action="/upload" method="POST">

      <input type="file" name="filename" placeholder="filename">

      <input type="text" name="keywords" placeholder="keywords">(comma seperatated, no spaces)

      <input type="checkbox" name="rand" value="true">Randomize filename

      <input type="submit" value="Upload File">

</form>

{{/* hero-unit */}}

{{/* span9 */}}
`

var listTemplate = template.Must(template.New("list").Parse(listTemplateHTML))
var listTemplateHTML = `
{{if .}}

{{range .}}

{{.Filename}}
[keywords:{{range $key := .Metadata.Keywords}} {{$key}}{{end}}]
[md5: {{.Md5 | printf "%8.8s"}}...]

{{end}}

{{end}}
`

var tagcloudTemplate = template.Must(template.New("tagcloud").Parse(tagcloudTemplateHTML))
var tagcloudTemplateHTML = `
{{if .}}

{{range .}}
{{.Id}}
{{end}}

{{end}}

`

var fileViewImageTemplate = template.Must(template.New("file").Parse(fileViewImageTemplateHTML))
var fileViewImageTemplateHTML = `
{{if .}}

{{end}}
`

var fileViewAudioTemplate = template.Must(template.New("file").Parse(fileViewAudioTemplateHTML))
var fileViewAudioTemplateHTML = `
{{if .}}

<audio controls>
  <source src="/f/{{.Filename}}" autoplay/>
  Your browser does not support the video tag.
</audio>

{{end}}
`

var fileViewVideoTemplate = template.Must(template.New("file").Parse(fileViewVideoTemplateHTML))
var fileViewVideoTemplateHTML = `
{{if .}}

<video width="320" height="240" controls>
  <source src="/f/{{.Filename}}" autoplay/>
  Your browser does not support the video tag.
</video>

{{end}}
`

var fileViewTemplate = template.Must(template.New("file").Parse(fileViewTemplateHTML))
var fileViewTemplateHTML = `
{{if .}}
{{.Filename}}
{{end}}
`

var funcs = template.FuncMap{
	"humanBytes": humanize.Bytes,
	"humanTime":  humanize.Time,
}

var fileViewInfoTemplate = template.Must(template.New("file").Funcs(funcs).Parse(fileViewInfoTemplateHTML))
var fileViewInfoTemplateHTML = `
{{if .}}

[keywords:{{range $key := .Metadata.Keywords}} {{$key}}{{end}}]

[md5: {{.Md5}}]

[size: {{humanBytes .Length}}]

[UploadDate: {{.Metadata.TimeStamp}} ({{humanTime .Metadata.TimeStamp}})]

[Delete]
{{end}}
`

func DeleteFilePage(w io.Writer, filename string) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv :: delete"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	err = formDeleteFileTemplate.Execute(w, &filename)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func UrliePage(w io.Writer) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv :: URLie"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = formGetUrlTemplate.Execute(w, &emptyInterface)
	if err != nil {
		return err
	}
	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func UploadPage(w io.Writer) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv :: Upload"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	// main context of this page
	err = formFileUploadTemplate.Execute(w, &emptyInterface)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func ImageViewPage(w io.Writer, file types.File) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	if file.IsImage() {
		err = fileViewImageTemplate.Execute(w, file)
	} else if file.IsAudio() {
		err = fileViewAudioTemplate.Execute(w, file)
	} else if file.IsVideo() {
		err = fileViewVideoTemplate.Execute(w, file)
	} else {
		err = fileViewTemplate.Execute(w, file)
	}
	if err != nil {
		return err
	}
	err = fileViewInfoTemplate.Execute(w, file)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func ListFilesPage(w io.Writer, files []types.File) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	// main context of this page
	err = listTemplate.Execute(w, files)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func ListTagCloudPage(w io.Writer, ic []types.IdCount) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	// main context of this page
	err = tagcloudTemplate.Execute(w, ic)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}

func ErrorPage(w io.Writer, e error) (err error) {
	err = headTemplate.Execute(w, map[string]string{"title": "FileSrv :: ERROR"})
	if err != nil {
		return err
	}
	err = navbarTemplate.Execute(w, nil)
	if err != nil {
		return err
	}
	err = containerBeginTemplate.Execute(w, nil)
	if err != nil {
		return err
	}

	// main context of this page
	err = template.Must(template.New("serverError").Parse(`
  {{if .}}

ERROR!

      {{.Error()}}

 {{/* hero-unit */}}

 {{/* span9 */}}
  {{end}}
  `)).Execute(w, e)
	if err != nil {
		return err
	}

	err = tailTemplate.Execute(w, map[string]string{"footer": fmt.Sprintf("Version: %s", VERSION)})
	if err != nil {
		return err
	}
	return
}
<|endoftext|>


<commit_before>package lazyboy

import (
	"errors"
	"github.com/robfig/revel"
)

var (
	Url      string
	Port     string
	Username string
	Password string
	Name     string
)

func AppInit() {
	var found bool
	if Url, found = revel.Config.String("couchdb.url"); !found {
		err := errors.New("lazyboy: Database URL not defined in app.conf")
		revel.ERROR.Panic(err)
	}

	if Port, found = revel.Config.String("couchdb.port"); !found {
		Port = "5984"
	}

	if Username, found = revel.Config.String("couchdb.username"); !found {
		Username = ""
	}

	if Password, found = revel.Config.String("couchdb.password"); !found {
		Password = ""
	}

	if Name, found = revel.Config.String("couchdb.name"); !found {
		err := errors.New("lazyboy: Database name not defined in app.conf")
		revel.ERROR.Panic(err)
	}
}
<commit_msg>added framework<commit_after>package lazyboy

import (
	"errors"
	"github.com/robfig/revel"
)

var (
	lbUrl      string
	lbPort     string
	lbUsername string
	lbPassword string
	lbName     string
)

func AppInit() {
	var found bool
	if lbUrl, found = revel.Config.String("couchdb.url"); !found {
		err := errors.New("lazyboy: Database URL not defined in app.conf")
		revel.ERROR.Panic(err)
	}

	if lbPort, found = revel.Config.String("couchdb.port"); !found {
		lbPort = "5984"
	}

	if lbUsername, found = revel.Config.String("couchdb.username"); !found {
		lbUsername = ""
	}

	if lbPassword, found = revel.Config.String("couchdb.password"); !found {
		lbPassword = ""
	}

	if lbName, found = revel.Config.String("couchdb.name"); !found {
		err := errors.New("lazyboy: Database name not defined in app.conf")
		revel.ERROR.Panic(err)
	}
}

type LazyboyController struct {
	*revel.Controller

}
<|endoftext|>


<commit_before>package plots

import (
	"fmt"
	"reflect"
	"sort"
)

var DEFAULT_STYLES = []string{"k.", "r.", "b.", "g."}

// Names of parameters relevant to MultiPlot
type GraphVars struct {
	// x and y axis of the plot
	X, Y string
	// additional parameters
	Params []string
	// Latex labels for additional parameters
	ParamLabels []string
	// if Y variable name is not given, use this to find value of Y
	YFunc func(data interface{}) float64
}

// Create a plot for each combination of vars.Params contained in data.
// graphParams[FILE_KEY] must specify a file path for output. grapherPath
// must specify the location of the Python graphing script.
func MultiPlot(data []interface{}, vars GraphVars, graphParams map[string]string, grapherPath string) error {
	data = stripNils(data)
	// we need to know the values of vars.Params to set up constraint
	allParamValues, err := extractParamValues(data, vars)
	if err != nil {
		return err
	}
	// iterate over combinations of parameters
	basePath := graphParams[FILE_KEY]
	primaryNames, primaryLabels, secondaries := paramCombinations(allParamValues, vars)
	for i := 0; i < len(primaryNames); i++ {
		// graphParams[FILE_KEY] needs to be modified for this combination of secondary params
		extraPath := ""
		for name, val := range secondaries[i] {
			extraPath = extraPath + fmt.Sprintf("%s_%f_", name, val)
		}
		graphParams[FILE_KEY] = basePath + extraPath

		series, primaryVals := ExtractSeries(data, []string{vars.X, vars.Y, primaryNames[i]}, secondaries[i], vars.YFunc)
		sp := MakeSeriesParams(primaryLabels[i], "%.2f", primaryVals, DEFAULT_STYLES)
		err := PlotMPL(series, graphParams, sp, grapherPath)
		if err != nil {
			return err
		}
	}

	return nil
}

func stripNils(data []interface{}) []interface{} {
	ret := make([]interface{}, 0)
	for _, d := range data {
		if d != nil {
			ret = append(ret, d)
		}
	}
	return ret
}

func extractParamValues(data []interface{}, vars GraphVars) (map[string][]float64, error) {
	// get values for vars.Params
	paramValues := make(map[string][]float64)
	for _, d := range data {
		dv := reflect.ValueOf(d)
		for _, p := range vars.Params {
			pv := dv.FieldByName(p)
			if !pv.IsValid() {
				return paramValues, fmt.Errorf("Invalid parameter name %s", p)
			}
			pf := pv.Float()
			_, ok := paramValues[p]
			if !ok {
				paramValues[p] = make([]float64, 0)
			}
			if !contains(paramValues[p], pf) {
				paramValues[p] = append(paramValues[p], pf)
			}
		}
	}
	// sort param values
	for _, v := range paramValues {
		sort.Float64s(v)
	}

	return paramValues, nil
}

// Return true if xs contains val and false otherwise
func contains(xs []float64, val float64) bool {
	for _, x := range xs {
		if x == val {
			return true
		}
	}
	return false
}

// Return primaryNames, primaryLabels, secondaries. All variables get a
// chance to be primary; while a variable is primary all possible
// combinations of secondary variables are iterated through.
func paramCombinations(allParamValues map[string][]float64, vars GraphVars) ([]string, []string, []map[string]float64) {
	if len(vars.Params) == 1 {
		primaryNames := []string{vars.Params[0]}
		primaryLabels := []string{vars.ParamLabels[0]}
		secondaries := []map[string]float64{map[string]float64{}}
		return primaryNames, primaryLabels, secondaries
	}
	primaryNames := make([]string, 0)
	primaryLabels := make([]string, 0)
	secondaries := make([]map[string]float64, 0)
	for i, primaryName := range vars.Params {
		secondaryNames := without(vars.Params, primaryName)
		combos := combinations(allParamValues, secondaryNames)
		for _, secondary := range combos {
			primaryNames = append(primaryNames, primaryName)
			primaryLabels = append(primaryLabels, vars.ParamLabels[i])
			secondaries = append(secondaries, secondary)
		}
	}
	return primaryNames, primaryLabels, secondaries
}

// Return a copy of vals with drop removed
func without(vals []string, drop string) []string {
	ret := make([]string, 0)
	for _, v := range vals {
		if v != drop {
			ret = append(ret, v)
		}
	}
	return ret
}

// Return all possible combinations of named parameters given in vals
func combinations(vals map[string][]float64, names []string) []map[string]float64 {
	if len(names) == 0 {
		return nil
	}
	if len(names) == 1 {
		ret := make([]map[string]float64, 0)
		for _, val := range vals[names[0]] {
			thisMap := make(map[string]float64)
			thisMap[names[0]] = val
			ret = append(ret, thisMap)
		}
		return ret
	}
	remainingNames := without(names, names[0])
	remainingCombos := combinations(vals, remainingNames)
	ret := make([]map[string]float64, 0)
	for _, combo := range remainingCombos {
		for _, val := range vals[names[0]] {
			thisMap := copyMap(combo)
			thisMap[names[0]] = val
			ret = append(ret, thisMap)
		}
	}
	return ret
}

// Return a copy of the given map
func copyMap(m map[string]float64) map[string]float64 {
	ret := make(map[string]float64)
	for k, v := range m {
		ret[k] = v
	}
	return ret
}
<commit_msg>extra digit for parameters in plots<commit_after>package plots

import (
	"fmt"
	"reflect"
	"sort"
)

var DEFAULT_STYLES = []string{"k.", "r.", "b.", "g."}

// Names of parameters relevant to MultiPlot
type GraphVars struct {
	// x and y axis of the plot
	X, Y string
	// additional parameters
	Params []string
	// Latex labels for additional parameters
	ParamLabels []string
	// if Y variable name is not given, use this to find value of Y
	YFunc func(data interface{}) float64
}

// Create a plot for each combination of vars.Params contained in data.
// graphParams[FILE_KEY] must specify a file path for output. grapherPath
// must specify the location of the Python graphing script.
func MultiPlot(data []interface{}, vars GraphVars, graphParams map[string]string, grapherPath string) error {
	data = stripNils(data)
	// we need to know the values of vars.Params to set up constraint
	allParamValues, err := extractParamValues(data, vars)
	if err != nil {
		return err
	}
	// iterate over combinations of parameters
	basePath := graphParams[FILE_KEY]
	primaryNames, primaryLabels, secondaries := paramCombinations(allParamValues, vars)
	for i := 0; i < len(primaryNames); i++ {
		// graphParams[FILE_KEY] needs to be modified for this combination of secondary params
		extraPath := ""
		for name, val := range secondaries[i] {
			extraPath = extraPath + fmt.Sprintf("%s_%f_", name, val)
		}
		graphParams[FILE_KEY] = basePath + extraPath

		series, primaryVals := ExtractSeries(data, []string{vars.X, vars.Y, primaryNames[i]}, secondaries[i], vars.YFunc)
		sp := MakeSeriesParams(primaryLabels[i], "%.3f", primaryVals, DEFAULT_STYLES)
		err := PlotMPL(series, graphParams, sp, grapherPath)
		if err != nil {
			return err
		}
	}

	return nil
}

func stripNils(data []interface{}) []interface{} {
	ret := make([]interface{}, 0)
	for _, d := range data {
		if d != nil {
			ret = append(ret, d)
		}
	}
	return ret
}

func extractParamValues(data []interface{}, vars GraphVars) (map[string][]float64, error) {
	// get values for vars.Params
	paramValues := make(map[string][]float64)
	for _, d := range data {
		dv := reflect.ValueOf(d)
		for _, p := range vars.Params {
			pv := dv.FieldByName(p)
			if !pv.IsValid() {
				return paramValues, fmt.Errorf("Invalid parameter name %s", p)
			}
			pf := pv.Float()
			_, ok := paramValues[p]
			if !ok {
				paramValues[p] = make([]float64, 0)
			}
			if !contains(paramValues[p], pf) {
				paramValues[p] = append(paramValues[p], pf)
			}
		}
	}
	// sort param values
	for _, v := range paramValues {
		sort.Float64s(v)
	}

	return paramValues, nil
}

// Return true if xs contains val and false otherwise
func contains(xs []float64, val float64) bool {
	for _, x := range xs {
		if x == val {
			return true
		}
	}
	return false
}

// Return primaryNames, primaryLabels, secondaries. All variables get a
// chance to be primary; while a variable is primary all possible
// combinations of secondary variables are iterated through.
func paramCombinations(allParamValues map[string][]float64, vars GraphVars) ([]string, []string, []map[string]float64) {
	if len(vars.Params) == 1 {
		primaryNames := []string{vars.Params[0]}
		primaryLabels := []string{vars.ParamLabels[0]}
		secondaries := []map[string]float64{map[string]float64{}}
		return primaryNames, primaryLabels, secondaries
	}
	primaryNames := make([]string, 0)
	primaryLabels := make([]string, 0)
	secondaries := make([]map[string]float64, 0)
	for i, primaryName := range vars.Params {
		secondaryNames := without(vars.Params, primaryName)
		combos := combinations(allParamValues, secondaryNames)
		for _, secondary := range combos {
			primaryNames = append(primaryNames, primaryName)
			primaryLabels = append(primaryLabels, vars.ParamLabels[i])
			secondaries = append(secondaries, secondary)
		}
	}
	return primaryNames, primaryLabels, secondaries
}

// Return a copy of vals with drop removed
func without(vals []string, drop string) []string {
	ret := make([]string, 0)
	for _, v := range vals {
		if v != drop {
			ret = append(ret, v)
		}
	}
	return ret
}

// Return all possible combinations of named parameters given in vals
func combinations(vals map[string][]float64, names []string) []map[string]float64 {
	if len(names) == 0 {
		return nil
	}
	if len(names) == 1 {
		ret := make([]map[string]float64, 0)
		for _, val := range vals[names[0]] {
			thisMap := make(map[string]float64)
			thisMap[names[0]] = val
			ret = append(ret, thisMap)
		}
		return ret
	}
	remainingNames := without(names, names[0])
	remainingCombos := combinations(vals, remainingNames)
	ret := make([]map[string]float64, 0)
	for _, combo := range remainingCombos {
		for _, val := range vals[names[0]] {
			thisMap := copyMap(combo)
			thisMap[names[0]] = val
			ret = append(ret, thisMap)
		}
	}
	return ret
}

// Return a copy of the given map
func copyMap(m map[string]float64) map[string]float64 {
	ret := make(map[string]float64)
	for k, v := range m {
		ret[k] = v
	}
	return ret
}
<|endoftext|>


<commit_before>package fls

import (
	"bytes"
	"io"
	"os"
	"strconv"
	"strings"
	"testing"
)

type T struct {
	*testing.T
}

const TestFileEOFPos = 588889

func init() {
	f, err := os.OpenFile("test.log", os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	for i := 0; i <= 99999; i++ {

		lineNum := strconv.Itoa(i)

		if i > 0 {
			lineNum = "\n" + lineNum
		}

		if _, err = f.WriteString(lineNum); err != nil {
			panic(err)
		}
	}
}

func TestSeekLineStart(t *testing.T) {
	myT := &T{t}

	f, err := os.OpenFile("test.log", os.O_CREATE|os.O_RDONLY, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()
	file := LineFile(f)

	_, err = file.SeekLine(-1, io.SeekStart)
	line := GetLine(file)
	myT.Ok(line, 0, err, true)

	_, err = file.SeekLine(0, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 0, err, false)

	_, err = file.SeekLine(1, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 1, err, false)

	_, err = file.SeekLine(100, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 100, err, false)

	// bigger than buffer
	_, err = file.SeekLine(10000, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 10000, err, false)

	_, err = file.SeekLine(50000, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 50000, err, false)

	_, err = file.SeekLine(90000, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 90000, err, false)

	_, err = file.SeekLine(100000, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 99999, err, true)
}

func TestSeekLineEnd(t *testing.T) {
	myT := &T{t}

	f, err := os.OpenFile("test.log", os.O_CREATE|os.O_RDONLY, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()
	file := LineFile(f)

	pos, _ := file.Seek(0, io.SeekEnd)

	// Test const TestFileEOFPos = 588889
	if pos != TestFileEOFPos {
		t.Error("\nEOF hit at unknown position: ", pos)
	}

	_, err = file.SeekLine(1, io.SeekEnd)
	line := GetLine(file)
	myT.Ok(line, 99999, err, true)

	_, err = file.SeekLine(0, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 99999, err, false)

	_, err = file.SeekLine(-1, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 99998, err, false)

	_, err = file.SeekLine(-100, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 99899, err, false)

	// bigger than buffer
	_, err = file.SeekLine(-10000, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 89999, err, false)

	_, err = file.SeekLine(-50000, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 49999, err, false)

	_, err = file.SeekLine(-90000, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 9999, err, false)

	_, err = file.SeekLine(-100000, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 0, err, true)
}

func TestSeekLineCurrent(t *testing.T) {
	myT := &T{t}

	f, err := os.OpenFile("test.log", os.O_CREATE|os.O_RDONLY, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()
	file := LineFile(f)

	_, err = file.SeekLine(-1, io.SeekCurrent)
	line := GetLine(file)
	myT.Ok(line, 0, err, true)

	_, err = file.SeekLine(0, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 0, err, false)

	_, err = file.SeekLine(1, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 1, err, false)

	_, err = file.SeekLine(100, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 101, err, false)

	// bigger than buffer
	_, err = file.SeekLine(10000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 10101, err, false)

	_, err = file.SeekLine(50000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 60101, err, false)

	_, err = file.SeekLine(50000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 99999, err, true)

	_, err = file.SeekLine(0, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 99999, err, false)

	_, err = file.SeekLine(-1, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 99998, err, false)

	_, err = file.SeekLine(-100, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 99898, err, false)

	// bigger than buffer
	_, err = file.SeekLine(-10000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 89898, err, false)

	_, err = file.SeekLine(-50000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 39898, err, false)

	_, err = file.SeekLine(-50000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 0, err, true)
}

// Test helper functions
func GetLine(file *File) int {
	pos, _ := file.Seek(0, io.SeekCurrent)

	if pos == TestFileEOFPos {
		return 99999
	}

	buf := make([]byte, 100)
	file.Read(buf)

	// strips null chars from end of buffer on EOF
	buf = bytes.Trim(buf, "\x00")

	s := string(buf)
	sp := strings.Split(s, "\n")

	// resets pos to pos before read
	file.Seek(pos, io.SeekStart)

	line, _ := strconv.Atoi(sp[0])
	return line
}

func (t *T) Ok(got int, expected int, err error, expectEOF bool) {
	if got != expected {
		t.Error("\nExpected line: ", expected, "\ngot: ", got)
	}

	if err != nil && err != io.EOF {
		t.Error("\nError: ", err)
	} else if expectEOF && err != io.EOF {
		t.Error("\nExpected to hit EOF")
	} else if !expectEOF && err == io.EOF {
		t.Error("\nDid not expect to hit EOF")
	}
}
<commit_msg>comment TestFileEOFPos<commit_after>package fls

import (
	"bytes"
	"io"
	"os"
	"strconv"
	"strings"
	"testing"
)

type T struct {
	*testing.T
}

// known EOF position for test.log
const TestFileEOFPos = 588889

func init() {
	f, err := os.OpenFile("test.log", os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	for i := 0; i <= 99999; i++ {

		lineNum := strconv.Itoa(i)

		if i > 0 {
			lineNum = "\n" + lineNum
		}

		if _, err = f.WriteString(lineNum); err != nil {
			panic(err)
		}
	}
}

func TestSeekLineStart(t *testing.T) {
	myT := &T{t}

	f, err := os.OpenFile("test.log", os.O_CREATE|os.O_RDONLY, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()
	file := LineFile(f)

	_, err = file.SeekLine(-1, io.SeekStart)
	line := GetLine(file)
	myT.Ok(line, 0, err, true)

	_, err = file.SeekLine(0, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 0, err, false)

	_, err = file.SeekLine(1, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 1, err, false)

	_, err = file.SeekLine(100, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 100, err, false)

	// bigger than buffer
	_, err = file.SeekLine(10000, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 10000, err, false)

	_, err = file.SeekLine(50000, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 50000, err, false)

	_, err = file.SeekLine(90000, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 90000, err, false)

	_, err = file.SeekLine(100000, io.SeekStart)
	line = GetLine(file)
	myT.Ok(line, 99999, err, true)
}

func TestSeekLineEnd(t *testing.T) {
	myT := &T{t}

	f, err := os.OpenFile("test.log", os.O_CREATE|os.O_RDONLY, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()
	file := LineFile(f)

	pos, _ := file.Seek(0, io.SeekEnd)

	// Test const TestFileEOFPos = 588889
	if pos != TestFileEOFPos {
		t.Error("\nEOF hit at unknown position: ", pos)
	}

	_, err = file.SeekLine(1, io.SeekEnd)
	line := GetLine(file)
	myT.Ok(line, 99999, err, true)

	_, err = file.SeekLine(0, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 99999, err, false)

	_, err = file.SeekLine(-1, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 99998, err, false)

	_, err = file.SeekLine(-100, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 99899, err, false)

	// bigger than buffer
	_, err = file.SeekLine(-10000, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 89999, err, false)

	_, err = file.SeekLine(-50000, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 49999, err, false)

	_, err = file.SeekLine(-90000, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 9999, err, false)

	_, err = file.SeekLine(-100000, io.SeekEnd)
	line = GetLine(file)
	myT.Ok(line, 0, err, true)
}

func TestSeekLineCurrent(t *testing.T) {
	myT := &T{t}

	f, err := os.OpenFile("test.log", os.O_CREATE|os.O_RDONLY, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()
	file := LineFile(f)

	_, err = file.SeekLine(-1, io.SeekCurrent)
	line := GetLine(file)
	myT.Ok(line, 0, err, true)

	_, err = file.SeekLine(0, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 0, err, false)

	_, err = file.SeekLine(1, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 1, err, false)

	_, err = file.SeekLine(100, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 101, err, false)

	// bigger than buffer
	_, err = file.SeekLine(10000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 10101, err, false)

	_, err = file.SeekLine(50000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 60101, err, false)

	_, err = file.SeekLine(50000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 99999, err, true)

	_, err = file.SeekLine(0, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 99999, err, false)

	_, err = file.SeekLine(-1, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 99998, err, false)

	_, err = file.SeekLine(-100, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 99898, err, false)

	// bigger than buffer
	_, err = file.SeekLine(-10000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 89898, err, false)

	_, err = file.SeekLine(-50000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 39898, err, false)

	_, err = file.SeekLine(-50000, io.SeekCurrent)
	line = GetLine(file)
	myT.Ok(line, 0, err, true)
}

// Test helper functions
func GetLine(file *File) int {
	pos, _ := file.Seek(0, io.SeekCurrent)

	if pos == TestFileEOFPos {
		return 99999
	}

	buf := make([]byte, 100)
	file.Read(buf)

	// strips null chars from end of buffer on EOF
	buf = bytes.Trim(buf, "\x00")

	s := string(buf)
	sp := strings.Split(s, "\n")

	// resets pos to pos before read
	file.Seek(pos, io.SeekStart)

	line, _ := strconv.Atoi(sp[0])
	return line
}

func (t *T) Ok(got int, expected int, err error, expectEOF bool) {
	if got != expected {
		t.Error("\nExpected line: ", expected, "\ngot: ", got)
	}

	if err != nil && err != io.EOF {
		t.Error("\nError: ", err)
	} else if expectEOF && err != io.EOF {
		t.Error("\nExpected to hit EOF")
	} else if !expectEOF && err == io.EOF {
		t.Error("\nDid not expect to hit EOF")
	}
}
<|endoftext|>


<commit_before>package seven5

import (
	"fmt"
	"net/http"
)

//generateStringPrinter creates a function suitable for use with a ServeMux's handle func.
func generateStringPrinter(content string, contentType string) func(http.ResponseWriter, *http.Request) {
	return generateBinPrinter([]byte(content), contentType)
}

//generateBinPrinter creates a function suitable for use with a ServeMux's handle func. It writes out the
//content type as a sequence bytes.
func generateBinPrinter(content []byte, contentType string) func(http.ResponseWriter, *http.Request) {
	return func(writer http.ResponseWriter, req *http.Request) {
		writer.Header().Add("Content-type", contentType)
		_, err := writer.Write(content)
		if err != nil {
			fmt.Printf("error writing constant binary string: %s\n", err)
		}
	}
}

//SetIcon creates a go handler in h that will return an icon to be displayed in response to /favicon.ico.
//The binaryIcon should be an array of bytes (usually created via 'seven5tool embedfile')
func SetIcon(mux *http.ServeMux, binaryIcon []byte) {
	mux.HandleFunc("/favicon.ico", generateBinPrinter(binaryIcon, "image/x-icon"))
}
<commit_msg>removed project craziness in favor of new buildpack style<commit_after><|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/siesta/neo4j"
	"github.com/streadway/amqp"
	"io/ioutil"
	oldNeo "koding/databases/neo4j"
	"koding/db/mongodb"
	"koding/tools/amqputil"
	"koding/tools/config"
	"labix.org/v2/mgo"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type strToInf map[string]interface{}

var (
	GRAPH_URL           = config.Current.Neo4j.Write + ":" + strconv.Itoa(config.Current.Neo4j.Port)
	EXCHANGENAME        = "graphFeederExchange"
	MAX_ITERATION_COUNT = 50
	GUEST_GROUP_ID      = "51f41f195f07655e560001c1"
)

func main() {
	log.Println("Sync worker started")

	amqpChannel := connectToRabbitMQ()
	log.Println("Connected to Rabbit")

	err := mongodb.Run("relationships", createQuery(amqpChannel))
	if err != nil {
		fmt.Println("Error >>>", err)
	}
}

func createQuery(amqpChannel *amqp.Channel) func(coll *mgo.Collection) error {

	return func(coll *mgo.Collection) error {
		filter := strToInf{
			"targetName": strToInf{"$nin": oldNeo.NotAllowedNames},
			"sourceName": strToInf{"$nin": oldNeo.NotAllowedNames},
		}
		query := coll.Find(filter)

		totalCount, err := query.Count()
		if err != nil {
			log.Println("Err while getting count, exiting", err)
			return err
		}

		skip := config.Skip
		// this is a starting point
		index := skip
		// this is the item count to be processed
		limit := config.Count
		// this will be the ending point
		count := index + limit

		var result oldNeo.Relationship

		iteration := 0
		for {
			// if we reach to the end of the all collection, exit
			if index >= totalCount {
				log.Println("All items are processed, exiting")
				break
			}

			// this is the max re-iterating count
			if iteration == MAX_ITERATION_COUNT {
				break
			}

			// if we processed all items then exit
			if index == count {
				break
			}

			iter := query.Skip(index).Limit(count - index).Iter()
			for iter.Next(&result) {
				time.Sleep(100 * time.Millisecond)

				if relationshipNeedsToBeSynced(result) {
					createRelationship(result, amqpChannel)
				}

				index++
				log.Println(index)
			}

			if err := iter.Close(); err != nil {
				log.Println(err)
			}

			if iter.Timeout() {
				continue
			}

			log.Printf("iter existed, starting over from %v  -- %v  item(s) are processsed on this iter", index+1, index-skip)
			iteration++
		}

		if iteration == MAX_ITERATION_COUNT {
			log.Printf("Max iteration count %v reached, exiting", iteration)
		}
		log.Printf("Synced %v entries on this process", index-skip)

		return nil
	}
}

func connectToRabbitMQ() *amqp.Channel {
	conn := amqputil.CreateConnection("syncWorker")
	amqpChannel, err := conn.Channel()
	if err != nil {
		panic(err)
	}
	return amqpChannel
}

func createRelationship(rel oldNeo.Relationship, amqpChannel *amqp.Channel) {
	data := make([]strToInf, 1)
	data[0] = strToInf{
		"_id":        rel.Id,
		"sourceId":   rel.SourceId,
		"sourceName": rel.SourceName,
		"targetId":   rel.TargetId,
		"targetName": rel.TargetName,
		"as":         rel.As,
	}

	eventData := strToInf{"event": "RelationshipSaved", "payload": data}

	neoMessage, err := json.Marshal(eventData)
	if err != nil {
		log.Println("unmarshall error")
		return
	}

	amqpChannel.Publish(
		EXCHANGENAME, // exchange name
		"",           // key
		false,        // mandatory
		false,        // immediate
		amqp.Publishing{
			Body: neoMessage,
		},
	)
}

func relationshipNeedsToBeSynced(result oldNeo.Relationship) bool {
	if result.SourceId.Hex() == GUEST_GROUP_ID || result.TargetId.Hex() == GUEST_GROUP_ID {
		return false
	}

	exists, sourceId := checkNodeExists(result.SourceId.Hex())
	if exists != true {
		logError(result, "No source node")
		return true
	}

	exists, targetId := checkNodeExists(result.TargetId.Hex())
	if exists != true {
		logError(result, "No target node")
		return true
	}

	// flip JGroup relationships since they take a long time to check
	var flipped = false
	if result.SourceName == "JGroup" {
		flipped = true

		tempId := sourceId
		sourceId = targetId
		targetId = tempId
	}

	exists = checkRelationshipExists(sourceId, targetId, result.As, flipped)
	if exists != true {
		logError(result, "No relationship")
		return true
	}

	// everything is fine
	return false
}

func getAndParse(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return body, nil
}

func checkRelationshipExists(sourceId, targetId, relType string, flipped bool) bool {
	url := fmt.Sprintf("%v/db/data/node/%v/relationships/all/%v", GRAPH_URL, sourceId, relType)

	body, err := getAndParse(url)
	if err != nil {
		return false
	}

	relResponse := make([]neo4j.RelationshipResponse, 1)
	err = json.Unmarshal(body, &relResponse)
	if err != nil {
		return false
	}

	var numberofRelsFound = 0
	var relIds []string

	for _, rl := range relResponse {
		var checkPos string
		if flipped {
			checkPos = rl.Start
		} else {
			checkPos = rl.End
		}

		id := strings.SplitAfter(checkPos, GRAPH_URL+"/db/data/node/")[1]
		if targetId == id {
			numberofRelsFound++
			relIds = append(relIds, getRelIdFromUrl(rl.Self))
		}
	}

	switch numberofRelsFound {
	case 0:
		return false
	case 1:
		return true
	default:
		log.Printf("multiple '%v' rel %v", relType, relIds)

		if relType == "member" || relType == "creator" || relType == "author" {
			deleteDuplicateRel(relIds[1:])
			log.Printf("deleted multiple '%v' rel", relType)
		}
	}

	return true
}

func deleteDuplicateRel(relIds []string) {
	for _, id := range relIds {
		rel := neo4j.Relationship{Id: id}

		neo4jConnection := neo4j.Connect(GRAPH_URL)
		batch := neo4jConnection.NewBatch().Delete(&rel)

		_, err := batch.Execute()
		if err != nil {
			log.Println("err deleting rel", err)
		}
	}
}

func checkNodeExists(id string) (bool, string) {
	url := fmt.Sprintf("%v/db/data/index/node/koding/id/%v", GRAPH_URL, id)
	body, err := getAndParse(url)
	if err != nil {
		return false, ""
	}

	nodeResponse := make([]neo4j.NodeResponse, 1)
	err = json.Unmarshal(body, &nodeResponse)
	if err != nil {
		return false, ""
	}

	if len(nodeResponse) < 1 {
		return false, ""
	}

	node := nodeResponse[0]
	idd := getNodeIdFromUrl(node.Self)

	nodeId := string(idd)
	if nodeId == "" {
		return false, ""
	}

	return true, nodeId
}

func getNodeIdFromUrl(nodeSelf string) string {
	return strings.SplitAfter(nodeSelf, GRAPH_URL+"/db/data/node/")[1]
}

func getRelIdFromUrl(nodeSelf string) string {
	return strings.SplitAfter(nodeSelf, GRAPH_URL+"/db/data/relationship/")[1]
}

func logError(result oldNeo.Relationship, errMsg string) {
	log.Printf("id: %v, type: %v, source: {%v %v} target: {%v %v}; err: %v", result.SourceId.Hex(), result.As, result.SourceId.Hex(), result.SourceName, result.TargetId.Hex(), result.TargetName, errMsg)
}
<commit_msg>[sync] Moved sleeping time to a constant<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/siesta/neo4j"
	"github.com/streadway/amqp"
	"io/ioutil"
	oldNeo "koding/databases/neo4j"
	"koding/db/mongodb"
	"koding/tools/amqputil"
	"koding/tools/config"
	"labix.org/v2/mgo"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type strToInf map[string]interface{}

var (
	GRAPH_URL           = config.Current.Neo4j.Write + ":" + strconv.Itoa(config.Current.Neo4j.Port)
	EXCHANGENAME        = "graphFeederExchange"
	MAX_ITERATION_COUNT = 50
	GUEST_GROUP_ID      = "51f41f195f07655e560001c1"
	SLEEPING_TIME_IN_MS = 100
)

func main() {
	log.Println("Sync worker started")

	amqpChannel := connectToRabbitMQ()
	log.Println("Connected to Rabbit")

	err := mongodb.Run("relationships", createQuery(amqpChannel))
	if err != nil {
		fmt.Println("Error >>>", err)
	}
}

func createQuery(amqpChannel *amqp.Channel) func(coll *mgo.Collection) error {

	return func(coll *mgo.Collection) error {
		filter := strToInf{
			"targetName": strToInf{"$nin": oldNeo.NotAllowedNames},
			"sourceName": strToInf{"$nin": oldNeo.NotAllowedNames},
		}
		query := coll.Find(filter)

		totalCount, err := query.Count()
		if err != nil {
			log.Println("Err while getting count, exiting", err)
			return err
		}

		skip := config.Skip
		// this is a starting point
		index := skip
		// this is the item count to be processed
		limit := config.Count
		// this will be the ending point
		count := index + limit

		var result oldNeo.Relationship

		iteration := 0
		for {
			// if we reach to the end of the all collection, exit
			if index >= totalCount {
				log.Println("All items are processed, exiting")
				break
			}

			// this is the max re-iterating count
			if iteration == MAX_ITERATION_COUNT {
				break
			}

			// if we processed all items then exit
			if index == count {
				break
			}

			iter := query.Skip(index).Limit(count - index).Iter()
			for iter.Next(&result) {
				time.Sleep(SLEEPING_TIME_IN_MS * time.Millisecond)

				if relationshipNeedsToBeSynced(result) || doNotCheck {
					createRelationship(result, amqpChannel)
				}

				index++
				log.Println(index)
			}

			if err := iter.Close(); err != nil {
				log.Println(err)
			}

			if iter.Timeout() {
				continue
			}

			log.Printf("iter existed, starting over from %v  -- %v  item(s) are processsed on this iter", index+1, index-skip)
			iteration++
		}

		if iteration == MAX_ITERATION_COUNT {
			log.Printf("Max iteration count %v reached, exiting", iteration)
		}
		log.Printf("Synced %v entries on this process", index-skip)

		return nil
	}
}

func connectToRabbitMQ() *amqp.Channel {
	conn := amqputil.CreateConnection("syncWorker")
	amqpChannel, err := conn.Channel()
	if err != nil {
		panic(err)
	}
	return amqpChannel
}

func createRelationship(rel oldNeo.Relationship, amqpChannel *amqp.Channel) {
	data := make([]strToInf, 1)
	data[0] = strToInf{
		"_id":        rel.Id,
		"sourceId":   rel.SourceId,
		"sourceName": rel.SourceName,
		"targetId":   rel.TargetId,
		"targetName": rel.TargetName,
		"as":         rel.As,
	}

	eventData := strToInf{"event": "RelationshipSaved", "payload": data}

	neoMessage, err := json.Marshal(eventData)
	if err != nil {
		log.Println("unmarshall error")
		return
	}

	amqpChannel.Publish(
		EXCHANGENAME, // exchange name
		"",           // key
		false,        // mandatory
		false,        // immediate
		amqp.Publishing{
			Body: neoMessage,
		},
	)
}

func relationshipNeedsToBeSynced(result oldNeo.Relationship) bool {
	if result.SourceId.Hex() == GUEST_GROUP_ID || result.TargetId.Hex() == GUEST_GROUP_ID {
		return false
	}

	exists, sourceId := checkNodeExists(result.SourceId.Hex())
	if exists != true {
		logError(result, "No source node")
		return true
	}

	exists, targetId := checkNodeExists(result.TargetId.Hex())
	if exists != true {
		logError(result, "No target node")
		return true
	}

	// flip JGroup relationships since they take a long time to check
	var flipped = false
	if result.SourceName == "JGroup" {
		flipped = true

		tempId := sourceId
		sourceId = targetId
		targetId = tempId
	}

	exists = checkRelationshipExists(sourceId, targetId, result.As, flipped)
	if exists != true {
		logError(result, "No relationship")
		return true
	}

	// everything is fine
	return false
}

func getAndParse(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return body, nil
}

func checkRelationshipExists(sourceId, targetId, relType string, flipped bool) bool {
	url := fmt.Sprintf("%v/db/data/node/%v/relationships/all/%v", GRAPH_URL, sourceId, relType)

	body, err := getAndParse(url)
	if err != nil {
		return false
	}

	relResponse := make([]neo4j.RelationshipResponse, 1)
	err = json.Unmarshal(body, &relResponse)
	if err != nil {
		return false
	}

	var numberofRelsFound = 0
	var relIds []string

	for _, rl := range relResponse {
		var checkPos string
		if flipped {
			checkPos = rl.Start
		} else {
			checkPos = rl.End
		}

		id := strings.SplitAfter(checkPos, GRAPH_URL+"/db/data/node/")[1]
		if targetId == id {
			numberofRelsFound++
			relIds = append(relIds, getRelIdFromUrl(rl.Self))
		}
	}

	switch numberofRelsFound {
	case 0:
		return false
	case 1:
		return true
	default:
		log.Printf("multiple '%v' rel %v", relType, relIds)

		if relType == "member" || relType == "creator" || relType == "author" {
			deleteDuplicateRel(relIds[1:])
			log.Printf("deleted multiple '%v' rel", relType)
		}
	}

	return true
}

func deleteDuplicateRel(relIds []string) {
	for _, id := range relIds {
		rel := neo4j.Relationship{Id: id}

		neo4jConnection := neo4j.Connect(GRAPH_URL)
		batch := neo4jConnection.NewBatch().Delete(&rel)

		_, err := batch.Execute()
		if err != nil {
			log.Println("err deleting rel", err)
		}
	}
}

func checkNodeExists(id string) (bool, string) {
	url := fmt.Sprintf("%v/db/data/index/node/koding/id/%v", GRAPH_URL, id)
	body, err := getAndParse(url)
	if err != nil {
		return false, ""
	}

	nodeResponse := make([]neo4j.NodeResponse, 1)
	err = json.Unmarshal(body, &nodeResponse)
	if err != nil {
		return false, ""
	}

	if len(nodeResponse) < 1 {
		return false, ""
	}

	node := nodeResponse[0]
	idd := getNodeIdFromUrl(node.Self)

	nodeId := string(idd)
	if nodeId == "" {
		return false, ""
	}

	return true, nodeId
}

func getNodeIdFromUrl(nodeSelf string) string {
	return strings.SplitAfter(nodeSelf, GRAPH_URL+"/db/data/node/")[1]
}

func getRelIdFromUrl(nodeSelf string) string {
	return strings.SplitAfter(nodeSelf, GRAPH_URL+"/db/data/relationship/")[1]
}

func logError(result oldNeo.Relationship, errMsg string) {
	log.Printf("id: %v, type: %v, source: {%v %v} target: {%v %v}; err: %v", result.SourceId.Hex(), result.As, result.SourceId.Hex(), result.SourceName, result.TargetId.Hex(), result.TargetName, errMsg)
}
<|endoftext|>


<commit_before>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tabletmanager

// This file handles the health check. It is enabled by passing a
// target_tablet_type command line parameter. The tablet will then go
// to the target tablet type if healthy, and to 'spare' if not.

import (
	"flag"
	"fmt"
	"html/template"
	"reflect"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/timer"
	"github.com/youtube/vitess/go/vt/servenv"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/topo/topoproto"
	"github.com/youtube/vitess/go/vt/topotools"

	pbt "github.com/youtube/vitess/go/vt/proto/topodata"
)

const (
	defaultDegradedThreshold  = time.Duration(30 * time.Second)
	defaultUnhealthyThreshold = time.Duration(2 * time.Hour)
)

var (
	healthCheckInterval = flag.Duration("health_check_interval", 20*time.Second, "Interval between health checks")
	targetTabletType    = flag.String("target_tablet_type", "", "The tablet type we are thriving to be when healthy. When not healthy, we'll go to spare.")
	degradedThreshold   = flag.Duration("degraded_threshold", defaultDegradedThreshold, "replication lag after which a replica is considered degraded")
	unhealthyThreshold  = flag.Duration("unhealthy_threshold", defaultUnhealthyThreshold, "replication lag  after which a replica is considered unhealthy")
)

// HealthRecord records one run of the health checker
type HealthRecord struct {
	Error            error
	ReplicationDelay time.Duration
	Time             time.Time
}

// Class returns a human-readable one word version of the health state.
func (r *HealthRecord) Class() string {
	switch {
	case r.Error != nil:
		return "unhealthy"
	case r.ReplicationDelay > *degradedThreshold:
		return "unhappy"
	default:
		return "healthy"
	}
}

// HTML returns a html version to be displayed on UIs
func (r *HealthRecord) HTML() template.HTML {
	switch {
	case r.Error != nil:
		return template.HTML(fmt.Sprintf("unhealthy: %v", r.Error))
	case r.ReplicationDelay > *degradedThreshold:
		return template.HTML(fmt.Sprintf("unhappy: %v behind on replication", r.ReplicationDelay))
	default:
		if r.ReplicationDelay > 0 {
			return template.HTML(fmt.Sprintf("healthy: only %v behind on replication", r.ReplicationDelay))
		}
		return template.HTML("healthy")
	}
}

// IsDuplicate implements history.Deduplicable
func (r *HealthRecord) IsDuplicate(other interface{}) bool {
	rother, ok := other.(*HealthRecord)
	if !ok {
		return false
	}
	if !reflect.DeepEqual(r.Error, rother.Error) {
		return false
	}
	unhealthy := r.ReplicationDelay > *degradedThreshold
	unhealthyOther := rother.ReplicationDelay > *degradedThreshold
	return (unhealthy && unhealthyOther) || (!unhealthy && !unhealthyOther)
}

// ConfigHTML returns a formatted summary of health checking config values.
func ConfigHTML() template.HTML {
	return template.HTML(fmt.Sprintf(
		"healthCheckInterval: %v; degradedThreshold: %v; unhealthyThreshold: %v",
		healthCheckInterval, degradedThreshold, unhealthyThreshold))
}

// IsRunningHealthCheck indicates if the agent is configured to run healthchecks.
func (agent *ActionAgent) IsRunningHealthCheck() bool {
	return *targetTabletType != ""
}

func (agent *ActionAgent) initHealthCheck() {
	if !agent.IsRunningHealthCheck() {
		log.Infof("No target_tablet_type specified, disabling any health check")
		return
	}

	tt, err := topoproto.ParseTabletType(*targetTabletType)
	if err != nil {
		log.Fatalf("Invalid target tablet type %v: %v", *targetTabletType, err)
	}

	log.Infof("Starting periodic health check every %v with target_tablet_type=%v", *healthCheckInterval, *targetTabletType)
	t := timer.NewTimer(*healthCheckInterval)
	servenv.OnTermSync(func() {
		// When we enter lameduck mode, we want to not call
		// the health check any more. After this returns, we
		// are guaranteed to not call it.
		log.Info("Stopping periodic health check timer")
		t.Stop()

		// Now we can finish up and force ourselves to not healthy.
		agent.terminateHealthChecks(tt)
	})
	t.Start(func() {
		agent.runHealthCheck(tt)
	})
	t.Trigger()
}

// runHealthCheck takes the action mutex, runs the health check,
// and if we need to change our state, do it.
// If we are the master, we don't change our type, healthy or not.
// If we are not the master, we change to spare if not healthy,
// or to the passed in targetTabletType if healthy.
//
// Note we only update the topo record if we need to, that is if our type or
// health details changed.
//
// This will not change the BinlogPlayerMap, but if it is not empty,
// we will think we should not be running the query service.
//
// This will not change the TabletControl record, but will use it
// to see if we should be running the query service.
func (agent *ActionAgent) runHealthCheck(targetTabletType pbt.TabletType) {
	agent.actionMutex.Lock()
	defer agent.actionMutex.Unlock()

	// read the current tablet record and tablet control
	agent.mutex.Lock()
	tablet := agent._tablet
	tabletControl := agent._tabletControl
	agent.mutex.Unlock()

	// figure out if we should be running the query service
	shouldQueryServiceBeRunning := false
	if topo.IsRunningQueryService(targetTabletType) && !agent.BinlogPlayerMap.isRunningFilteredReplication() {
		shouldQueryServiceBeRunning = true
		if tabletControl != nil {
			if tabletControl.DisableQueryService {
				shouldQueryServiceBeRunning = false
			}
		}
	}

	// run the health check
	isSlaveType := true
	if tablet.Type == pbt.TabletType_MASTER {
		isSlaveType = false
	}
	replicationDelay, err := agent.HealthReporter.Report(isSlaveType, shouldQueryServiceBeRunning)
	health := make(map[string]string)
	if err == nil {
		if replicationDelay > *unhealthyThreshold {
			err = fmt.Errorf("reported replication lag: %v higher than unhealthy threshold: %v", replicationDelay.Seconds(), unhealthyThreshold.Seconds())
		} else if replicationDelay > *degradedThreshold {
			health[topo.ReplicationLag] = topo.ReplicationLagHigh
		}
	}
	agent.lastHealthMapCount.Set(int64(len(health)))

	// Figure out if we should be running QueryService, see if we are,
	// and reconcile.
	if err != nil {
		if tablet.Type != pbt.TabletType_WORKER {
			// We are not healthy and must shut down QueryService.
			// At the moment, the only exception to this are "worker" tablets which
			// still must serve queries e.g. as source tablet during a "SplitClone".
			shouldQueryServiceBeRunning = false
		}
	}
	isQueryServiceRunning := agent.QueryServiceControl.IsServing()
	if shouldQueryServiceBeRunning {
		if !isQueryServiceRunning {
			// send the type we want to be, not the type we are
			desiredType := tablet.Type
			if desiredType == pbt.TabletType_SPARE {
				desiredType = targetTabletType
			}

			// we remember this new possible error
			err = agent.allowQueries(desiredType)
		}
	} else {
		if isQueryServiceRunning {
			// We are not healthy or should not be running the
			// query service, shut it down.
			// Note this is possibly sending 'spare' as
			// the tablet type, we will clean it up later.
			agent.disallowQueries(tablet.Tablet.Type,
				fmt.Sprintf("health-check failure(%v)", err),
			)
		}
	}

	// save the health record
	record := &HealthRecord{
		Error:            err,
		ReplicationDelay: replicationDelay,
		Time:             time.Now(),
	}
	agent.History.Add(record)

	// try to figure out the mysql port if we don't have it yet
	if _, ok := tablet.PortMap["mysql"]; !ok {
		// we don't know the port, try to get it from mysqld
		mysqlPort, err := agent.MysqlDaemon.GetMysqlPort()
		if err != nil {
			// Don't log if we're already in a waiting-for-mysql state.
			agent.mutex.Lock()
			if !agent._waitingForMysql {
				log.Warningf("Can't get mysql port, won't populate Tablet record in topology (will retry silently at healthcheck interval %v): %v", *healthCheckInterval, err)
				agent._waitingForMysql = true
			}
			agent.mutex.Unlock()
		} else {
			log.Infof("Updating tablet mysql port to %v", mysqlPort)
			if err := agent.TopoServer.UpdateTabletFields(agent.batchCtx, tablet.Alias, func(tablet *pbt.Tablet) error {
				tablet.PortMap["mysql"] = mysqlPort
				return nil
			}); err != nil {
				log.Infof("Error updating mysql port in tablet record, will try again: %v", err)
				return
			}

			// save the port so we don't update it again next time
			// we do the health check.
			agent.mutex.Lock()
			agent._tablet.PortMap["mysql"] = mysqlPort
			agent._waitingForMysql = false
			agent.mutex.Unlock()
		}
	}

	// remember our health status
	agent.mutex.Lock()
	agent._healthy = err
	agent._healthyTime = time.Now()
	agent._replicationDelay = replicationDelay
	agent.mutex.Unlock()

	// send it to our observers
	agent.broadcastHealth()

	// Update our topo.Server state, start with no change
	newTabletType := tablet.Type
	if err != nil {
		// The tablet is not healthy, let's see what we need to do
		if tablet.Type != targetTabletType {
			if tablet.Type != pbt.TabletType_SPARE {
				// we only log if we're not in spare,
				// as the spare state is normal for a
				// failed health check.
				log.Infof("Tablet not healthy and in state %v, not changing it: %v", tablet.Type, err)
			}
			return
		}

		// Note that if the query service is running, we may
		// need to stop it. The post-action callback will do
		// it, and it will be done after we change our state,
		// so it's the right order, let it do it.
		log.Infof("Tablet not healthy, converting it from %v to spare: %v", targetTabletType, err)
		newTabletType = pbt.TabletType_SPARE
	} else {
		// We are healthy, maybe with health, see if we need
		// to update the record. We only change from spare to
		// our target type.
		if tablet.Type == pbt.TabletType_SPARE {
			newTabletType = targetTabletType
		}
		if tablet.Type == newTabletType && topo.IsHealthEqual(health, tablet.HealthMap) {
			// no change in health, not logging anything,
			// and we're done
			return
		}

		// we need to update our state
		log.Infof("Updating tablet record as healthy type %v -> %v with health details %v -> %v", tablet.Type, newTabletType, tablet.HealthMap, health)
	}

	// Change the Type, update the health. Note we pass in a map
	// that's not nil, meaning if it's empty, we will clear it.
	if err := topotools.ChangeType(agent.batchCtx, agent.TopoServer, tablet.Alias, newTabletType, health); err != nil {
		log.Infof("Error updating tablet record: %v", err)
		return
	}
	tablet.HealthMap = health
	tablet.Type = newTabletType

	// Rebuild the serving graph in our cell, only if we're dealing with
	// a serving type
	if err := agent.updateServingGraph(tablet, targetTabletType); err != nil {
		log.Warningf("updateServingGraph failed (will still run post action callbacks, serving graph might be out of date): %v", err)
	}

	// run the post action callbacks, not much we can do with returned error
	if err := agent.refreshTablet(agent.batchCtx, "healthcheck"); err != nil {
		log.Warningf("refreshTablet failed: %v", err)
	}
}

// terminateHealthChecks is called when we enter lame duck mode.
// We will clean up our state, and shut down query service.
// We only do something if we are in targetTabletType state, and then
// we just go to spare.
func (agent *ActionAgent) terminateHealthChecks(targetTabletType pbt.TabletType) {
	agent.actionMutex.Lock()
	defer agent.actionMutex.Unlock()
	log.Info("agent.terminateHealthChecks is starting")

	// read the current tablet record
	tablet := agent.Tablet()
	if tablet.Type != targetTabletType {
		log.Infof("Tablet in state %v, not changing it", tablet.Type)
		return
	}

	// Change the Type to spare, update the health. Note we pass in a map
	// that's not nil, meaning we will clear it.
	if err := topotools.ChangeType(agent.batchCtx, agent.TopoServer, tablet.Alias, pbt.TabletType_SPARE, make(map[string]string)); err != nil {
		log.Infof("Error updating tablet record: %v", err)
		return
	}
	// The change above succeeded, so update our local copy.
	tablet, err := agent.updateTabletFromTopo(agent.batchCtx)
	if err != nil {
		log.Infof("Error re-reading tablet record: %v", err)
		return
	}

	// Update the serving graph in our cell, only if we're dealing with
	// a serving type
	if err := agent.updateServingGraph(tablet, targetTabletType); err != nil {
		log.Warningf("updateServingGraph failed (will still run post action callbacks, serving graph might be out of date): %v", err)
	}

	// We've already rebuilt the shard, which is the only reason we registered
	// ourself as OnTermSync (synchronous). The rest can be done asynchronously.
	go func() {
		// Run the post action callbacks (let them shutdown the query service)
		if err := agent.refreshTablet(agent.batchCtx, "terminatehealthcheck"); err != nil {
			log.Warningf("refreshTablet failed: %v", err)
		}
	}()
}

// updateServingGraph will update the serving graph if we need to.
func (agent *ActionAgent) updateServingGraph(tablet *topo.TabletInfo, targetTabletType pbt.TabletType) error {
	if topo.IsInServingGraph(targetTabletType) {
		if err := topotools.UpdateTabletEndpoints(agent.batchCtx, agent.TopoServer, tablet.Tablet); err != nil {
			return fmt.Errorf("UpdateTabletEndpoints failed: %v", err)
		}
	}
	return nil
}
<commit_msg>Renaming a couple variables.<commit_after>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tabletmanager

// This file handles the health check. It is enabled by passing a
// target_tablet_type command line parameter. The tablet will then go
// to the target tablet type if healthy, and to 'spare' if not.

import (
	"flag"
	"fmt"
	"html/template"
	"reflect"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/timer"
	"github.com/youtube/vitess/go/vt/servenv"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/topo/topoproto"
	"github.com/youtube/vitess/go/vt/topotools"

	pbt "github.com/youtube/vitess/go/vt/proto/topodata"
)

const (
	defaultDegradedThreshold  = time.Duration(30 * time.Second)
	defaultUnhealthyThreshold = time.Duration(2 * time.Hour)
)

var (
	healthCheckInterval = flag.Duration("health_check_interval", 20*time.Second, "Interval between health checks")
	targetTabletType    = flag.String("target_tablet_type", "", "The tablet type we are thriving to be when healthy. When not healthy, we'll go to spare.")
	degradedThreshold   = flag.Duration("degraded_threshold", defaultDegradedThreshold, "replication lag after which a replica is considered degraded")
	unhealthyThreshold  = flag.Duration("unhealthy_threshold", defaultUnhealthyThreshold, "replication lag  after which a replica is considered unhealthy")
)

// HealthRecord records one run of the health checker
type HealthRecord struct {
	Error            error
	ReplicationDelay time.Duration
	Time             time.Time
}

// Class returns a human-readable one word version of the health state.
func (r *HealthRecord) Class() string {
	switch {
	case r.Error != nil:
		return "unhealthy"
	case r.ReplicationDelay > *degradedThreshold:
		return "unhappy"
	default:
		return "healthy"
	}
}

// HTML returns a html version to be displayed on UIs
func (r *HealthRecord) HTML() template.HTML {
	switch {
	case r.Error != nil:
		return template.HTML(fmt.Sprintf("unhealthy: %v", r.Error))
	case r.ReplicationDelay > *degradedThreshold:
		return template.HTML(fmt.Sprintf("unhappy: %v behind on replication", r.ReplicationDelay))
	default:
		if r.ReplicationDelay > 0 {
			return template.HTML(fmt.Sprintf("healthy: only %v behind on replication", r.ReplicationDelay))
		}
		return template.HTML("healthy")
	}
}

// IsDuplicate implements history.Deduplicable
func (r *HealthRecord) IsDuplicate(other interface{}) bool {
	rother, ok := other.(*HealthRecord)
	if !ok {
		return false
	}
	if !reflect.DeepEqual(r.Error, rother.Error) {
		return false
	}
	unhealthy := r.ReplicationDelay > *degradedThreshold
	unhealthyOther := rother.ReplicationDelay > *degradedThreshold
	return (unhealthy && unhealthyOther) || (!unhealthy && !unhealthyOther)
}

// ConfigHTML returns a formatted summary of health checking config values.
func ConfigHTML() template.HTML {
	return template.HTML(fmt.Sprintf(
		"healthCheckInterval: %v; degradedThreshold: %v; unhealthyThreshold: %v",
		healthCheckInterval, degradedThreshold, unhealthyThreshold))
}

// IsRunningHealthCheck indicates if the agent is configured to run healthchecks.
func (agent *ActionAgent) IsRunningHealthCheck() bool {
	return *targetTabletType != ""
}

func (agent *ActionAgent) initHealthCheck() {
	if !agent.IsRunningHealthCheck() {
		log.Infof("No target_tablet_type specified, disabling any health check")
		return
	}

	tt, err := topoproto.ParseTabletType(*targetTabletType)
	if err != nil {
		log.Fatalf("Invalid target tablet type %v: %v", *targetTabletType, err)
	}

	log.Infof("Starting periodic health check every %v with target_tablet_type=%v", *healthCheckInterval, *targetTabletType)
	t := timer.NewTimer(*healthCheckInterval)
	servenv.OnTermSync(func() {
		// When we enter lameduck mode, we want to not call
		// the health check any more. After this returns, we
		// are guaranteed to not call it.
		log.Info("Stopping periodic health check timer")
		t.Stop()

		// Now we can finish up and force ourselves to not healthy.
		agent.terminateHealthChecks(tt)
	})
	t.Start(func() {
		agent.runHealthCheck(tt)
	})
	t.Trigger()
}

// runHealthCheck takes the action mutex, runs the health check,
// and if we need to change our state, do it.
// If we are the master, we don't change our type, healthy or not.
// If we are not the master, we change to spare if not healthy,
// or to the passed in targetTabletType if healthy.
//
// Note we only update the topo record if we need to, that is if our type or
// health details changed.
//
// This will not change the BinlogPlayerMap, but if it is not empty,
// we will think we should not be running the query service.
//
// This will not change the TabletControl record, but will use it
// to see if we should be running the query service.
func (agent *ActionAgent) runHealthCheck(targetTabletType pbt.TabletType) {
	agent.actionMutex.Lock()
	defer agent.actionMutex.Unlock()

	// read the current tablet record and tablet control
	agent.mutex.Lock()
	tablet := agent._tablet
	tabletControl := agent._tabletControl
	agent.mutex.Unlock()

	// figure out if we should be running the query service
	shouldBeServing := false
	if topo.IsRunningQueryService(targetTabletType) && !agent.BinlogPlayerMap.isRunningFilteredReplication() {
		shouldBeServing = true
		if tabletControl != nil {
			if tabletControl.DisableQueryService {
				shouldBeServing = false
			}
		}
	}

	// run the health check
	isSlaveType := true
	if tablet.Type == pbt.TabletType_MASTER {
		isSlaveType = false
	}
	replicationDelay, err := agent.HealthReporter.Report(isSlaveType, shouldBeServing)
	health := make(map[string]string)
	if err == nil {
		if replicationDelay > *unhealthyThreshold {
			err = fmt.Errorf("reported replication lag: %v higher than unhealthy threshold: %v", replicationDelay.Seconds(), unhealthyThreshold.Seconds())
		} else if replicationDelay > *degradedThreshold {
			health[topo.ReplicationLag] = topo.ReplicationLagHigh
		}
	}
	agent.lastHealthMapCount.Set(int64(len(health)))

	// Figure out if we should be running QueryService, see if we are,
	// and reconcile.
	if err != nil {
		if tablet.Type != pbt.TabletType_WORKER {
			// We are not healthy and must shut down QueryService.
			// At the moment, the only exception to this are "worker" tablets which
			// still must serve queries e.g. as source tablet during a "SplitClone".
			shouldBeServing = false
		}
	}
	isServing := agent.QueryServiceControl.IsServing()
	if shouldBeServing {
		if !isServing {
			// send the type we want to be, not the type we are
			desiredType := tablet.Type
			if desiredType == pbt.TabletType_SPARE {
				desiredType = targetTabletType
			}

			// we remember this new possible error
			err = agent.allowQueries(desiredType)
		}
	} else {
		if isServing {
			// We are not healthy or should not be running the
			// query service, shut it down.
			// Note this is possibly sending 'spare' as
			// the tablet type, we will clean it up later.
			agent.disallowQueries(tablet.Tablet.Type,
				fmt.Sprintf("health-check failure(%v)", err),
			)
		}
	}

	// save the health record
	record := &HealthRecord{
		Error:            err,
		ReplicationDelay: replicationDelay,
		Time:             time.Now(),
	}
	agent.History.Add(record)

	// try to figure out the mysql port if we don't have it yet
	if _, ok := tablet.PortMap["mysql"]; !ok {
		// we don't know the port, try to get it from mysqld
		mysqlPort, err := agent.MysqlDaemon.GetMysqlPort()
		if err != nil {
			// Don't log if we're already in a waiting-for-mysql state.
			agent.mutex.Lock()
			if !agent._waitingForMysql {
				log.Warningf("Can't get mysql port, won't populate Tablet record in topology (will retry silently at healthcheck interval %v): %v", *healthCheckInterval, err)
				agent._waitingForMysql = true
			}
			agent.mutex.Unlock()
		} else {
			log.Infof("Updating tablet mysql port to %v", mysqlPort)
			if err := agent.TopoServer.UpdateTabletFields(agent.batchCtx, tablet.Alias, func(tablet *pbt.Tablet) error {
				tablet.PortMap["mysql"] = mysqlPort
				return nil
			}); err != nil {
				log.Infof("Error updating mysql port in tablet record, will try again: %v", err)
				return
			}

			// save the port so we don't update it again next time
			// we do the health check.
			agent.mutex.Lock()
			agent._tablet.PortMap["mysql"] = mysqlPort
			agent._waitingForMysql = false
			agent.mutex.Unlock()
		}
	}

	// remember our health status
	agent.mutex.Lock()
	agent._healthy = err
	agent._healthyTime = time.Now()
	agent._replicationDelay = replicationDelay
	agent.mutex.Unlock()

	// send it to our observers
	agent.broadcastHealth()

	// Update our topo.Server state, start with no change
	newTabletType := tablet.Type
	if err != nil {
		// The tablet is not healthy, let's see what we need to do
		if tablet.Type != targetTabletType {
			if tablet.Type != pbt.TabletType_SPARE {
				// we only log if we're not in spare,
				// as the spare state is normal for a
				// failed health check.
				log.Infof("Tablet not healthy and in state %v, not changing it: %v", tablet.Type, err)
			}
			return
		}

		// Note that if the query service is running, we may
		// need to stop it. The post-action callback will do
		// it, and it will be done after we change our state,
		// so it's the right order, let it do it.
		log.Infof("Tablet not healthy, converting it from %v to spare: %v", targetTabletType, err)
		newTabletType = pbt.TabletType_SPARE
	} else {
		// We are healthy, maybe with health, see if we need
		// to update the record. We only change from spare to
		// our target type.
		if tablet.Type == pbt.TabletType_SPARE {
			newTabletType = targetTabletType
		}
		if tablet.Type == newTabletType && topo.IsHealthEqual(health, tablet.HealthMap) {
			// no change in health, not logging anything,
			// and we're done
			return
		}

		// we need to update our state
		log.Infof("Updating tablet record as healthy type %v -> %v with health details %v -> %v", tablet.Type, newTabletType, tablet.HealthMap, health)
	}

	// Change the Type, update the health. Note we pass in a map
	// that's not nil, meaning if it's empty, we will clear it.
	if err := topotools.ChangeType(agent.batchCtx, agent.TopoServer, tablet.Alias, newTabletType, health); err != nil {
		log.Infof("Error updating tablet record: %v", err)
		return
	}
	tablet.HealthMap = health
	tablet.Type = newTabletType

	// Rebuild the serving graph in our cell, only if we're dealing with
	// a serving type
	if err := agent.updateServingGraph(tablet, targetTabletType); err != nil {
		log.Warningf("updateServingGraph failed (will still run post action callbacks, serving graph might be out of date): %v", err)
	}

	// run the post action callbacks, not much we can do with returned error
	if err := agent.refreshTablet(agent.batchCtx, "healthcheck"); err != nil {
		log.Warningf("refreshTablet failed: %v", err)
	}
}

// terminateHealthChecks is called when we enter lame duck mode.
// We will clean up our state, and shut down query service.
// We only do something if we are in targetTabletType state, and then
// we just go to spare.
func (agent *ActionAgent) terminateHealthChecks(targetTabletType pbt.TabletType) {
	agent.actionMutex.Lock()
	defer agent.actionMutex.Unlock()
	log.Info("agent.terminateHealthChecks is starting")

	// read the current tablet record
	tablet := agent.Tablet()
	if tablet.Type != targetTabletType {
		log.Infof("Tablet in state %v, not changing it", tablet.Type)
		return
	}

	// Change the Type to spare, update the health. Note we pass in a map
	// that's not nil, meaning we will clear it.
	if err := topotools.ChangeType(agent.batchCtx, agent.TopoServer, tablet.Alias, pbt.TabletType_SPARE, make(map[string]string)); err != nil {
		log.Infof("Error updating tablet record: %v", err)
		return
	}
	// The change above succeeded, so update our local copy.
	tablet, err := agent.updateTabletFromTopo(agent.batchCtx)
	if err != nil {
		log.Infof("Error re-reading tablet record: %v", err)
		return
	}

	// Update the serving graph in our cell, only if we're dealing with
	// a serving type
	if err := agent.updateServingGraph(tablet, targetTabletType); err != nil {
		log.Warningf("updateServingGraph failed (will still run post action callbacks, serving graph might be out of date): %v", err)
	}

	// We've already rebuilt the shard, which is the only reason we registered
	// ourself as OnTermSync (synchronous). The rest can be done asynchronously.
	go func() {
		// Run the post action callbacks (let them shutdown the query service)
		if err := agent.refreshTablet(agent.batchCtx, "terminatehealthcheck"); err != nil {
			log.Warningf("refreshTablet failed: %v", err)
		}
	}()
}

// updateServingGraph will update the serving graph if we need to.
func (agent *ActionAgent) updateServingGraph(tablet *topo.TabletInfo, targetTabletType pbt.TabletType) error {
	if topo.IsInServingGraph(targetTabletType) {
		if err := topotools.UpdateTabletEndpoints(agent.batchCtx, agent.TopoServer, tablet.Tablet); err != nil {
			return fmt.Errorf("UpdateTabletEndpoints failed: %v", err)
		}
	}
	return nil
}
<|endoftext|>