<commit_before>package job

import (
	"bytes"
	"compress/gzip"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"text/template"

	"github.com/falling-sky/fsbuilder/config"
	"github.com/falling-sky/fsbuilder/fileutil"
	"github.com/falling-sky/fsbuilder/gitinfo"
	"github.com/falling-sky/fsbuilder/po"
)

// rePROCESS matches on   [% PROCESS "filename" %]
// and captures the entire template directivel as well as the inside filename.
var rePROCESS = regexp.MustCompile(`\[\%\s*PROCESS\s*"(.*?)"\s*\%\]`)
var reTRANSLATE = regexp.MustCompile(`(?ms){{(.*?)}}`)

// PostType describes a directory, and how to process it.
type PostInfoType struct {
	Directory   string
	Extension   string
	PostProcess []string
	EscapeQuote bool
	MultiLocale bool
	Compress    bool
}

// QueueItem represents a single job to be queued, and ran as capacity allows.
// This is so we can generate the work list up front; and then pace out the work
// based on number of avaialble CPUs.
type QueueItem struct {
	Config   *config.Record
	RootDir  string
	Filename string
	PoFile   *po.File
	PotFile  *po.File
	Data     *TemplateData
	PostInfo PostInfoType
}

// QueueTracker is an object for managing QueueItem jobs.
type QueueTracker struct {
	Channel chan *QueueItem
	WG      *sync.WaitGroup
}

// TemplateData is passed when adding the job to the queue.
// This is used by Go's text/template to extract info before expansion.
// Because templates are cached after being processed,
// and before translation, we should not place anything here that is
// locale specific.
type TemplateData struct {
	GitInfo      *gitinfo.GitInfo
	PoMap        po.MapStringFile
	Basename     string
	AddLanguage  string
	DirSignature string
}

// ParsedCacheType provides properly mutex locked cache access to
// the expanded (but untranslated) templates.
type ParsedCacheType struct {
	lock   sync.RWMutex
	byname map[string]string
}

// ParsedCache holds the actual cache of expanded (but not translated) templates.
var ParsedCache ParsedCacheType

func init() {
	ParsedCache.byname = make(map[string]string)
}

// GrabContent grabs a file.  Takes into account the QueueItem variables
// such as the iput directory path.  The file is cached for future requests.
func GrabContent(qi *QueueItem) string {
	topName := qi.RootDir + "/" + qi.Filename
	// log.Printf("GrabContent(%s)  (%s)\n", qi.Filename, qi.PoFile.Language)

	grab := func(fn string) string {
		//		log.Printf("GrabContent(%s)  (%s) (fn=%s)\n", qi.Filename, qi.PoFile.Language, fn)

		fullname := qi.RootDir + "/" + fn
		c, err := fileutil.ReadFile(fullname)
		if err != nil {
			log.Fatalf("tried to load %s (via %s): %s", fullname, topName, err)
		}
		//		log.Printf("read %v (%v bytes)\n", fullname, len(c))

		UpdatePot(qi, c, fn)

		return c
	}

	content := grab(qi.Filename)
	// Do we see PROCESS lines?
	for {
		matches := rePROCESS.FindStringSubmatch(content)
		if len(matches) == 0 {
			break
		}
		if len(matches) < 2 {
			log.Fatalf("I don't know what happened, but %s is interesting", matches[0])
		}
		wrapperString := matches[0]
		insideName := matches[1]
		newContent := grab(insideName)
		content = strings.Replace(content, wrapperString, newContent, -1)
	}
	return content
}

// ProcessTemplate runs text.Template against the given text.
// Note we use [% %]  for text.Template directorives, since these
// are fewer than translations. And we prefer to do translations
// without the template ugliness.
func ProcessTemplate(qi *QueueItem, content string) string {
	topName := qi.RootDir + "/" + qi.Filename

	// Do we need any custom functions?
	FuncMap := make(template.FuncMap)
	FuncMap["EXAMPLE"] = func(name string) (string, error) {
		//log.Printf("PROCESS: %v\n", name)
		return "", nil
	}

	// Parse the template.  Just looks for markers and implied commands.
	root := template.New(qi.Filename).Delims(`[%`, `%]`).Funcs(FuncMap)
	tmpl, err := root.Parse(content)
	if err != nil {
		log.Fatalf("Parsing template for %v: %v", topName, err)
	}

	// Execute the template.
	wr := &bytes.Buffer{}
	err = tmpl.Execute(wr, qi.Data)
	if err != nil {
		log.Fatalf("Executing template for %v: %v", topName, err)
	}

	return string(wr.Bytes())
}

func UpdatePot(qi *QueueItem, content string, fn string) {
	//	log.Printf("UpdatePot fn=%s\n", fn)
	for {

		matches := reTRANSLATE.FindStringSubmatch(content)
		if len(matches) == 0 {
			break
		}
		if len(matches) < 2 {
			log.Fatalf("I don't know what happened, but %s is interesting", matches[0])
		}
		wrapperString := matches[0]
		insideName := matches[1]

		//		log.Printf("UpdatePot inside=%s fn=%s escape=%v\n", insideName, fn, qi.EscapeQuotes)

		qi.PotFile.Add(insideName, fn, qi.PostInfo.EscapeQuote)

		content = strings.Replace(content, wrapperString, insideName, -1)

	}
}

// TranslateContent  looks for {{ text }} and replaces it with
// either translated text, or the original text.
func TranslateContent(qi *QueueItem, content string) string {
	for {

		matches := reTRANSLATE.FindStringSubmatch(content)
		if len(matches) == 0 {
			break
		}
		if len(matches) < 2 {
			log.Fatalf("I don't know what happened, but %s is interesting", matches[0])
		}
		wrapperString := matches[0]
		insideName := matches[1]

		//	log.Printf("grabbing %v\n", insideName)
		newContent := qi.PoFile.Translate(insideName, qi.PostInfo.EscapeQuote)

		//	log.Printf("Replacing %s with %s\n", wrapperString, newContent)

		content = strings.Replace(content, wrapperString, newContent, -1)

	}
	return content
}

func ProcessContentFancy(qi *QueueItem, content string) {

	tasks := qi.PostInfo.PostProcess

	basename := qi.Filename
	if t, ok := qi.Config.Map[qi.Filename]; ok {
		basename = t
	}

	// Prepare the macros that we support for running external commands.
	macros := make(map[string]string)
	macros["NAME"] = basename
	macros["NAMEGZ"] = macros["NAME"] + ".gz"
	if qi.PostInfo.MultiLocale == true {
		macros["NAME"] = macros["NAME"] + "." + qi.PoFile.Language
		macros["NAMEGZ"] = macros["NAMEGZ"] + "." + qi.PoFile.Language
	}
	macros["INPUT"] = macros["NAME"] + ".orig"
	macros["OUTPUT"] = macros["NAME"]

	// Supported varables are:
	// INPUT                  - same as [FILENAME].orig - the file written by builder before calling 3rd party fancy tools
	// FILENAME or OUTPUT     - Final output filename (with path)
	// FILENAMEGZ or OUTPUTGZ - Final output filename, gzipped (with path)
	// BASENAME               - Final output name (no path). Use this if inserting text into files.
	// BASENAMEGZ             - Final output name, gzipped (no path).  Use this if inserting text into files.

	fixup := func(s string) string {
		for k, v := range macros {
			kk := "[" + k + "]"
			s = strings.Replace(s, kk, v, -1)
		}
		return s
	}

	// First, write the file to disk.
	outputfilename := qi.Config.Directories.OutputDir + "/" + macros["INPUT"]
	os.MkdirAll(filepath.Dir(outputfilename), 0755)

	err := ioutil.WriteFile(outputfilename, []byte(content), 0755)
	if err != nil {
		log.Fatal(err)
	}
	// log.Printf("wrote %s etc (%v bytes)\n", outputfilename, len(content))

	// Post processing defined from the config file, 3rd party tools
	for _, task := range tasks {
		runcmd := fixup(task)
		//log.Printf("running: %s\n", runcmd)

		shellscript := bytes.NewBufferString(runcmd)
		stderr := &bytes.Buffer{}

		c := exec.Cmd{}
		c.Path = "/bin/sh"
		c.Dir = qi.Config.Directories.OutputDir
		c.Stdin = shellscript
		c.Stderr = stderr
		// log.Printf("About to run: %#v\n", runcmd)
		e := c.Run()

		// HACK HACK HACK ignore tidy exit code 1
		if e != nil {
			if strings.HasPrefix(runcmd, "tidy ") {
				if e.Error() == "exit status 1" {
					e = nil
				}
			}
		}
		if e != nil {
			log.Printf("stderr: %s\n", stderr.String())
			log.Fatalf("While running %#v .. got: %#v\n", runcmd, e.Error())
		}
	}

}

func ProcessContent(qi *QueueItem, content string) {

	// See if there are commands specified. IF so, run those.
	tasks := qi.PostInfo.PostProcess
	if len(tasks) > 0 {
		ProcessContentFancy(qi, content)
		return
	}

	basename := qi.Filename
	if t, ok := qi.Config.Map[qi.Filename]; ok {
		basename = t
	}

	// Otherwise, do writes directly, and do our own compression.
	uncompressed := qi.Config.Directories.OutputDir + "/" + basename
	compressed := qi.Config.Directories.OutputDir + "/" + basename + ".gz"
	if qi.PostInfo.MultiLocale == true {
		uncompressed = uncompressed + "." + qi.PoFile.Language
		compressed = compressed + "." + qi.PoFile.Language
	}

	// Make sure the directory exists.
	// We may need to keep track of this;
	// do we really want to do this 1000+ times?
	os.MkdirAll(filepath.Dir(uncompressed), 0755)

	err := ioutil.WriteFile(uncompressed, []byte(content), 0644)
	if err != nil {
		log.Fatal(err)
	}
	// log.Printf("wrote %s etc (%v bytes)\n", outputfilename, len(content))

	if qi.PostInfo.Compress {
		if strings.HasSuffix(qi.Filename, ".html") {
			content = strings.Replace(content, `src="/index.js`, `src="/index.js.gz`, -1)
			content = strings.Replace(content, `href="/index.css`, `href="/index.css.gz`, -1)
		}

		// Compress in memory
		b := &bytes.Buffer{}
		w, err := gzip.NewWriterLevel(b, gzip.BestCompression)
		if err != nil {
			log.Fatal(err)
		}
		w.Write([]byte(content))
		w.Close()

		// And write
		err = ioutil.WriteFile(compressed, b.Bytes(), 0644)
		if err != nil {
			log.Fatal(err)
		}
	}

}

// RunJob takes a single QueueItem, and expands, translates, optimizes,
// and writes files for that single file for a single language.  These are spoon-fed
// by RunQueue.
func RunJob(qi *QueueItem) {
	log.Printf("RunJob Filename=%s PoLang=%s\n", qi.Filename, qi.PoFile.Language)
	readFilename := qi.RootDir + "/" + qi.Filename

	var content string
	ParsedCache.lock.Lock()
	if c, ok := ParsedCache.byname[readFilename]; ok {
		// log.Printf("cached: %s", readFilename)
		content = c
	} else {
		//log.Printf("not cached: %s", readFilename)
		content = GrabContent(qi)
		content = ProcessTemplate(qi, content)
		ParsedCache.byname[readFilename] = content
	}
	ParsedCache.lock.Unlock()

	// TODO process translations
	content = TranslateContent(qi, content)
	ProcessContent(qi, content)

}

// RunQueue is a goroutine that listens to a channel for jobs.
// If jobs are accepted, they are given to RunJob.
func (qt *QueueTracker) RunQueue() {
	for {
		job, ok := <-qt.Channel
		if ok {
			RunJob(job)  // Run the job.
			qt.WG.Done() // Decrement WaitGroup counter
		} else {
			return
		}
	}
}

// Add a job to the queue.  Sends it to the channel.
func (qt *QueueTracker) Add(qi *QueueItem) {
	qt.WG.Add(1)     // Increment the WaitGroup counter.
	qt.Channel <- qi // Put the job in the queue.
}

// Wait will wait for all existing jobs to finish.
func (qt *QueueTracker) Wait() {
	log.Printf("WAITING\n")
	qt.WG.Wait()
}

// StartQueue will start a goroutine for jobs, and return
// a handle to be used for adding and waiting on jobs.
func StartQueue(maxjobs int) *QueueTracker {
	qt := &QueueTracker{}
	qt.Channel = make(chan *QueueItem, 10000)
	qt.WG = &sync.WaitGroup{}

	if maxjobs == 0 {
		maxjobs = runtime.NumCPU()
		maxjobs = 1
	}
	//maxjobs = 1
	for i := 0; i < maxjobs; i++ {
		go qt.RunQueue()
	}

	return qt
}
<commit_msg>set max jobs to match cpu<commit_after>package job

import (
	"bytes"
	"compress/gzip"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"text/template"

	"github.com/falling-sky/fsbuilder/config"
	"github.com/falling-sky/fsbuilder/fileutil"
	"github.com/falling-sky/fsbuilder/gitinfo"
	"github.com/falling-sky/fsbuilder/po"
)

// rePROCESS matches on   [% PROCESS "filename" %]
// and captures the entire template directivel as well as the inside filename.
var rePROCESS = regexp.MustCompile(`\[\%\s*PROCESS\s*"(.*?)"\s*\%\]`)
var reTRANSLATE = regexp.MustCompile(`(?ms){{(.*?)}}`)

// PostType describes a directory, and how to process it.
type PostInfoType struct {
	Directory   string
	Extension   string
	PostProcess []string
	EscapeQuote bool
	MultiLocale bool
	Compress    bool
}

// QueueItem represents a single job to be queued, and ran as capacity allows.
// This is so we can generate the work list up front; and then pace out the work
// based on number of avaialble CPUs.
type QueueItem struct {
	Config   *config.Record
	RootDir  string
	Filename string
	PoFile   *po.File
	PotFile  *po.File
	Data     *TemplateData
	PostInfo PostInfoType
}

// QueueTracker is an object for managing QueueItem jobs.
type QueueTracker struct {
	Channel chan *QueueItem
	WG      *sync.WaitGroup
}

// TemplateData is passed when adding the job to the queue.
// This is used by Go's text/template to extract info before expansion.
// Because templates are cached after being processed,
// and before translation, we should not place anything here that is
// locale specific.
type TemplateData struct {
	GitInfo      *gitinfo.GitInfo
	PoMap        po.MapStringFile
	Basename     string
	AddLanguage  string
	DirSignature string
}

// ParsedCacheType provides properly mutex locked cache access to
// the expanded (but untranslated) templates.
type ParsedCacheType struct {
	lock   sync.RWMutex
	byname map[string]string
}

// ParsedCache holds the actual cache of expanded (but not translated) templates.
var ParsedCache ParsedCacheType

func init() {
	ParsedCache.byname = make(map[string]string)
}

// GrabContent grabs a file.  Takes into account the QueueItem variables
// such as the iput directory path.  The file is cached for future requests.
func GrabContent(qi *QueueItem) string {
	topName := qi.RootDir + "/" + qi.Filename
	// log.Printf("GrabContent(%s)  (%s)\n", qi.Filename, qi.PoFile.Language)

	grab := func(fn string) string {
		//		log.Printf("GrabContent(%s)  (%s) (fn=%s)\n", qi.Filename, qi.PoFile.Language, fn)

		fullname := qi.RootDir + "/" + fn
		c, err := fileutil.ReadFile(fullname)
		if err != nil {
			log.Fatalf("tried to load %s (via %s): %s", fullname, topName, err)
		}
		//		log.Printf("read %v (%v bytes)\n", fullname, len(c))

		UpdatePot(qi, c, fn)

		return c
	}

	content := grab(qi.Filename)
	// Do we see PROCESS lines?
	for {
		matches := rePROCESS.FindStringSubmatch(content)
		if len(matches) == 0 {
			break
		}
		if len(matches) < 2 {
			log.Fatalf("I don't know what happened, but %s is interesting", matches[0])
		}
		wrapperString := matches[0]
		insideName := matches[1]
		newContent := grab(insideName)
		content = strings.Replace(content, wrapperString, newContent, -1)
	}
	return content
}

// ProcessTemplate runs text.Template against the given text.
// Note we use [% %]  for text.Template directorives, since these
// are fewer than translations. And we prefer to do translations
// without the template ugliness.
func ProcessTemplate(qi *QueueItem, content string) string {
	topName := qi.RootDir + "/" + qi.Filename

	// Do we need any custom functions?
	FuncMap := make(template.FuncMap)
	FuncMap["EXAMPLE"] = func(name string) (string, error) {
		//log.Printf("PROCESS: %v\n", name)
		return "", nil
	}

	// Parse the template.  Just looks for markers and implied commands.
	root := template.New(qi.Filename).Delims(`[%`, `%]`).Funcs(FuncMap)
	tmpl, err := root.Parse(content)
	if err != nil {
		log.Fatalf("Parsing template for %v: %v", topName, err)
	}

	// Execute the template.
	wr := &bytes.Buffer{}
	err = tmpl.Execute(wr, qi.Data)
	if err != nil {
		log.Fatalf("Executing template for %v: %v", topName, err)
	}

	return string(wr.Bytes())
}

func UpdatePot(qi *QueueItem, content string, fn string) {
	//	log.Printf("UpdatePot fn=%s\n", fn)
	for {

		matches := reTRANSLATE.FindStringSubmatch(content)
		if len(matches) == 0 {
			break
		}
		if len(matches) < 2 {
			log.Fatalf("I don't know what happened, but %s is interesting", matches[0])
		}
		wrapperString := matches[0]
		insideName := matches[1]

		//		log.Printf("UpdatePot inside=%s fn=%s escape=%v\n", insideName, fn, qi.EscapeQuotes)

		qi.PotFile.Add(insideName, fn, qi.PostInfo.EscapeQuote)

		content = strings.Replace(content, wrapperString, insideName, -1)

	}
}

// TranslateContent  looks for {{ text }} and replaces it with
// either translated text, or the original text.
func TranslateContent(qi *QueueItem, content string) string {
	for {

		matches := reTRANSLATE.FindStringSubmatch(content)
		if len(matches) == 0 {
			break
		}
		if len(matches) < 2 {
			log.Fatalf("I don't know what happened, but %s is interesting", matches[0])
		}
		wrapperString := matches[0]
		insideName := matches[1]

		//	log.Printf("grabbing %v\n", insideName)
		newContent := qi.PoFile.Translate(insideName, qi.PostInfo.EscapeQuote)

		//	log.Printf("Replacing %s with %s\n", wrapperString, newContent)

		content = strings.Replace(content, wrapperString, newContent, -1)

	}
	return content
}

func ProcessContentFancy(qi *QueueItem, content string) {

	tasks := qi.PostInfo.PostProcess

	basename := qi.Filename
	if t, ok := qi.Config.Map[qi.Filename]; ok {
		basename = t
	}

	// Prepare the macros that we support for running external commands.
	macros := make(map[string]string)
	macros["NAME"] = basename
	macros["NAMEGZ"] = macros["NAME"] + ".gz"
	if qi.PostInfo.MultiLocale == true {
		macros["NAME"] = macros["NAME"] + "." + qi.PoFile.Language
		macros["NAMEGZ"] = macros["NAMEGZ"] + "." + qi.PoFile.Language
	}
	macros["INPUT"] = macros["NAME"] + ".orig"
	macros["OUTPUT"] = macros["NAME"]

	// Supported varables are:
	// INPUT                  - same as [FILENAME].orig - the file written by builder before calling 3rd party fancy tools
	// FILENAME or OUTPUT     - Final output filename (with path)
	// FILENAMEGZ or OUTPUTGZ - Final output filename, gzipped (with path)
	// BASENAME               - Final output name (no path). Use this if inserting text into files.
	// BASENAMEGZ             - Final output name, gzipped (no path).  Use this if inserting text into files.

	fixup := func(s string) string {
		for k, v := range macros {
			kk := "[" + k + "]"
			s = strings.Replace(s, kk, v, -1)
		}
		return s
	}

	// First, write the file to disk.
	outputfilename := qi.Config.Directories.OutputDir + "/" + macros["INPUT"]
	os.MkdirAll(filepath.Dir(outputfilename), 0755)

	err := ioutil.WriteFile(outputfilename, []byte(content), 0755)
	if err != nil {
		log.Fatal(err)
	}
	// log.Printf("wrote %s etc (%v bytes)\n", outputfilename, len(content))

	// Post processing defined from the config file, 3rd party tools
	for _, task := range tasks {
		runcmd := fixup(task)
		//log.Printf("running: %s\n", runcmd)

		shellscript := bytes.NewBufferString(runcmd)
		stderr := &bytes.Buffer{}

		c := exec.Cmd{}
		c.Path = "/bin/sh"
		c.Dir = qi.Config.Directories.OutputDir
		c.Stdin = shellscript
		c.Stderr = stderr
		// log.Printf("About to run: %#v\n", runcmd)
		e := c.Run()

		// HACK HACK HACK ignore tidy exit code 1
		if e != nil {
			if strings.HasPrefix(runcmd, "tidy ") {
				if e.Error() == "exit status 1" {
					e = nil
				}
			}
		}
		if e != nil {
			log.Printf("stderr: %s\n", stderr.String())
			log.Fatalf("While running %#v .. got: %#v\n", runcmd, e.Error())
		}
	}

}

func ProcessContent(qi *QueueItem, content string) {

	// See if there are commands specified. IF so, run those.
	tasks := qi.PostInfo.PostProcess
	if len(tasks) > 0 {
		ProcessContentFancy(qi, content)
		return
	}

	basename := qi.Filename
	if t, ok := qi.Config.Map[qi.Filename]; ok {
		basename = t
	}

	// Otherwise, do writes directly, and do our own compression.
	uncompressed := qi.Config.Directories.OutputDir + "/" + basename
	compressed := qi.Config.Directories.OutputDir + "/" + basename + ".gz"
	if qi.PostInfo.MultiLocale == true {
		uncompressed = uncompressed + "." + qi.PoFile.Language
		compressed = compressed + "." + qi.PoFile.Language
	}

	// Make sure the directory exists.
	// We may need to keep track of this;
	// do we really want to do this 1000+ times?
	os.MkdirAll(filepath.Dir(uncompressed), 0755)

	err := ioutil.WriteFile(uncompressed, []byte(content), 0644)
	if err != nil {
		log.Fatal(err)
	}
	// log.Printf("wrote %s etc (%v bytes)\n", outputfilename, len(content))

	if qi.PostInfo.Compress {
		if strings.HasSuffix(qi.Filename, ".html") {
			content = strings.Replace(content, `src="/index.js`, `src="/index.js.gz`, -1)
			content = strings.Replace(content, `href="/index.css`, `href="/index.css.gz`, -1)
		}

		// Compress in memory
		b := &bytes.Buffer{}
		w, err := gzip.NewWriterLevel(b, gzip.BestCompression)
		if err != nil {
			log.Fatal(err)
		}
		w.Write([]byte(content))
		w.Close()

		// And write
		err = ioutil.WriteFile(compressed, b.Bytes(), 0644)
		if err != nil {
			log.Fatal(err)
		}
	}

}

// RunJob takes a single QueueItem, and expands, translates, optimizes,
// and writes files for that single file for a single language.  These are spoon-fed
// by RunQueue.
func RunJob(qi *QueueItem) {
	log.Printf("RunJob Filename=%s PoLang=%s\n", qi.Filename, qi.PoFile.Language)
	readFilename := qi.RootDir + "/" + qi.Filename

	var content string
	ParsedCache.lock.Lock()
	if c, ok := ParsedCache.byname[readFilename]; ok {
		// log.Printf("cached: %s", readFilename)
		content = c
	} else {
		//log.Printf("not cached: %s", readFilename)
		content = GrabContent(qi)
		content = ProcessTemplate(qi, content)
		ParsedCache.byname[readFilename] = content
	}
	ParsedCache.lock.Unlock()

	// TODO process translations
	content = TranslateContent(qi, content)
	ProcessContent(qi, content)

}

// RunQueue is a goroutine that listens to a channel for jobs.
// If jobs are accepted, they are given to RunJob.
func (qt *QueueTracker) RunQueue() {
	for {
		job, ok := <-qt.Channel
		if ok {
			RunJob(job)  // Run the job.
			qt.WG.Done() // Decrement WaitGroup counter
		} else {
			return
		}
	}
}

// Add a job to the queue.  Sends it to the channel.
func (qt *QueueTracker) Add(qi *QueueItem) {
	qt.WG.Add(1)     // Increment the WaitGroup counter.
	qt.Channel <- qi // Put the job in the queue.
}

// Wait will wait for all existing jobs to finish.
func (qt *QueueTracker) Wait() {
	log.Printf("WAITING\n")
	qt.WG.Wait()
}

// StartQueue will start a goroutine for jobs, and return
// a handle to be used for adding and waiting on jobs.
func StartQueue(maxjobs int) *QueueTracker {
	qt := &QueueTracker{}
	qt.Channel = make(chan *QueueItem, 10000)
	qt.WG = &sync.WaitGroup{}

	if maxjobs == 0 {
		maxjobs = runtime.NumCPU()
	}
	for i := 0; i < maxjobs; i++ {
		go qt.RunQueue()
	}

	return qt
}
<|endoftext|>


<commit_before>package speed

import (
	"errors"
	"sync"
)

// InstanceDomain defines the interface for an instance domain
type InstanceDomain interface {
	ID() uint32                    // unique identifier for the instance domain
	Name() string                  // name of the instance domain
	Description() string           // description for the instance domain
	AddInstance(name string) error // adds an instance to the indom
	HasInstance(name string) bool  // checks if an instance is in the indom
	InstanceCount() int            // returns the number of instances in the indom
}

// PCPInstanceDomainBitLength is the maximum bit length of a PCP Instance Domain
//
// see: https://github.com/performancecopilot/pcp/blob/master/src/include/pcp/impl.h#L102-L121
const PCPInstanceDomainBitLength = 22

// PCPInstanceDomain wraps a PCP compatible instance domain
type PCPInstanceDomain struct {
	sync.RWMutex
	id                          uint32
	name                        string
	instances                   map[uint32]*pcpInstance // the instances for this InstanceDomain stored as a map
	offset                      int
	instanceOffset              int
	shortHelpText, longHelpText *PCPString
}

// NewPCPInstanceDomain creates a new instance domain or returns an already created one for the passed name
// NOTE: this is different from parfait's idea of generating ids for InstanceDomains
// We simply generate a unique 32 bit hash for an instance domain name, and if it has not
// already been created, we create it, otherwise we return the already created version
func NewPCPInstanceDomain(name, shortDescription, longDescription string) *PCPInstanceDomain {
	return &PCPInstanceDomain{
		id:            getHash(name, PCPInstanceDomainBitLength),
		name:          name,
		instances:     make(map[uint32]*pcpInstance),
		shortHelpText: NewPCPString(shortDescription),
		longHelpText:  NewPCPString(longDescription),
	}
}

func (indom *PCPInstanceDomain) HasInstance(name string) bool {
	indom.RLock()
	defer indom.RUnlock()

	h := getHash(name, 0)
	_, present := indom.instances[h]
	return present
}

// AddInstance adds a new instance to the current PCPInstanceDomain
func (indom *PCPInstanceDomain) AddInstance(name string) error {
	indom.Lock()
	defer indom.Unlock()

	h := getHash(name, 0)

	// not using HasInstance here to prevent the cost of
	// hashing twice
	_, present := indom.instances[h]
	if present {
		return errors.New("Instance with same name already created for the InstanceDomain")
	}

	ins := newpcpInstance(h, name, indom)
	indom.instances[h] = ins

	return nil
}

// ID returns the id for PCPInstanceDomain
func (indom *PCPInstanceDomain) ID() uint32 { return indom.id }

// Name returns the name for PCPInstanceDomain
func (indom *PCPInstanceDomain) Name() string { return indom.name }

// InstanceCount returns the number of instances in the current instance domain
func (indom *PCPInstanceDomain) InstanceCount() int {
	indom.RLock()
	defer indom.RUnlock()

	return len(indom.instances)
}

// Description returns the description for PCPInstanceDomain
func (indom *PCPInstanceDomain) Description() string {
	s, l := indom.shortHelpText.val, indom.longHelpText.val
	if l != "" {
		return s + "\n\n" + l
	}
	return s
}

func (indom *PCPInstanceDomain) Offset() int { return indom.offset }

func (indom *PCPInstanceDomain) setOffset(offset int) { indom.offset = offset }

func (indom *PCPInstanceDomain) String() string {
	indom.RLock()
	defer indom.RUnlock()

	s := "InstanceDomain: " + indom.name
	if len(indom.instances) > 0 {
		s += "["
		for _, i := range indom.instances {
			s += i.name + ","
		}
		s += "]"
	}
	return s
}
<commit_msg>instance domain: add AddInstances definition<commit_after>package speed

import (
	"errors"
	"sync"
)

// InstanceDomain defines the interface for an instance domain
type InstanceDomain interface {
	ID() uint32                        // unique identifier for the instance domain
	Name() string                      // name of the instance domain
	Description() string               // description for the instance domain
	AddInstance(name string) error     // adds an instance to the indom
	AddInstances(names []string) error // adds multiple instances to the indom
	HasInstance(name string) bool      // checks if an instance is in the indom
	InstanceCount() int                // returns the number of instances in the indom
}

// PCPInstanceDomainBitLength is the maximum bit length of a PCP Instance Domain
//
// see: https://github.com/performancecopilot/pcp/blob/master/src/include/pcp/impl.h#L102-L121
const PCPInstanceDomainBitLength = 22

// PCPInstanceDomain wraps a PCP compatible instance domain
type PCPInstanceDomain struct {
	sync.RWMutex
	id                          uint32
	name                        string
	instances                   map[uint32]*pcpInstance // the instances for this InstanceDomain stored as a map
	offset                      int
	instanceOffset              int
	shortHelpText, longHelpText *PCPString
}

// NewPCPInstanceDomain creates a new instance domain or returns an already created one for the passed name
// NOTE: this is different from parfait's idea of generating ids for InstanceDomains
// We simply generate a unique 32 bit hash for an instance domain name, and if it has not
// already been created, we create it, otherwise we return the already created version
func NewPCPInstanceDomain(name, shortDescription, longDescription string) *PCPInstanceDomain {
	return &PCPInstanceDomain{
		id:            getHash(name, PCPInstanceDomainBitLength),
		name:          name,
		instances:     make(map[uint32]*pcpInstance),
		shortHelpText: NewPCPString(shortDescription),
		longHelpText:  NewPCPString(longDescription),
	}
}

func (indom *PCPInstanceDomain) HasInstance(name string) bool {
	indom.RLock()
	defer indom.RUnlock()

	h := getHash(name, 0)
	_, present := indom.instances[h]
	return present
}

// AddInstance adds a new instance to the current PCPInstanceDomain
func (indom *PCPInstanceDomain) AddInstance(name string) error {
	indom.Lock()
	defer indom.Unlock()

	h := getHash(name, 0)

	// not using HasInstance here to prevent the cost of
	// hashing twice
	_, present := indom.instances[h]
	if present {
		return errors.New("Instance with same name already created for the InstanceDomain")
	}

	ins := newpcpInstance(h, name, indom)
	indom.instances[h] = ins

	return nil
}

// AddInstances adds new instances to the current PCPInstanceDomain
func (indom *PCPInstanceDomain) AddInstances(names []string) error {
	for _, name := range names {
		err := indom.AddInstance(name)
		if err != nil {
			return err
		}
	}
	return nil
}

// ID returns the id for PCPInstanceDomain
func (indom *PCPInstanceDomain) ID() uint32 { return indom.id }

// Name returns the name for PCPInstanceDomain
func (indom *PCPInstanceDomain) Name() string { return indom.name }

// InstanceCount returns the number of instances in the current instance domain
func (indom *PCPInstanceDomain) InstanceCount() int {
	indom.RLock()
	defer indom.RUnlock()

	return len(indom.instances)
}

// Description returns the description for PCPInstanceDomain
func (indom *PCPInstanceDomain) Description() string {
	s, l := indom.shortHelpText.val, indom.longHelpText.val
	if l != "" {
		return s + "\n\n" + l
	}
	return s
}

func (indom *PCPInstanceDomain) Offset() int { return indom.offset }

func (indom *PCPInstanceDomain) setOffset(offset int) { indom.offset = offset }

func (indom *PCPInstanceDomain) String() string {
	indom.RLock()
	defer indom.RUnlock()

	s := "InstanceDomain: " + indom.name
	if len(indom.instances) > 0 {
		s += "["
		for _, i := range indom.instances {
			s += i.name + ","
		}
		s += "]"
	}
	return s
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"net/url"
)

// generateSignature generates the HMAC-SHA1 signature for a key and a message.
func generateSignature(key []byte, message string) (string, error) {
	mac := hmac.New(sha1.New, key)
	mac.Write([]byte(message))
	return base64.URLEncoding.EncodeToString(mac.Sum(nil)), nil
}

// SignURL signs a url with a clientID and signature.
// The signature is assumed to be in URL safe base64 encoding.
// The returned signature string is URLEncoded.
// See: https://developers.google.com/maps/documentation/business/webservices/auth#digital_signatures
func SignURL(path, clientID string, signature []byte, q url.Values) (string, error) {
	q.Set("client", clientID)
	encodedQuery := q.Encode()
	message := fmt.Sprintf("%s?%s", path, encodedQuery)
	s, err := generateSignature(signature, message)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%s&signature=%s", encodedQuery, s), nil
}
<commit_msg>Adding type information to docstring<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"net/url"
)

// generateSignature generates the base64 URL-encoded HMAC-SHA1 signature for the key and plaintext message.
func generateSignature(key []byte, message string) (string, error) {
	mac := hmac.New(sha1.New, key)
	mac.Write([]byte(message))
	return base64.URLEncoding.EncodeToString(mac.Sum(nil)), nil
}

// SignURL signs a url with a clientID and signature.
// The signature is assumed to be in URL safe base64 encoding.
// The returned signature string is URLEncoded.
// See: https://developers.google.com/maps/documentation/business/webservices/auth#digital_signatures
func SignURL(path, clientID string, signature []byte, q url.Values) (string, error) {
	q.Set("client", clientID)
	encodedQuery := q.Encode()
	message := fmt.Sprintf("%s?%s", path, encodedQuery)
	s, err := generateSignature(signature, message)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%s&signature=%s", encodedQuery, s), nil
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"io"
	"io/ioutil"
	"sync"
)

type gzipPool struct {
	mutex sync.Mutex
	top   *gzipPoolEntry
}

type gzipPoolEntry struct {
	gz   *gzip.Writer
	next *gzipPoolEntry
}

func newGzipPool(n int) *gzipPool {
	pool := new(gzipPool)

	for i := 0; i < n; i++ {
		pool.grow()
	}

	return pool
}

func (p *gzipPool) grow() {
	gz, err := gzip.NewWriterLevel(ioutil.Discard, conf.GZipCompression)
	if err != nil {
		logFatal("Can't init GZip compression: %s", err)
	}

	p.top = &gzipPoolEntry{
		gz:   gz,
		next: p.top,
	}
}

func (p *gzipPool) Get(w io.Writer) *gzip.Writer {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if p.top == nil {
		p.grow()
	}

	gz := p.top.gz
	gz.Reset(w)

	p.top = p.top.next

	return gz
}

func (p *gzipPool) Put(gz *gzip.Writer) {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	p.top = &gzipPoolEntry{gz: gz, next: p.top}
}
<commit_msg>Reset gzip encoder on put to pool<commit_after>package main

import (
	"compress/gzip"
	"io"
	"io/ioutil"
	"sync"
)

type gzipPool struct {
	mutex sync.Mutex
	top   *gzipPoolEntry
}

type gzipPoolEntry struct {
	gz   *gzip.Writer
	next *gzipPoolEntry
}

func newGzipPool(n int) *gzipPool {
	pool := new(gzipPool)

	for i := 0; i < n; i++ {
		pool.grow()
	}

	return pool
}

func (p *gzipPool) grow() {
	gz, err := gzip.NewWriterLevel(ioutil.Discard, conf.GZipCompression)
	if err != nil {
		logFatal("Can't init GZip compression: %s", err)
	}

	p.top = &gzipPoolEntry{
		gz:   gz,
		next: p.top,
	}
}

func (p *gzipPool) Get(w io.Writer) *gzip.Writer {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if p.top == nil {
		p.grow()
	}

	gz := p.top.gz
	gz.Reset(w)

	p.top = p.top.next

	return gz
}

func (p *gzipPool) Put(gz *gzip.Writer) {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	gz.Reset(ioutil.Discard)

	p.top = &gzipPoolEntry{gz: gz, next: p.top}
}
<|endoftext|>


<commit_before>// Copyright 2016 Marcel Gotsch. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package goserv

type ErrorHandler interface {
	ServeHTTP(ResponseWriter, *Request, error)
}

type ErrorHandlerFunc func(ResponseWriter, *Request, error)

func (e ErrorHandlerFunc) ServeHTTP(res ResponseWriter, req *Request, err error) {
	e(res, req, err)
}

type Handler interface {
	ServeHTTP(ResponseWriter, *Request)
}

type HandlerFunc func(ResponseWriter, *Request)

func (h HandlerFunc) ServeHTTP(res ResponseWriter, req *Request) {
	h(res, req)
}

type ParamHandler interface {
	ServeHTTP(ResponseWriter, *Request, string)
}

type ParamHandlerFunc func(ResponseWriter, *Request, string)

func (p ParamHandlerFunc) ServeHTTP(res ResponseWriter, req *Request, value string) {
	p(res, req, value)
}
<commit_msg>Document handlers<commit_after>// Copyright 2016 Marcel Gotsch. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package goserv

// A ErrorHandler is the last handler in the request chain and
// is responsible for handling errors that occur during the
// request processing.
//
// A ErrorHandler should always write a response!
type ErrorHandler interface {
	ServeHTTP(ResponseWriter, *Request, error)
}

// ErrorHandlerFunc is an adapter to allow ordinary functions
// to be ErrorHandlers.
type ErrorHandlerFunc func(ResponseWriter, *Request, error)

// ServeHTTP calls the actual function.
func (e ErrorHandlerFunc) ServeHTTP(res ResponseWriter, req *Request, err error) {
	e(res, req, err)
}

// A Handler processes an HTTP request and may respond to it.
type Handler interface {
	ServeHTTP(ResponseWriter, *Request)
}

// HandlerFunc is an adapter to allow ordinary functions to be Handlers.
type HandlerFunc func(ResponseWriter, *Request)

// ServeHTTP calls the actual Handler function.
func (h HandlerFunc) ServeHTTP(res ResponseWriter, req *Request) {
	h(res, req)
}

// A ParamHandler can be registered to a Router using the parameter's name.
// It gets invoked with the corresponding value extracted from the request's
// path.
//
// Parameters are part of a Route's path. To learn more about parameters take
// a look at the documentation of Route.
type ParamHandler interface {
	ServeHTTP(ResponseWriter, *Request, string)
}

// ParamHandlerFunc is an adapter to allow ordinary functions to be ParamHandlers.
type ParamHandlerFunc func(ResponseWriter, *Request, string)

// ServeHTTP calls the actual function.
func (p ParamHandlerFunc) ServeHTTP(res ResponseWriter, req *Request, value string) {
	p(res, req, value)
}
<|endoftext|>


<commit_before>package controllers

import (
	"github.com/kayac/alphawing/app/routes"
	r "github.com/revel/revel"
	"net/url"
)

type AuthController struct {
	AlphaWingController
}

func (c *AuthController) CheckLogin() r.Result {
	if c.isLogin() {
		return nil
	}

	loginUrl := routes.AlphaWingController.GetLogin()
	next := c.Request.URL.Path
	return c.Redirect(loginUrl + "?next=" + url.QueryEscape(next))
}
<commit_msg>do not use shortening name<commit_after>package controllers

import (
	"net/url"

	"github.com/kayac/alphawing/app/routes"
	"github.com/revel/revel"
)

type AuthController struct {
	AlphaWingController
}

func (c *AuthController) CheckLogin() revel.Result {
	if c.isLogin() {
		return nil
	}

	loginUrl := routes.AlphaWingController.GetLogin()
	next := c.Request.URL.Path
	return c.Redirect(loginUrl + "?next=" + url.QueryEscape(next))
}
<|endoftext|>


<commit_before>package controllers

import (
	"database/sql"
	"fmt"

	_ "github.com/go-sql-driver/mysql"
	_ "github.com/mattn/go-sqlite3"

	"github.com/kayac/alphawing/app/models"

	"github.com/coopernurse/gorp"
	"github.com/revel/revel"
	"github.com/revel/revel/modules/db/app"
)

var (
	Dbm *gorp.DbMap
)

func InitDB() {
	db.Init()
	Dbm = getDbm()

	appTableMap := Dbm.AddTableWithName(models.App{}, "app")
	appTableMap.SetKeys(true, "Id")
	appTableMap.ColMap("ApiToken").SetUnique(true)

	bundleTableMap := Dbm.AddTableWithName(models.Bundle{}, "bundle")
	bundleTableMap.SetKeys(true, "Id")

	authorityTableMap := Dbm.AddTableWithName(models.Authority{}, "authority")
	authorityTableMap.SetKeys(true, "Id")

	userTableMap := Dbm.AddTableWithName(models.User{}, "user")
	userTableMap.SetKeys(true, "Id")

	auditTableMap := Dbm.AddTableWithName(models.Audit{}, "audit")
	auditTableMap.SetKeys(true, "Id")

	Dbm.TraceOn("[gorp]", revel.INFO)
	Dbm.CreateTablesIfNotExists()
}

func getDbm() *gorp.DbMap {
	driver, ok := revel.Config.String("db.driver")
	if !ok {
		panic("require config: db.driver")
	}
	switch driver {
	case "mysql":
		return &gorp.DbMap{Db: db.Db, Dialect: gorp.MySQLDialect{"InnoDB", "UTF8"}}
	case "sqlite3":
		return &gorp.DbMap{Db: db.Db, Dialect: gorp.SqliteDialect{}}
	default:
		panic(fmt.Sprintf("unsupported driver: %s", driver))
	}
}

type GorpController struct {
	*revel.Controller
	Txn *gorp.Transaction
}

func Transact(f func(gorp.SqlExecutor) error) error {
	txn, err := Dbm.Begin()
	if err != nil {
		return err
	}
	defer func() {
		if txn == nil {
			return
		}
		if err := txn.Rollback(); err != nil && err != sql.ErrTxDone {
			panic(err)
		}
	}()

	err = f(txn)
	if err != nil {
		return err
	}

	err = txn.Commit()
	if err != nil && err != sql.ErrTxDone {
		return err
	}
	txn = nil
	return nil
}
<commit_msg>c.Txn はもう使わないので削除<commit_after>package controllers

import (
	"database/sql"
	"fmt"

	_ "github.com/go-sql-driver/mysql"
	_ "github.com/mattn/go-sqlite3"

	"github.com/kayac/alphawing/app/models"

	"github.com/coopernurse/gorp"
	"github.com/revel/revel"
	"github.com/revel/revel/modules/db/app"
)

var (
	Dbm *gorp.DbMap
)

func InitDB() {
	db.Init()
	Dbm = getDbm()

	appTableMap := Dbm.AddTableWithName(models.App{}, "app")
	appTableMap.SetKeys(true, "Id")
	appTableMap.ColMap("ApiToken").SetUnique(true)

	bundleTableMap := Dbm.AddTableWithName(models.Bundle{}, "bundle")
	bundleTableMap.SetKeys(true, "Id")

	authorityTableMap := Dbm.AddTableWithName(models.Authority{}, "authority")
	authorityTableMap.SetKeys(true, "Id")

	userTableMap := Dbm.AddTableWithName(models.User{}, "user")
	userTableMap.SetKeys(true, "Id")

	auditTableMap := Dbm.AddTableWithName(models.Audit{}, "audit")
	auditTableMap.SetKeys(true, "Id")

	Dbm.TraceOn("[gorp]", revel.INFO)
	Dbm.CreateTablesIfNotExists()
}

func getDbm() *gorp.DbMap {
	driver, ok := revel.Config.String("db.driver")
	if !ok {
		panic("require config: db.driver")
	}
	switch driver {
	case "mysql":
		return &gorp.DbMap{Db: db.Db, Dialect: gorp.MySQLDialect{"InnoDB", "UTF8"}}
	case "sqlite3":
		return &gorp.DbMap{Db: db.Db, Dialect: gorp.SqliteDialect{}}
	default:
		panic(fmt.Sprintf("unsupported driver: %s", driver))
	}
}

type GorpController struct {
	*revel.Controller
}

func Transact(f func(gorp.SqlExecutor) error) error {
	txn, err := Dbm.Begin()
	if err != nil {
		return err
	}
	defer func() {
		if txn == nil {
			return
		}
		if err := txn.Rollback(); err != nil && err != sql.ErrTxDone {
			panic(err)
		}
	}()

	err = f(txn)
	if err != nil {
		return err
	}

	err = txn.Commit()
	if err != nil && err != sql.ErrTxDone {
		return err
	}
	txn = nil
	return nil
}
<|endoftext|>


<commit_before>package image

import (
	"encoding/json"
	"fmt"
	img "image"
	"image/draw"
	"log"
	"math"
	"os"
	"os/exec"
	"time"

	"pixur.org/pixur/schema"

	"github.com/nfnt/resize"

	"image/gif"
	"image/jpeg"
	_ "image/png"
)

// TODO: maybe make this into it's own package
const (
	DefaultThumbnailWidth  = 160
	DefaultThumbnailHeight = 160

	maxWebmDuration = 60*2 + 1 // Two minutes, with 1 second of leeway
)

type BadWebmFormatErr struct {
	error
}

func FillImageConfig(f *os.File, p *schema.Pic) (img.Image, error) {
	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
		return nil, err
	}
	defer f.Seek(0, os.SEEK_SET)

	im, imgType, err := img.Decode(f)
	if err == img.ErrFormat {
		// Try Webm
		im, err = fillImageConfigFromWebm(f, p)
		if err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	} else {
		// TODO: handle this error
		p.Mime, _ = schema.FromImageFormat(imgType)
		p.Width = int64(im.Bounds().Dx())
		p.Height = int64(im.Bounds().Dy())
	}

	if p.Mime == schema.Pic_GIF {
		if _, err := f.Seek(0, os.SEEK_SET); err != nil {
			return nil, err
		}

		GIF, err := gif.DecodeAll(f)
		if err != nil {
			return nil, err
		}
		// Ignore gifs that have only one frame
		if len(GIF.Delay) > 1 {
			p.AnimationInfo = &schema.AnimationInfo{
				Duration: GetGifDuration(GIF),
			}
		}
	}

	return im, nil
}

// TODO: add tests for this
func GetGifDuration(g *gif.GIF) *schema.Duration {
	var duration time.Duration
	// TODO: check for overflow
	// each delay unit is 1/100 of a second
	for _, frameHundredths := range g.Delay {
		duration += time.Millisecond * time.Duration(10*frameHundredths)
	}

	return schema.FromDuration(duration)
}

// TODO: interpret image rotation metadata
func MakeThumbnail(im img.Image) img.Image {
	bounds := findMaxSquare(im.Bounds())
	largeSquareImage := img.NewNRGBA(bounds)
	draw.Draw(largeSquareImage, bounds, im, bounds.Min, draw.Src)
	return resize.Resize(DefaultThumbnailWidth, DefaultThumbnailHeight, largeSquareImage,
		resize.NearestNeighbor)
}

func SaveThumbnail(im img.Image, p *schema.Pic, pixPath string) error {
	f, err := os.Create(p.ThumbnailPath(pixPath))
	if err != nil {
		return err
	}
	defer f.Close()
	return jpeg.Encode(f, im, nil)
}

func findMaxSquare(bounds img.Rectangle) img.Rectangle {
	width := bounds.Dx()
	height := bounds.Dy()
	if height < width {
		missingSpace := width - height
		return img.Rectangle{
			Min: img.Point{
				X: bounds.Min.X + missingSpace/2,
				Y: bounds.Min.Y,
			},
			Max: img.Point{
				X: bounds.Min.X + missingSpace/2 + height,
				Y: bounds.Max.Y,
			},
		}
	} else {
		missingSpace := height - width
		return img.Rectangle{
			Min: img.Point{
				X: bounds.Min.X,
				Y: bounds.Min.Y + missingSpace/2,
			},
			Max: img.Point{
				X: bounds.Max.X,
				Y: bounds.Min.Y + missingSpace/2 + width,
			},
		}
	}
}

type FFprobeConfig struct {
	Streams []FFprobeStream `json:"streams"`
	Format  FFprobeFormat   `json:"format"`
}

type FFprobeFormat struct {
	StreamCount int     `json:"nb_streams"`
	FormatName  string  `json:"format_name"`
	Duration    float64 `json:"duration,string"`
}

type FFprobeStream struct {
	CodecName string `json:"codec_name"`
	CodecType string `json:"codec_type"`
	Width     int64  `json:"width"`
	Height    int64  `json:"height"`
}

func fillImageConfigFromWebm(tempFile *os.File, p *schema.Pic) (img.Image, error) {
	config, err := GetWebmConfig(tempFile.Name())
	if err != nil {
		return nil, err
	}
	p.Mime = schema.Pic_WEBM
	// Handle the 0 and 1 case
	for _, stream := range config.Streams {
		p.Width = stream.Width
		p.Height = stream.Height
		break
	}

	if dur, success := ConvertFloatDuration(config.Format.Duration); success {
		if dur >= time.Nanosecond {
			p.AnimationInfo = &schema.AnimationInfo{
				Duration: schema.FromDuration(dur),
			}
		}
	} else {
		log.Println("Invalid duration from ffmpeg: ", config.Format.Duration)
	}

	return getFirstWebmFrame(tempFile.Name())
}

func ConvertFloatDuration(seconds float64) (time.Duration, bool) {
	if math.IsNaN(seconds) || math.IsInf(seconds, 0) || seconds > math.MaxInt64 {
		return time.Duration(0), false
	}
	if seconds < 0 {
		return time.Duration(0), false
	}
	return time.Duration(seconds * 1e9), true
}

func GetWebmConfig(filepath string) (*FFprobeConfig, error) {
	cmd := exec.Command("ffprobe",
		"-print_format", "json",
		"-v", "quiet", // disable version info
		"-show_format",
		"-show_streams",
		filepath)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	config := new(FFprobeConfig)
	if err := json.NewDecoder(stdout).Decode(config); err != nil {
		return nil, err
	}
	if err := cmd.Wait(); err != nil {
		return nil, err
	}

	if config.Format.FormatName != "matroska,webm" {
		return nil, fmt.Errorf("Only webm supported: %v", config)
	}
	if config.Format.StreamCount == 0 {
		return nil, fmt.Errorf("No Streams found: %v", config)
	}
	if config.Format.Duration < 0 || config.Format.Duration > maxWebmDuration {
		return nil, fmt.Errorf("Invalid Duration %v", config)
	}

	var videoFound bool
	// Only check for a video stream, since we will just mute it on output.
	for _, stream := range config.Streams {
		if stream.CodecType == "video" {
			if stream.CodecName == "vp8" || stream.CodecName == "vp9" {
				videoFound = true
				break
			}
		}
	}

	if !videoFound {
		return nil, &BadWebmFormatErr{fmt.Errorf("Bad Video %v", config)}
	}

	return config, nil
}

func getFirstWebmFrame(filepath string) (img.Image, error) {
	cmd := exec.Command("ffmpeg",
		"-i", filepath,
		"-v", "quiet", // disable version info
		"-frames:v", "1",
		"-ss", "1", // Grab the last frame before the first second
		"-codec:v", "png",
		"-f", "image2pipe",
		"-")
	// PNG data comes across stdout
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	im, _, err := img.Decode(stdout)
	if err != nil {
		return nil, err
	}
	if err := cmd.Wait(); err != nil {
		return nil, err
	}

	return im, nil
}
<commit_msg>Fix handling of short webms<commit_after>package image

import (
	"encoding/json"
	"fmt"
	img "image"
	"image/draw"
	"io"
	"log"
	"math"
	"os"
	"os/exec"
	"time"

	"pixur.org/pixur/schema"

	"github.com/nfnt/resize"

	"image/gif"
	"image/jpeg"
	"image/png"
)

// TODO: maybe make this into it's own package
const (
	DefaultThumbnailWidth  = 160
	DefaultThumbnailHeight = 160

	maxWebmDuration = 60*2 + 1 // Two minutes, with 1 second of leeway
)

type BadWebmFormatErr struct {
	error
}

func FillImageConfig(f *os.File, p *schema.Pic) (img.Image, error) {
	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
		return nil, err
	}
	defer f.Seek(0, os.SEEK_SET)

	im, imgType, err := img.Decode(f)
	if err == img.ErrFormat {
		// Try Webm
		im, err = fillImageConfigFromWebm(f, p)
		if err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	} else {
		// TODO: handle this error
		p.Mime, _ = schema.FromImageFormat(imgType)
		p.Width = int64(im.Bounds().Dx())
		p.Height = int64(im.Bounds().Dy())
	}

	if p.Mime == schema.Pic_GIF {
		if _, err := f.Seek(0, os.SEEK_SET); err != nil {
			return nil, err
		}

		GIF, err := gif.DecodeAll(f)
		if err != nil {
			return nil, err
		}
		// Ignore gifs that have only one frame
		if len(GIF.Delay) > 1 {
			p.AnimationInfo = &schema.AnimationInfo{
				Duration: GetGifDuration(GIF),
			}
			// TODO: maybe skip the first second of frames like webm
		}
	}

	return im, nil
}

// TODO: add tests for this
func GetGifDuration(g *gif.GIF) *schema.Duration {
	var duration time.Duration
	// TODO: check for overflow
	// each delay unit is 1/100 of a second
	for _, frameHundredths := range g.Delay {
		duration += time.Millisecond * time.Duration(10*frameHundredths)
	}

	return schema.FromDuration(duration)
}

// TODO: interpret image rotation metadata
func MakeThumbnail(im img.Image) img.Image {
	bounds := findMaxSquare(im.Bounds())
	largeSquareImage := img.NewNRGBA(bounds)
	draw.Draw(largeSquareImage, bounds, im, bounds.Min, draw.Src)
	return resize.Resize(DefaultThumbnailWidth, DefaultThumbnailHeight, largeSquareImage,
		resize.NearestNeighbor)
}

func SaveThumbnail(im img.Image, p *schema.Pic, pixPath string) error {
	f, err := os.Create(p.ThumbnailPath(pixPath))
	if err != nil {
		return err
	}
	defer f.Close()
	return jpeg.Encode(f, im, nil)
}

func findMaxSquare(bounds img.Rectangle) img.Rectangle {
	width := bounds.Dx()
	height := bounds.Dy()
	if height < width {
		missingSpace := width - height
		return img.Rectangle{
			Min: img.Point{
				X: bounds.Min.X + missingSpace/2,
				Y: bounds.Min.Y,
			},
			Max: img.Point{
				X: bounds.Min.X + missingSpace/2 + height,
				Y: bounds.Max.Y,
			},
		}
	} else {
		missingSpace := height - width
		return img.Rectangle{
			Min: img.Point{
				X: bounds.Min.X,
				Y: bounds.Min.Y + missingSpace/2,
			},
			Max: img.Point{
				X: bounds.Max.X,
				Y: bounds.Min.Y + missingSpace/2 + width,
			},
		}
	}
}

type FFprobeConfig struct {
	Streams []FFprobeStream `json:"streams"`
	Format  FFprobeFormat   `json:"format"`
}

type FFprobeFormat struct {
	StreamCount int     `json:"nb_streams"`
	FormatName  string  `json:"format_name"`
	Duration    float64 `json:"duration,string"`
}

type FFprobeStream struct {
	CodecName string `json:"codec_name"`
	CodecType string `json:"codec_type"`
	Width     int64  `json:"width"`
	Height    int64  `json:"height"`
}

func fillImageConfigFromWebm(tempFile *os.File, p *schema.Pic) (img.Image, error) {
	config, err := GetWebmConfig(tempFile.Name())
	if err != nil {
		return nil, err
	}
	p.Mime = schema.Pic_WEBM
	// Handle the 0 and 1 case
	for _, stream := range config.Streams {
		p.Width = stream.Width
		p.Height = stream.Height
		break
	}

	if dur, success := ConvertFloatDuration(config.Format.Duration); success {
		if dur >= time.Nanosecond {
			p.AnimationInfo = &schema.AnimationInfo{
				Duration: schema.FromDuration(dur),
			}
		}
	} else {
		log.Println("Invalid duration from ffmpeg: ", config.Format.Duration)
	}

	return getFirstWebmFrame(tempFile.Name())
}

func ConvertFloatDuration(seconds float64) (time.Duration, bool) {
	if math.IsNaN(seconds) || math.IsInf(seconds, 0) || seconds > math.MaxInt64 {
		return time.Duration(0), false
	}
	if seconds < 0 {
		return time.Duration(0), false
	}
	return time.Duration(seconds * 1e9), true
}

func GetWebmConfig(filepath string) (*FFprobeConfig, error) {
	cmd := exec.Command("ffprobe",
		"-print_format", "json",
		"-v", "quiet", // disable version info
		"-show_format",
		"-show_streams",
		filepath)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	config := new(FFprobeConfig)
	if err := json.NewDecoder(stdout).Decode(config); err != nil {
		return nil, err
	}
	if err := cmd.Wait(); err != nil {
		return nil, err
	}

	if config.Format.FormatName != "matroska,webm" {
		return nil, fmt.Errorf("Only webm supported: %v", config)
	}
	if config.Format.StreamCount == 0 {
		return nil, fmt.Errorf("No Streams found: %v", config)
	}
	if config.Format.Duration < 0 || config.Format.Duration > maxWebmDuration {
		return nil, fmt.Errorf("Invalid Duration %v", config)
	}

	var videoFound bool
	// Only check for a video stream, since we will just mute it on output.
	for _, stream := range config.Streams {
		if stream.CodecType == "video" {
			if stream.CodecName == "vp8" || stream.CodecName == "vp9" {
				videoFound = true
				break
			}
		}
	}

	if !videoFound {
		return nil, &BadWebmFormatErr{fmt.Errorf("Bad Video %v", config)}
	}

	return config, nil
}

// The Idea here is to read the first 1 second of video data, and stop after
// that.  The last frame received will be the thumbnail.  Normally the first
// frame of video will be an intro or darker frame.  This is undesirable, so
// we give the video 1 second of warm up time.
// This doesn't use the -ss argument, because if the video is less than one
// second, no output will be produced.  In general, if there is not a frame
// after the first duration, no output will be produced.  If there was a video
// with a single frame that had a duration of 2 seconds, no output would be
// produced.  This also means that we can't just seek to some percentage of
// the way through the video and get a frame.
// This takes a different approach: output every frame, stopping after the
// first second of video time.  Then, keep the last frame produced.  This
// takes advantage of PNGs being self contained, allowing this to read the
// concatenated image output produced by ffmpeg.  Thus, videos shorter than
// 1 second will produce *something*, and videos longer than 1 don't need to
// be completely parsed.
// TODO: This is pretty slow, find a faster way.
func getFirstWebmFrame(filepath string) (img.Image, error) {
	cmd := exec.Command("ffmpeg",
		"-i", filepath,
		"-v", "quiet", // disable version info
		"-t", "1.0", // Grab the last frame before the first second
		"-frames:v", "120", // Handle up to 120fps video, then give up.
		"-codec:v", "png",
		"-compression_level", "0", // Don't bother compressing
		"-f", "image2pipe",
		"-")
	// PNG data comes across stdout
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	defer stdout.Close()
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	defer cmd.Process.Kill()
	// Avoid checking more than 120 frames
	maxFrames := 120
	var im img.Image

	for i := 0; i < maxFrames; i++ {
		// Can't use image.Decode because it reads too far ahead.
		lastIm, err := png.Decode(stdout)
		if err == io.ErrUnexpectedEOF {
			break
		} else if err != nil {
			return nil, err
		}
		im = lastIm
	}
	if err := cmd.Wait(); err != nil {
		return nil, err
	}

	if im == nil {
		return nil, &BadWebmFormatErr{fmt.Errorf("No frames in webm")}
	}

	return im, nil
}
<|endoftext|>


<commit_before>package images

import (
	"encoding/json"
	"errors"
)

type IdentityImage struct {
	KeyUID       string
	Name         string
	Payload      []byte
	Width        int
	Height       int
	FileSize     int
	ResizeTarget int
}

func (i IdentityImage) GetType() (ImageType, error) {
	it := GetType(i.Payload)
	if it == UNKNOWN {
		return it, errors.New("unsupported file type")
	}

	return it, nil
}

func (i IdentityImage) GetDataURI() (string, error) {
	return GetPayloadDataURI(i.Payload)
}

func (i IdentityImage) MarshalJSON() ([]byte, error) {
	uri, err := i.GetDataURI()
	if err != nil {
		return nil, err
	}

	temp := struct {
		KeyUID       string `json:"key_uid"`
		Name         string `json:"type"`
		URI          string `json:"uri"`
		Width        int    `json:"width"`
		Height       int    `json:"height"`
		FileSize     int    `json:"file_size"`
		ResizeTarget int    `json:"resize_target"`
	}{
		KeyUID:       i.KeyUID,
		Name:         i.Name,
		URI:          uri,
		Width:        i.Width,
		Height:       i.Height,
		FileSize:     i.FileSize,
		ResizeTarget: i.ResizeTarget,
	}

	return json.Marshal(temp)
}
<commit_msg>Address consistent json field case convention - use camelCase<commit_after>package images

import (
	"encoding/json"
	"errors"
)

type IdentityImage struct {
	KeyUID       string
	Name         string
	Payload      []byte
	Width        int
	Height       int
	FileSize     int
	ResizeTarget int
}

func (i IdentityImage) GetType() (ImageType, error) {
	it := GetType(i.Payload)
	if it == UNKNOWN {
		return it, errors.New("unsupported file type")
	}

	return it, nil
}

func (i IdentityImage) GetDataURI() (string, error) {
	return GetPayloadDataURI(i.Payload)
}

func (i IdentityImage) MarshalJSON() ([]byte, error) {
	uri, err := i.GetDataURI()
	if err != nil {
		return nil, err
	}

	temp := struct {
		KeyUID       string `json:"keyUid"`
		Name         string `json:"type"`
		URI          string `json:"uri"`
		Width        int    `json:"width"`
		Height       int    `json:"height"`
		FileSize     int    `json:"fileSize"`
		ResizeTarget int    `json:"resizeTarget"`
	}{
		KeyUID:       i.KeyUID,
		Name:         i.Name,
		URI:          uri,
		Width:        i.Width,
		Height:       i.Height,
		FileSize:     i.FileSize,
		ResizeTarget: i.ResizeTarget,
	}

	return json.Marshal(temp)
}
<|endoftext|>


<commit_before>// LoL Cruncher - A Historical League of Legends Statistics Tracker
// Copyright (C) 2015  Jason Chu (1lann) 1lanncontact@gmail.com

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.

// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package controllers

import (
	"cruncher/app/models/dataFormat"
	"cruncher/app/models/database"
	"cruncher/app/models/query"
	"github.com/revel/revel"
	// "github.com/revel/revel/cache"
	"strings"
	// "time"
)

type View struct {
	*revel.Controller
}

func (c View) Index() revel.Result {
	if !database.IsConnected {
		return c.RenderTemplate("errors/down.html")
	}
	return c.Render()
}

func (c View) About() revel.Result {
	return c.Render()
}

func (c View) Robots() revel.Result {
	return c.RenderText("Sitemap: https://lolcruncher.tk/sitemap.xml")
}

func (c View) Request(region, name string) revel.Result {
	region = strings.ToLower(region)

	if !(region == "na" || region == "euw" || region == "eune" ||
		region == "lan" || region == "las" || region == "oce" ||
		region == "br" || region == "ru" || region == "kr" ||
		region == "tr") {
		c.Flash.Error("Sorry, that region isn't supported!")
		return c.Redirect(View.Index)
	}

	var err error
	player := dataFormat.PlayerData{}
	new := false

	// if err = cache.Get(region+":"+dataFormat.NormalizeName(name),
	// 	&player); err != nil {
	// 	player, new, err = query.GetStats(name, region, false)
	// 	revel.WARN.Println("Storing", name, region, "in cache.")
	// 	go cache.Set(region+":"+dataFormat.NormalizeName(player.SummonerName),
	// 		player, time.Hour*2)
	// }

	if err != nil {
		if err == query.ErrDatabaseError {
			return c.RenderTemplate("errors/database.html")
		} else if err == query.ErrDatabaseDisconnected {
			return c.RenderTemplate("errors/down.html")
		} else if err == query.ErrNotFound {
			c.Flash.Error("Sorry, that summoner could not be found!")
			return c.Redirect(View.Index)
		} else if err == query.ErrAPIError {
			c.Flash.Error("Could not connect to Riot Games' servers! Try again later.")
			return c.Redirect(View.Index)
		} else {
			c.Flash.Error("An unknown error has occured, please try again in a few seconds.")
			return c.Redirect(View.Index)
		}
	}

	resolvedName := player.SummonerName

	if strings.Trim(resolvedName, " ") != strings.Trim(name, " ") {
		return c.Redirect("/" + region + "/" + strings.Trim(resolvedName, " "))
	}

	player.RecordStartString = player.RecordStart.Format("2 January 2006")

	c.RenderArgs["new"] = new
	c.RenderArgs["player"] = player
	c.RenderArgs["name"] = resolvedName
	c.RenderArgs["titleName"] = resolvedName + " - LoL Cruncher"
	c.RenderArgs["description"] = "View " + resolvedName +
		"'s League of Legends statistics and champion breakdowns for all " +
		"queues (since " + player.RecordStart.Format("2 January 2006") + ")"
	return c.Render()
}
<commit_msg>Actually make the call to get statistics<commit_after>// LoL Cruncher - A Historical League of Legends Statistics Tracker
// Copyright (C) 2015  Jason Chu (1lann) 1lanncontact@gmail.com

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.

// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package controllers

import (
	"cruncher/app/models/dataFormat"
	"cruncher/app/models/database"
	"cruncher/app/models/query"
	"github.com/revel/revel"
	// "github.com/revel/revel/cache"
	"strings"
	// "time"
)

type View struct {
	*revel.Controller
}

func (c View) Index() revel.Result {
	if !database.IsConnected {
		return c.RenderTemplate("errors/down.html")
	}
	return c.Render()
}

func (c View) About() revel.Result {
	return c.Render()
}

func (c View) Robots() revel.Result {
	return c.RenderText("Sitemap: https://lolcruncher.tk/sitemap.xml")
}

func (c View) Request(region, name string) revel.Result {
	region = strings.ToLower(region)

	if !(region == "na" || region == "euw" || region == "eune" ||
		region == "lan" || region == "las" || region == "oce" ||
		region == "br" || region == "ru" || region == "kr" ||
		region == "tr") {
		c.Flash.Error("Sorry, that region isn't supported!")
		return c.Redirect(View.Index)
	}

	var err error
	player := dataFormat.PlayerData{}
	new := false

	player, new, err = query.GetStats(name, region, false)

	// if err = cache.Get(region+":"+dataFormat.NormalizeName(name),
	// 	&player); err != nil {
	// 	player, new, err = query.GetStats(name, region, false)
	// 	revel.WARN.Println("Storing", name, region, "in cache.")
	// 	go cache.Set(region+":"+dataFormat.NormalizeName(player.SummonerName),
	// 		player, time.Hour*2)
	// }

	if err != nil {
		if err == query.ErrDatabaseError {
			return c.RenderTemplate("errors/database.html")
		} else if err == query.ErrDatabaseDisconnected {
			return c.RenderTemplate("errors/down.html")
		} else if err == query.ErrNotFound {
			c.Flash.Error("Sorry, that summoner could not be found!")
			return c.Redirect(View.Index)
		} else if err == query.ErrAPIError {
			c.Flash.Error("Could not connect to Riot Games' servers! Try again later.")
			return c.Redirect(View.Index)
		} else {
			c.Flash.Error("An unknown error has occured, please try again in a few seconds.")
			return c.Redirect(View.Index)
		}
	}

	resolvedName := player.SummonerName

	if strings.Trim(resolvedName, " ") != strings.Trim(name, " ") {
		return c.Redirect("/" + region + "/" + strings.Trim(resolvedName, " "))
	}

	player.RecordStartString = player.RecordStart.Format("2 January 2006")

	c.RenderArgs["new"] = new
	c.RenderArgs["player"] = player
	c.RenderArgs["name"] = resolvedName
	c.RenderArgs["titleName"] = resolvedName + " - LoL Cruncher"
	c.RenderArgs["description"] = "View " + resolvedName +
		"'s League of Legends statistics and champion breakdowns for all " +
		"queues (since " + player.RecordStart.Format("2 January 2006") + ")"
	return c.Render()
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cpu_test

import (
	. "internal/cpu"
	"internal/godebug"
	"internal/testenv"
	"os"
	"os/exec"
	"strings"
	"testing"
)

func MustHaveDebugOptionsSupport(t *testing.T) {
	if !DebugOptions {
		t.Skipf("skipping test: cpu feature options not supported by OS")
	}
}

func MustSupportFeatureDectection(t *testing.T) {
	// TODO: add platforms that do not have CPU feature detection support.
}

func runDebugOptionsTest(t *testing.T, test string, options string) {
	MustHaveDebugOptionsSupport(t)

	testenv.MustHaveExec(t)

	env := "GODEBUG=" + options

	cmd := exec.Command(os.Args[0], "-test.run="+test)
	cmd.Env = append(cmd.Env, env)

	output, err := cmd.CombinedOutput()
	lines := strings.Fields(string(output))
	lastline := lines[len(lines)-1]

	got := strings.TrimSpace(lastline)
	want := "PASS"
	if err != nil || got != want {
		t.Fatalf("%s with %s: want %s, got %v", test, env, want, got)
	}
}

func TestDisableAllCapabilities(t *testing.T) {
	MustSupportFeatureDectection(t)
	runDebugOptionsTest(t, "TestAllCapabilitiesDisabled", "cpu.all=off")
}

func TestAllCapabilitiesDisabled(t *testing.T) {
	MustHaveDebugOptionsSupport(t)

	if godebug.Get("cpu.all") != "off" {
		t.Skipf("skipping test: GODEBUG=cpu.all=off not set")
	}

	for _, o := range Options {
		want := false
		if got := *o.Feature; got != want {
			t.Errorf("%v: expected %v, got %v", o.Name, want, got)
		}
	}
}
<commit_msg>internal/cpu: revise test to make it work properly with -cover<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cpu_test

import (
	. "internal/cpu"
	"internal/godebug"
	"internal/testenv"
	"os"
	"os/exec"
	"testing"
)

func MustHaveDebugOptionsSupport(t *testing.T) {
	if !DebugOptions {
		t.Skipf("skipping test: cpu feature options not supported by OS")
	}
}

func MustSupportFeatureDectection(t *testing.T) {
	// TODO: add platforms that do not have CPU feature detection support.
}

func runDebugOptionsTest(t *testing.T, test string, options string) {
	MustHaveDebugOptionsSupport(t)

	testenv.MustHaveExec(t)

	env := "GODEBUG=" + options

	cmd := exec.Command(os.Args[0], "-test.run="+test)
	cmd.Env = append(cmd.Env, env)

	output, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("%s with %s: run failed: %v output:\n%s\n",
			test, env, err, string(output))
	}
}

func TestDisableAllCapabilities(t *testing.T) {
	MustSupportFeatureDectection(t)
	runDebugOptionsTest(t, "TestAllCapabilitiesDisabled", "cpu.all=off")
}

func TestAllCapabilitiesDisabled(t *testing.T) {
	MustHaveDebugOptionsSupport(t)

	if godebug.Get("cpu.all") != "off" {
		t.Skipf("skipping test: GODEBUG=cpu.all=off not set")
	}

	for _, o := range Options {
		want := false
		if got := *o.Feature; got != want {
			t.Errorf("%v: expected %v, got %v", o.Name, want, got)
		}
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

type transferStats struct {
	HeaderSize int
	BodySize   int
	Start      time.Time
	Stop       time.Time
}

type transfer struct {
	requestStats  *transferStats
	responseStats *transferStats
}

var (
	// TODO should use some locks
	transfers           = make(map[*http.Response]*transfer)
	transferBuckets     = make(map[string][]*http.Response)
	transfersLock       sync.Mutex
	transferBucketsLock sync.Mutex
)

func LogTransfer(key string, res *http.Response) {
	if Config.isLoggingStats {
		transferBucketsLock.Lock()
		transferBuckets[key] = append(transferBuckets[key], res)
		transferBucketsLock.Unlock()
	}
}

type HttpClient struct {
	*http.Client
}

func (c *HttpClient) Do(req *http.Request) (*http.Response, error) {
	traceHttpRequest(req)

	crc := countingRequest(req)
	if req.Body != nil {
		// Only set the body if we have a body, but create the countingRequest
		// anyway to make using zeroed stats easier.
		req.Body = crc
	}

	start := time.Now()
	res, err := c.Client.Do(req)
	if err != nil {
		return res, err
	}

	traceHttpResponse(res)

	cresp := countingResponse(res)
	res.Body = cresp

	if Config.isLoggingStats {
		reqHeaderSize := 0
		resHeaderSize := 0

		if dump, err := httputil.DumpRequest(req, false); err == nil {
			reqHeaderSize = len(dump)
		}

		if dump, err := httputil.DumpResponse(res, false); err == nil {
			resHeaderSize = len(dump)
		}

		reqstats := &transferStats{HeaderSize: reqHeaderSize, BodySize: crc.Count}

		// Response body size cannot be figured until it is read. Do not rely on a Content-Length
		// header because it may not exist or be -1 in the case of chunked responses.
		resstats := &transferStats{HeaderSize: resHeaderSize, Start: start}
		t := &transfer{requestStats: reqstats, responseStats: resstats}
		transfersLock.Lock()
		transfers[res] = t
		transfersLock.Unlock()
	}

	return res, err
}

func (c *Configuration) HttpClient() *HttpClient {
	if c.httpClient != nil {
		return c.httpClient
	}

	tr := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   5 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 5 * time.Second,
	}

	sslVerify, _ := c.GitConfig("http.sslverify")
	if sslVerify == "false" || Config.GetenvBool("GIT_SSL_NO_VERIFY", false) {
		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	c.httpClient = &HttpClient{
		&http.Client{Transport: tr, CheckRedirect: checkRedirect},
	}

	return c.httpClient
}

func checkRedirect(req *http.Request, via []*http.Request) error {
	if len(via) >= 3 {
		return errors.New("stopped after 3 redirects")
	}

	oldest := via[0]
	for key, _ := range oldest.Header {
		if key == "Authorization" {
			if req.URL.Scheme != oldest.URL.Scheme || req.URL.Host != oldest.URL.Host {
				continue
			}
		}
		req.Header.Set(key, oldest.Header.Get(key))
	}

	tracerx.Printf("api: redirect %s %s to %s", oldest.Method, oldest.URL, req.URL)

	return nil
}

var tracedTypes = []string{"json", "text", "xml", "html"}

func traceHttpRequest(req *http.Request) {
	tracerx.Printf("HTTP: %s %s", req.Method, req.URL.String())

	if Config.isTracingHttp == false {
		return
	}

	dump, err := httputil.DumpRequest(req, false)
	if err != nil {
		return
	}

	scanner := bufio.NewScanner(bytes.NewBuffer(dump))
	for scanner.Scan() {
		fmt.Fprintf(os.Stderr, "> %s\n", scanner.Text())
	}
}

func traceHttpResponse(res *http.Response) {
	if res == nil {
		return
	}

	tracerx.Printf("HTTP: %d", res.StatusCode)

	if Config.isTracingHttp == false {
		return
	}

	dump, err := httputil.DumpResponse(res, false)
	if err != nil {
		return
	}

	scanner := bufio.NewScanner(bytes.NewBuffer(dump))
	for scanner.Scan() {
		fmt.Fprintf(os.Stderr, "< %s\n", scanner.Text())
	}
}

func countingRequest(req *http.Request) *countingReadCloser {
	return &countingReadCloser{request: req, ReadCloser: req.Body}
}

func countingResponse(res *http.Response) *countingReadCloser {
	return &countingReadCloser{response: res, ReadCloser: res.Body}
}

type countingReadCloser struct {
	Count    int
	request  *http.Request
	response *http.Response
	io.ReadCloser
}

func (c *countingReadCloser) Read(b []byte) (int, error) {
	n, err := c.ReadCloser.Read(b)
	if err != nil && err != io.EOF {
		return n, err
	}

	c.Count += n

	if Config.isTracingHttp {
		contentType := ""
		if c.response != nil { // Response, only print certain kinds of data
			contentType = strings.ToLower(strings.SplitN(c.response.Header.Get("Content-Type"), ";", 2)[0])
		} else {
			contentType = strings.ToLower(strings.SplitN(c.request.Header.Get("Content-Type"), ";", 2)[0])
		}

		for _, tracedType := range tracedTypes {
			if strings.Contains(contentType, tracedType) {
				fmt.Fprint(os.Stderr, string(b[0:n]))
			}
		}
	}

	if err == io.EOF && Config.isLoggingStats {
		// This transfer is done, we're checking it this way so we can also
		// catch transfers where the caller forgets to Close() the Body.
		if c.response != nil {
			transfersLock.Lock()
			if transfer, ok := transfers[c.response]; ok {
				transfer.responseStats.BodySize = c.Count
				transfer.responseStats.Stop = time.Now()
			}
			transfersLock.Unlock()
		}
	}
	return n, err
}

// LogHttpStats is intended to be called after all HTTP operations for the
// commmand have finished. It dumps k/v logs, one line per transfer into
// a log file with the current timestamp.
func LogHttpStats() {
	if !Config.isLoggingStats {
		return
	}

	file, err := statsLogFile()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error logging http stats: %s\n", err)
		return
	}

	fmt.Fprintf(file, "concurrent=%d batch=%v time=%d version=%s\n", Config.ConcurrentTransfers(), Config.BatchTransfer(), time.Now().Unix(), Version)

	for key, responses := range transferBuckets {
		for _, response := range responses {
			stats := transfers[response]
			fmt.Fprintf(file, "key=%s reqheader=%d reqbody=%d resheader=%d resbody=%d restime=%d status=%d url=%s\n",
				key,
				stats.requestStats.HeaderSize,
				stats.requestStats.BodySize,
				stats.responseStats.HeaderSize,
				stats.responseStats.BodySize,
				stats.responseStats.Stop.Sub(stats.responseStats.Start).Nanoseconds(),
				response.StatusCode,
				response.Request.URL)
		}
	}

	fmt.Fprintf(os.Stderr, "HTTP Stats logged to file %s\n", file.Name())
}

func statsLogFile() (*os.File, error) {
	logBase := filepath.Join(LocalLogDir, "http")
	if err := os.MkdirAll(logBase, 0755); err != nil {
		return nil, err
	}

	logFile := fmt.Sprintf("http-%d.log", time.Now().Unix())
	return os.Create(filepath.Join(logBase, logFile))
}
<commit_msg>cleaner http dump output<commit_after>package lfs

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

type transferStats struct {
	HeaderSize int
	BodySize   int
	Start      time.Time
	Stop       time.Time
}

type transfer struct {
	requestStats  *transferStats
	responseStats *transferStats
}

var (
	// TODO should use some locks
	transfers           = make(map[*http.Response]*transfer)
	transferBuckets     = make(map[string][]*http.Response)
	transfersLock       sync.Mutex
	transferBucketsLock sync.Mutex
)

func LogTransfer(key string, res *http.Response) {
	if Config.isLoggingStats {
		transferBucketsLock.Lock()
		transferBuckets[key] = append(transferBuckets[key], res)
		transferBucketsLock.Unlock()
	}
}

type HttpClient struct {
	*http.Client
}

func (c *HttpClient) Do(req *http.Request) (*http.Response, error) {
	traceHttpRequest(req)

	crc := countingRequest(req)
	if req.Body != nil {
		// Only set the body if we have a body, but create the countingRequest
		// anyway to make using zeroed stats easier.
		req.Body = crc
	}

	start := time.Now()
	res, err := c.Client.Do(req)
	if err != nil {
		return res, err
	}

	traceHttpResponse(res)

	cresp := countingResponse(res)
	res.Body = cresp

	if Config.isLoggingStats {
		reqHeaderSize := 0
		resHeaderSize := 0

		if dump, err := httputil.DumpRequest(req, false); err == nil {
			reqHeaderSize = len(dump)
		}

		if dump, err := httputil.DumpResponse(res, false); err == nil {
			resHeaderSize = len(dump)
		}

		reqstats := &transferStats{HeaderSize: reqHeaderSize, BodySize: crc.Count}

		// Response body size cannot be figured until it is read. Do not rely on a Content-Length
		// header because it may not exist or be -1 in the case of chunked responses.
		resstats := &transferStats{HeaderSize: resHeaderSize, Start: start}
		t := &transfer{requestStats: reqstats, responseStats: resstats}
		transfersLock.Lock()
		transfers[res] = t
		transfersLock.Unlock()
	}

	return res, err
}

func (c *Configuration) HttpClient() *HttpClient {
	if c.httpClient != nil {
		return c.httpClient
	}

	tr := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   5 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 5 * time.Second,
	}

	sslVerify, _ := c.GitConfig("http.sslverify")
	if sslVerify == "false" || Config.GetenvBool("GIT_SSL_NO_VERIFY", false) {
		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	c.httpClient = &HttpClient{
		&http.Client{Transport: tr, CheckRedirect: checkRedirect},
	}

	return c.httpClient
}

func checkRedirect(req *http.Request, via []*http.Request) error {
	if len(via) >= 3 {
		return errors.New("stopped after 3 redirects")
	}

	oldest := via[0]
	for key, _ := range oldest.Header {
		if key == "Authorization" {
			if req.URL.Scheme != oldest.URL.Scheme || req.URL.Host != oldest.URL.Host {
				continue
			}
		}
		req.Header.Set(key, oldest.Header.Get(key))
	}

	tracerx.Printf("api: redirect %s %s to %s", oldest.Method, oldest.URL, req.URL)

	return nil
}

var tracedTypes = []string{"json", "text", "xml", "html"}

func traceHttpRequest(req *http.Request) {
	tracerx.Printf("HTTP: %s %s", req.Method, req.URL.String())

	if Config.isTracingHttp == false {
		return
	}

	dump, err := httputil.DumpRequest(req, false)
	if err != nil {
		return
	}

	traceHttpDump(dump)
}

func traceHttpResponse(res *http.Response) {
	if res == nil {
		return
	}

	tracerx.Printf("HTTP: %d", res.StatusCode)

	if Config.isTracingHttp == false {
		return
	}

	dump, err := httputil.DumpResponse(res, false)
	if err != nil {
		return
	}

	if isTraceableContent(res.Header) {
		fmt.Fprintf(os.Stderr, "\n\n")
	} else {
		fmt.Fprintf(os.Stderr, "\n")
	}

	traceHttpDump(dump)
}

func traceHttpDump(dump []byte) {
	scanner := bufio.NewScanner(bytes.NewBuffer(dump))

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix("authorization: basic", strings.ToLower(line)) {
			fmt.Fprintf(os.Stderr, "< Authorization: Basic * * * * *\n")
		} else {
			fmt.Fprintf(os.Stderr, "< %s\n", line)
		}
	}
}

func isTraceableContent(h http.Header) bool {
	ctype := strings.ToLower(strings.SplitN(h.Get("Content-Type"), ";", 2)[0])
	for _, tracedType := range tracedTypes {
		if strings.Contains(ctype, tracedType) {
			return true
		}
	}
	return false
}

func countingRequest(req *http.Request) *countingReadCloser {
	return &countingReadCloser{request: req, ReadCloser: req.Body, isTraceableType: isTraceableContent(req.Header)}
}

func countingResponse(res *http.Response) *countingReadCloser {
	return &countingReadCloser{response: res, ReadCloser: res.Body, isTraceableType: isTraceableContent(res.Header)}
}

type countingReadCloser struct {
	Count           int
	request         *http.Request
	response        *http.Response
	isTraceableType bool
	io.ReadCloser
}

func (c *countingReadCloser) Read(b []byte) (int, error) {
	n, err := c.ReadCloser.Read(b)
	if err != nil && err != io.EOF {
		return n, err
	}

	c.Count += n

	if Config.isTracingHttp && c.isTraceableType {
		fmt.Fprint(os.Stderr, string(b[0:n]))
	}

	if err == io.EOF && Config.isLoggingStats {
		// This transfer is done, we're checking it this way so we can also
		// catch transfers where the caller forgets to Close() the Body.
		if c.response != nil {
			transfersLock.Lock()
			if transfer, ok := transfers[c.response]; ok {
				transfer.responseStats.BodySize = c.Count
				transfer.responseStats.Stop = time.Now()
			}
			transfersLock.Unlock()
		}
	}
	return n, err
}

// LogHttpStats is intended to be called after all HTTP operations for the
// commmand have finished. It dumps k/v logs, one line per transfer into
// a log file with the current timestamp.
func LogHttpStats() {
	if !Config.isLoggingStats {
		return
	}

	file, err := statsLogFile()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error logging http stats: %s\n", err)
		return
	}

	fmt.Fprintf(file, "concurrent=%d batch=%v time=%d version=%s\n", Config.ConcurrentTransfers(), Config.BatchTransfer(), time.Now().Unix(), Version)

	for key, responses := range transferBuckets {
		for _, response := range responses {
			stats := transfers[response]
			fmt.Fprintf(file, "key=%s reqheader=%d reqbody=%d resheader=%d resbody=%d restime=%d status=%d url=%s\n",
				key,
				stats.requestStats.HeaderSize,
				stats.requestStats.BodySize,
				stats.responseStats.HeaderSize,
				stats.responseStats.BodySize,
				stats.responseStats.Stop.Sub(stats.responseStats.Start).Nanoseconds(),
				response.StatusCode,
				response.Request.URL)
		}
	}

	fmt.Fprintf(os.Stderr, "HTTP Stats logged to file %s\n", file.Name())
}

func statsLogFile() (*os.File, error) {
	logBase := filepath.Join(LocalLogDir, "http")
	if err := os.MkdirAll(logBase, 0755); err != nil {
		return nil, err
	}

	logFile := fmt.Sprintf("http-%d.log", time.Now().Unix())
	return os.Create(filepath.Join(logBase, logFile))
}
<|endoftext|>


<commit_before>package parse

import (
	"github.com/socialmachines/soma/ast"
	"github.com/socialmachines/soma/rt"
	"github.com/socialmachines/soma/scan"
)

// expression :=
//	primary [messages cascaded_messages]
//    | assignment
//
func (p *Parser) parseExpr() rt.Expr {
	recv := p.parsePrimary()

	if define, ok := recv.(*ast.Define); ok {
		return p.parseDefinition(define)
	} else {
		return p.parseMessagePortion(recv)
	}
}

func (p *Parser) parseMessagePortion(recv rt.Expr) rt.Expr {
	if p.isMessageStart() {
		msg := p.parseMessages(recv)
		if p.tok == scan.CASCADE {
			cascade := p.parseCascadeMessages(recv, []rt.Expr{msg})
			return &ast.Cascade{Messages: cascade}
		}
		return msg
	}
	return recv
}

// primary :=
//	ident | assignment | global | block | paren_expr
//
func (p *Parser) parsePrimary() (recv rt.Expr) {
	switch p.tok {
	case scan.IDENT:
		name := p.expect(scan.IDENT)
		if p.tok == scan.ASSIGN {
			recv = p.parseAssignment(name)
		} else {
			recv = &ast.Local{Value: name}
		}
	case scan.GLOBAL:
		name := p.expect(scan.GLOBAL)
		recv = &ast.Global{Value: name}
	case scan.LBRACE:
		recv = p.parseBlock()
	case scan.LPAREN:
		recv = p.parseParenExpr()
	default:
		p.error(p.pos, "expected identifier, block, array, map, number, or string found '%s'", p.lit)
		p.next()
	}
	return
}

func (p *Parser) isPrimary() bool {
	return p.tok == scan.IDENT ||
		p.tok == scan.GLOBAL ||
		p.tok == scan.LBRACE ||
		p.tok == scan.LPAREN
}

// paren_expr :=
//	 LPAREN GLOBAL RPAREN
// | LPAREN IDENT GLOBAL RPAREN
// | LPAREN expression RPAREN
//
func (p *Parser) parseParenExpr() (expr rt.Expr) {
	p.expect(scan.LPAREN)

	recv := p.parsePrimary()
	switch recv.(type) {
	case *ast.Global:
		global := recv.(*ast.Global)
		if p.tok == scan.RPAREN {
			expr = createDefinition(global.Value, "")
		} else {
			expr = p.parseMessagePortion(recv)
		}
	case *ast.Local:
		local := recv.(*ast.Local)
		if p.tok == scan.GLOBAL {
			global := p.expect(scan.GLOBAL)
			expr = createDefinition(global, local.Value)
		}
	default:
		expr = p.parseMessagePortion(recv)
	}

	p.expect(scan.RPAREN)
	return
}
<commit_msg>Corrected parsePrimary error message<commit_after>package parse

import (
	"github.com/socialmachines/soma/ast"
	"github.com/socialmachines/soma/rt"
	"github.com/socialmachines/soma/scan"
)

// expression :=
//	primary [messages cascaded_messages]
//    | assignment
//
func (p *Parser) parseExpr() rt.Expr {
	recv := p.parsePrimary()

	if define, ok := recv.(*ast.Define); ok {
		return p.parseDefinition(define)
	} else {
		return p.parseMessagePortion(recv)
	}
}

func (p *Parser) parseMessagePortion(recv rt.Expr) rt.Expr {
	if p.isMessageStart() {
		msg := p.parseMessages(recv)
		if p.tok == scan.CASCADE {
			cascade := p.parseCascadeMessages(recv, []rt.Expr{msg})
			return &ast.Cascade{Messages: cascade}
		}
		return msg
	}
	return recv
}

// primary :=
//	ident | assignment | global | block | paren_expr
//
func (p *Parser) parsePrimary() (recv rt.Expr) {
	switch p.tok {
	case scan.IDENT:
		name := p.expect(scan.IDENT)
		if p.tok == scan.ASSIGN {
			recv = p.parseAssignment(name)
		} else {
			recv = &ast.Local{Value: name}
		}
	case scan.GLOBAL:
		name := p.expect(scan.GLOBAL)
		recv = &ast.Global{Value: name}
	case scan.LBRACE:
		recv = p.parseBlock()
	case scan.LPAREN:
		recv = p.parseParenExpr()
	default:
		p.error(p.pos, "expected identifier, block, or expression. found '%s'", p.lit)
		p.next()
	}
	return
}

func (p *Parser) isPrimary() bool {
	return p.tok == scan.IDENT ||
		p.tok == scan.GLOBAL ||
		p.tok == scan.LBRACE ||
		p.tok == scan.LPAREN
}

// paren_expr :=
//	 LPAREN GLOBAL RPAREN
// | LPAREN IDENT GLOBAL RPAREN
// | LPAREN expression RPAREN
//
func (p *Parser) parseParenExpr() (expr rt.Expr) {
	p.expect(scan.LPAREN)

	recv := p.parsePrimary()
	switch recv.(type) {
	case *ast.Global:
		global := recv.(*ast.Global)
		if p.tok == scan.RPAREN {
			expr = createDefinition(global.Value, "")
		} else {
			expr = p.parseMessagePortion(recv)
		}
	case *ast.Local:
		local := recv.(*ast.Local)
		if p.tok == scan.GLOBAL {
			global := p.expect(scan.GLOBAL)
			expr = createDefinition(global, local.Value)
		}
	default:
		expr = p.parseMessagePortion(recv)
	}

	p.expect(scan.RPAREN)
	return
}
<|endoftext|>


<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

// To enable debug output when testing, use "go test -debug"

package parse

import (
	"encoding/json"
	"github.com/demizer/go-elog"
	"reflect"
	"strings"
	"testing"
)

var lexParseTests LexTests

func TestSectionLevelsAdd(t *testing.T) {
	var p sectionLevels
	lvl := p.Add('=', true, 5)
	if lvl != 1 {
		t.Errorf("Improper level on first add, Got level: %d, expected: %d", lvl, 1)
	}
}

func TestSectionLevelsString(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	out := p.String()
	expect := "level: 1, rune: '=', overline: true, length: 5\nlevel: 2, rune: '-', " +
		"overline: true, length: 8\nlevel: 3, rune: '~', overline: false, length: 6\n"
	if out != expect {
		t.Errorf("String output mismatch!\nExpect:\n\n\t%q,\nGot:\n\n\t%q\n", expect, out)
	}
}

func TestSectionLevelsFind(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	lvl := p.Find('-')
	if lvl == -1 {
		t.Errorf("Level not found!\nExpect:\n\n\t%d\nGot:\n\n\t%t\n", 2, lvl)
	}
	if lvl != 2 {
		t.Errorf("Level not correct!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", 2, lvl)
	}
}

func TestSectionLevelsFindNoResult(t *testing.T) {
	var p sectionLevels
	lvl := p.Find('-')
	if lvl > 0 {
		t.Errorf("Should not find any levels!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", -1, lvl)
	}
}

func TestSectionLevelsLevelEmpty(t *testing.T) {
	var p sectionLevels
	lvl := p.Level()
	expect := 0
	if lvl != expect {
		t.Errorf("Empty sectionLevels should return \"%d\"!\nExpect:\t%d\nGot:\t%d\n",
			expect, expect, lvl)
	}

}

func TestSectionLevelsLevel(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	lvl := p.Level()
	if lvl != 3 {
		t.Errorf("Level() returned incorrect level!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", 3, lvl)
	}
}

func parseTest(t *testing.T, testName string) (tree *Tree, err error) {
	if lexParseTests == nil {
		lexParseTests, err = ParseTestData("../testdata/test_lex_sections.dat")
		if err != nil {
			t.Fatal(err)
		}
	}
	test := lexParseTests.SearchByName(testName)
	if test != nil {
		log.Debugf("Test Name: \t%s\n", test.name)
		log.Debugf("Description: \t%s\n", test.description)
		log.Debugf("Test Input:\n-----------\n%s\n----------\n", test.data)
		tree, err = Parse(test.name, test.data)
		if err != nil {
			t.Fatal(err)
		}
	}
	return
}

type checkNode struct {
	t          *testing.T
	testName   string
	pNodeName  string
	pFieldName string
	pFieldVal  interface{}
	pFieldType reflect.Type
	eFieldName string
	eFieldVal  interface{}
	eFieldType reflect.Type
	id         int
}

func (c *checkNode) error(args ...interface{}) {
	c.t.Error(args...)
}

func (c *checkNode) errorf(format string, args ...interface{}) {
	c.t.Errorf(format, args...)
}

func (c *checkNode) dError() {
	if c.pFieldName == "Rune" {
		c.t.Errorf("Got: %s.%s = %#v (%v) (%v) (Id: %d),\n\tExpect: %s.%s = %#v (%v)\n",
			c.pNodeName, c.pFieldName, c.pFieldVal, string(c.pFieldVal.(int32)),
			c.pFieldType, c.id, c.testName, c.eFieldName, c.eFieldVal, c.eFieldType)
		return
	}
	c.t.Errorf("Got: %s.%s = %#v (%v) (Id: %d),\n\tExpect: %s.%s = %#v (%v)\n",
		c.pNodeName, c.pFieldName, c.pFieldVal, c.pFieldType, c.id, c.testName,
		c.eFieldName, c.eFieldVal, c.eFieldType)
}

func (c *checkNode) updateState(pVal reflect.Value, eVal interface{}, field int) bool {
	// Actual parsed metadata
	c.pNodeName = pVal.Type().Name()
	c.pFieldName = pVal.Type().Field(field).Name
	c.pFieldVal = pVal.Field(field).Interface()
	c.pFieldType = pVal.Type().Field(field).Type
	c.id = pVal.FieldByName("Id").Interface().(int)

	// Expected parser metadata
	c.eFieldName = strings.ToLower(string(c.pFieldName[0])) + c.pFieldName[1:]
	c.eFieldVal = eVal.(map[string]interface{})[c.eFieldName]
	c.eFieldType = reflect.TypeOf(c.eFieldVal)

	pField := pVal.Field(field)
	if pField.Kind() == reflect.Ptr && pField.IsNil() {
		// Overline adornment nodes can be null
		if c.pFieldName == "OverLine" {
			return false
		} else {
			c.dError()
			return false
		}
	} else if c.pFieldVal == nil {
		c.dError()
		return false
	}

	return true
}

func (c *checkNode) checkFields(pNode Node, expect interface{}) {
	pVal := reflect.ValueOf(pNode).Elem()
	for i := 0; i < pVal.NumField(); i++ {

		if c.updateState(pVal, expect, i) == false {
			continue
		}

		switch c.pFieldName {
		case "Type":
			if c.pFieldVal.(NodeType).String() != c.eFieldVal {
				c.dError()
			}
		case "Id", "Level", "Length":
			if float64(c.pFieldVal.(int)) != c.eFieldVal {
				c.dError()
			}
		case "Line":
			if float64(c.pFieldVal.(Line)) != c.eFieldVal {
				c.dError()
			}
		case "StartPosition":
			if float64(c.pFieldVal.(StartPosition)) != c.eFieldVal {
				c.dError()
			}
		case "OverLine", "UnderLine":
			// c.checkAdornmentNode(c.pFieldVal.(*AdornmentNode), c.eFieldVal, c.testName)
			c.error("Overline/Underline not implemented!")
		case "NodeList":
			c.error("NodeList not implemented!")
		default:
			if c.pFieldVal != c.eFieldVal {
				c.dError()
			}
		}
	}

}

// checkParseNodes is a recursive function that compares the resulting nodes (pNodes) from the
// parser with the expected output from the testdata (eNodes).
func checkParseNodes(t *testing.T, pNodes *NodeList, eNodes []interface{}, testName string) (errors []error) {
	state := &checkNode{t: t, testName: testName}
	for pNum, pNode := range *pNodes {
		state.checkFields(pNode, eNodes[pNum])
	}
	return
}

func TestParseSectionTitlePara(t *testing.T) {
	testName := "SectionTitlePara"
	tree, err := parseTest(t, testName)
	if err != nil {
		t.Error(err)
	}
	test := lexParseTests.SearchByName(testName)
	var nodeList []interface{}
	err = json.Unmarshal([]byte(test.expectTree), &nodeList)
	if err != nil {
		t.Error(err)
	}
	errors := checkParseNodes(t, tree.Nodes, nodeList, testName)
	if errors != nil {
		for _, err := range errors {
			t.Error(err)
		}
	}
}
<commit_msg>parse_test.go: Add Rune case to checkFields()<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

// To enable debug output when testing, use "go test -debug"

package parse

import (
	"encoding/json"
	"github.com/demizer/go-elog"
	"reflect"
	"strings"
	"testing"
)

var lexParseTests LexTests

func TestSectionLevelsAdd(t *testing.T) {
	var p sectionLevels
	lvl := p.Add('=', true, 5)
	if lvl != 1 {
		t.Errorf("Improper level on first add, Got level: %d, expected: %d", lvl, 1)
	}
}

func TestSectionLevelsString(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	out := p.String()
	expect := "level: 1, rune: '=', overline: true, length: 5\nlevel: 2, rune: '-', " +
		"overline: true, length: 8\nlevel: 3, rune: '~', overline: false, length: 6\n"
	if out != expect {
		t.Errorf("String output mismatch!\nExpect:\n\n\t%q,\nGot:\n\n\t%q\n", expect, out)
	}
}

func TestSectionLevelsFind(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	lvl := p.Find('-')
	if lvl == -1 {
		t.Errorf("Level not found!\nExpect:\n\n\t%d\nGot:\n\n\t%t\n", 2, lvl)
	}
	if lvl != 2 {
		t.Errorf("Level not correct!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", 2, lvl)
	}
}

func TestSectionLevelsFindNoResult(t *testing.T) {
	var p sectionLevels
	lvl := p.Find('-')
	if lvl > 0 {
		t.Errorf("Should not find any levels!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", -1, lvl)
	}
}

func TestSectionLevelsLevelEmpty(t *testing.T) {
	var p sectionLevels
	lvl := p.Level()
	expect := 0
	if lvl != expect {
		t.Errorf("Empty sectionLevels should return \"%d\"!\nExpect:\t%d\nGot:\t%d\n",
			expect, expect, lvl)
	}

}

func TestSectionLevelsLevel(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	lvl := p.Level()
	if lvl != 3 {
		t.Errorf("Level() returned incorrect level!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", 3, lvl)
	}
}

func parseTest(t *testing.T, testName string) (tree *Tree, err error) {
	if lexParseTests == nil {
		lexParseTests, err = ParseTestData("../testdata/test_lex_sections.dat")
		if err != nil {
			t.Fatal(err)
		}
	}
	test := lexParseTests.SearchByName(testName)
	if test != nil {
		log.Debugf("Test Name: \t%s\n", test.name)
		log.Debugf("Description: \t%s\n", test.description)
		log.Debugf("Test Input:\n-----------\n%s\n----------\n", test.data)
		tree, err = Parse(test.name, test.data)
		if err != nil {
			t.Fatal(err)
		}
	}
	return
}

type checkNode struct {
	t          *testing.T
	testName   string
	pNodeName  string
	pFieldName string
	pFieldVal  interface{}
	pFieldType reflect.Type
	eFieldName string
	eFieldVal  interface{}
	eFieldType reflect.Type
	id         int
}

func (c *checkNode) error(args ...interface{}) {
	c.t.Error(args...)
}

func (c *checkNode) errorf(format string, args ...interface{}) {
	c.t.Errorf(format, args...)
}

func (c *checkNode) dError() {
	if c.pFieldName == "Rune" {
		c.t.Errorf("Got: %s.%s = %#v (%v) (%v) (Id: %d),\n\tExpect: %s.%s = %#v (%v)\n",
			c.pNodeName, c.pFieldName, c.pFieldVal, string(c.pFieldVal.(int32)),
			c.pFieldType, c.id, c.testName, c.eFieldName, c.eFieldVal, c.eFieldType)
		return
	}
	c.t.Errorf("Got: %s.%s = %#v (%v) (Id: %d),\n\tExpect: %s.%s = %#v (%v)\n",
		c.pNodeName, c.pFieldName, c.pFieldVal, c.pFieldType, c.id, c.testName,
		c.eFieldName, c.eFieldVal, c.eFieldType)
}

func (c *checkNode) updateState(pVal reflect.Value, eVal interface{}, field int) bool {
	// Actual parsed metadata
	c.pNodeName = pVal.Type().Name()
	c.pFieldName = pVal.Type().Field(field).Name
	c.pFieldVal = pVal.Field(field).Interface()
	c.pFieldType = pVal.Type().Field(field).Type
	c.id = pVal.FieldByName("Id").Interface().(int)

	// Expected parser metadata
	c.eFieldName = strings.ToLower(string(c.pFieldName[0])) + c.pFieldName[1:]
	c.eFieldVal = eVal.(map[string]interface{})[c.eFieldName]
	c.eFieldType = reflect.TypeOf(c.eFieldVal)

	pField := pVal.Field(field)
	if pField.Kind() == reflect.Ptr && pField.IsNil() {
		// Overline adornment nodes can be null
		if c.pFieldName == "OverLine" {
			return false
		} else {
			c.dError()
			return false
		}
	} else if c.pFieldVal == nil {
		c.dError()
		return false
	}

	return true
}

func (c *checkNode) checkFields(pNode Node, expect interface{}) {
	pVal := reflect.ValueOf(pNode).Elem()
	for i := 0; i < pVal.NumField(); i++ {

		if c.updateState(pVal, expect, i) == false {
			continue
		}

		switch c.pFieldName {
		case "Type":
			if c.pFieldVal.(NodeType).String() != c.eFieldVal {
				c.dError()
			}
		case "Id", "Level", "Length":
			if float64(c.pFieldVal.(int)) != c.eFieldVal {
				c.dError()
			}
		case "Line":
			if float64(c.pFieldVal.(Line)) != c.eFieldVal {
				c.dError()
			}
		case "StartPosition":
			if float64(c.pFieldVal.(StartPosition)) != c.eFieldVal {
				c.dError()
			}
		case "OverLine", "UnderLine":
			// c.checkAdornmentNode(c.pFieldVal.(*AdornmentNode), c.eFieldVal, c.testName)
			c.error("Overline/Underline not implemented!")
		case "NodeList":
			c.error("NodeList not implemented!")
		case "Rune":
			if string(c.pFieldVal.(rune)) != c.eFieldVal {
				c.dError()
			}
		default:
			if c.pFieldVal != c.eFieldVal {
				c.dError()
			}
		}
	}

}

// checkParseNodes is a recursive function that compares the resulting nodes (pNodes) from the
// parser with the expected output from the testdata (eNodes).
func checkParseNodes(t *testing.T, pNodes *NodeList, eNodes []interface{}, testName string) (errors []error) {
	state := &checkNode{t: t, testName: testName}
	for pNum, pNode := range *pNodes {
		state.checkFields(pNode, eNodes[pNum])
	}
	return
}

func TestParseSectionTitlePara(t *testing.T) {
	testName := "SectionTitlePara"
	tree, err := parseTest(t, testName)
	if err != nil {
		t.Error(err)
	}
	test := lexParseTests.SearchByName(testName)
	var nodeList []interface{}
	err = json.Unmarshal([]byte(test.expectTree), &nodeList)
	if err != nil {
		t.Error(err)
	}
	errors := checkParseNodes(t, tree.Nodes, nodeList, testName)
	if errors != nil {
		for _, err := range errors {
			t.Error(err)
		}
	}
}
<|endoftext|>