<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// A package of simple functions to manipulate strings.
package strings

import (
	"unicode"
	"utf8"
)

// explode splits s into an array of UTF-8 sequences, one per Unicode character (still strings) up to a maximum of n (n <= 0 means no limit).
// Invalid UTF-8 sequences become correct encodings of U+FFF8.
func explode(s string, n int) []string {
	if n <= 0 {
		n = len(s)
	}
	a := make([]string, n)
	var size, rune int
	na := 0
	for len(s) > 0 {
		if na+1 >= n {
			a[na] = s
			na++
			break
		}
		rune, size = utf8.DecodeRuneInString(s)
		s = s[size:]
		a[na] = string(rune)
		na++
	}
	return a[0:na]
}

// Count counts the number of non-overlapping instances of sep in s.
func Count(s, sep string) int {
	if sep == "" {
		return utf8.RuneCountInString(s) + 1
	}
	c := sep[0]
	n := 0
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			n++
			i += len(sep) - 1
		}
	}
	return n
}

// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s.
func Index(s, sep string) int {
	n := len(sep)
	if n == 0 {
		return 0
	}
	c := sep[0]
	if n == 1 {
		// special case worth making fast
		for i := 0; i < len(s); i++ {
			if s[i] == c {
				return i
			}
		}
		return -1
	}
	// n > 1
	for i := 0; i+n <= len(s); i++ {
		if s[i] == c && s[i:i+n] == sep {
			return i
		}
	}
	return -1
}

// LastIndex returns the index of the last instance of sep in s, or -1 if sep is not present in s.
func LastIndex(s, sep string) int {
	n := len(sep)
	if n == 0 {
		return len(s)
	}
	c := sep[0]
	if n == 1 {
		// special case worth making fast
		for i := len(s) - 1; i >= 0; i-- {
			if s[i] == c {
				return i
			}
		}
		return -1
	}
	// n > 1
	for i := len(s) - n; i >= 0; i-- {
		if s[i] == c && s[i:i+n] == sep {
			return i
		}
	}
	return -1
}

// Generic split: splits after each instance of sep,
// including sepSave bytes of sep in the subarrays.
func genSplit(s, sep string, sepSave, n int) []string {
	if sep == "" {
		return explode(s, n)
	}
	if n <= 0 {
		n = Count(s, sep) + 1
	}
	c := sep[0]
	start := 0
	a := make([]string, n)
	na := 0
	for i := 0; i+len(sep) <= len(s) && na+1 < n; i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			a[na] = s[start : i+sepSave]
			na++
			start = i + len(sep)
			i += len(sep) - 1
		}
	}
	a[na] = s[start:]
	return a[0 : na+1]
}

// Split splits the string s around each instance of sep, returning an array of substrings of s.
// If sep is empty, Split splits s after each UTF-8 sequence.
// If n > 0, Split splits s into at most n substrings; the last substring will be the unsplit remainder.
func Split(s, sep string, n int) []string { return genSplit(s, sep, 0, n) }

// SplitAfter splits the string s after each instance of sep, returning an array of substrings of s.
// If sep is empty, SplitAfter splits s after each UTF-8 sequence.
// If n > 0, SplitAfter splits s into at most n substrings; the last substring will be the unsplit remainder.
func SplitAfter(s, sep string, n int) []string {
	return genSplit(s, sep, len(sep), n)
}

// Fields splits the string s around each instance of one or more consecutive white space
// characters, returning an array of substrings of s or an empty list if s contains only white space.
func Fields(s string) []string {
	n := 0
	inField := false
	for _, rune := range s {
		wasInField := inField
		inField = !unicode.IsSpace(rune)
		if inField && !wasInField {
			n++
		}
	}

	a := make([]string, n)
	na := 0
	fieldStart := -1
	for i, rune := range s {
		if unicode.IsSpace(rune) {
			if fieldStart >= 0 {
				a[na] = s[fieldStart:i]
				na++
				fieldStart = -1
			}
		} else if fieldStart == -1 {
			fieldStart = i
		}
	}
	if fieldStart != -1 {
		a[na] = s[fieldStart:]
		na++
	}
	return a[0:na]
}

// Join concatenates the elements of a to create a single string.   The separator string
// sep is placed between elements in the resulting string.
func Join(a []string, sep string) string {
	if len(a) == 0 {
		return ""
	}
	if len(a) == 1 {
		return a[0]
	}
	n := len(sep) * (len(a) - 1)
	for i := 0; i < len(a); i++ {
		n += len(a[i])
	}

	b := make([]byte, n)
	bp := 0
	for i := 0; i < len(a); i++ {
		s := a[i]
		for j := 0; j < len(s); j++ {
			b[bp] = s[j]
			bp++
		}
		if i+1 < len(a) {
			s = sep
			for j := 0; j < len(s); j++ {
				b[bp] = s[j]
				bp++
			}
		}
	}
	return string(b)
}

// HasPrefix tests whether the string s begins with prefix.
func HasPrefix(s, prefix string) bool {
	return len(s) >= len(prefix) && s[0:len(prefix)] == prefix
}

// HasSuffix tests whether the string s ends with suffix.
func HasSuffix(s, suffix string) bool {
	return len(s) >= len(suffix) && s[len(s)-len(suffix):] == suffix
}

// Map returns a copy of the string s with all its characters modified
// according to the mapping function. If mapping returns a negative value, the character is
// dropped from the string with no replacement.
func Map(mapping func(rune int) int, s string) string {
	// In the worst case, the string can grow when mapped, making
	// things unpleasant.  But it's so rare we barge in assuming it's
	// fine.  It could also shrink but that falls out naturally.
	maxbytes := len(s) // length of b
	nbytes := 0        // number of bytes encoded in b
	b := make([]byte, maxbytes)
	for _, c := range s {
		rune := mapping(c)
		if rune >= 0 {
			wid := 1
			if rune >= utf8.RuneSelf {
				wid = utf8.RuneLen(rune)
			}
			if nbytes+wid > maxbytes {
				// Grow the buffer.
				maxbytes = maxbytes*2 + utf8.UTFMax
				nb := make([]byte, maxbytes)
				for i, c := range b[0:nbytes] {
					nb[i] = c
				}
				b = nb
			}
			nbytes += utf8.EncodeRune(rune, b[nbytes:maxbytes])
		}
	}
	return string(b[0:nbytes])
}

// Repeat returns a new string consisting of count copies of the string s.
func Repeat(s string, count int) string {
	b := make([]byte, len(s)*count)
	bp := 0
	for i := 0; i < count; i++ {
		for j := 0; j < len(s); j++ {
			b[bp] = s[j]
			bp++
		}
	}
	return string(b)
}

// ToUpper returns a copy of the string s with all Unicode letters mapped to their upper case.
func ToUpper(s string) string { return Map(unicode.ToUpper, s) }

// ToLower returns a copy of the string s with all Unicode letters mapped to their lower case.
func ToLower(s string) string { return Map(unicode.ToLower, s) }

// ToTitle returns a copy of the string s with all Unicode letters mapped to their title case.
func ToTitle(s string) string { return Map(unicode.ToTitle, s) }

// Trim returns a slice of the string s, with all leading and trailing white space
// removed, as defined by Unicode.
func TrimSpace(s string) string {
	start, end := 0, len(s)
	for start < end {
		wid := 1
		rune := int(s[start])
		if rune >= utf8.RuneSelf {
			rune, wid = utf8.DecodeRuneInString(s[start:end])
		}
		if !unicode.IsSpace(rune) {
			break
		}
		start += wid
	}
	for start < end {
		wid := 1
		rune := int(s[end-1])
		if rune >= utf8.RuneSelf {
			// Back up carefully looking for beginning of rune. Mustn't pass start.
			for wid = 2; start <= end-wid && !utf8.RuneStart(s[end-wid]); wid++ {
			}
			if start > end-wid { // invalid UTF-8 sequence; stop processing
				return s[start:end]
			}
			rune, wid = utf8.DecodeRuneInString(s[end-wid : end])
		}
		if !unicode.IsSpace(rune) {
			break
		}
		end -= wid
	}
	return s[start:end]
}
<commit_msg>strings: make Split(s, "", n) faster<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// A package of simple functions to manipulate strings.
package strings

import (
	"unicode"
	"utf8"
)

// explode splits s into an array of UTF-8 sequences, one per Unicode character (still strings) up to a maximum of n (n <= 0 means no limit).
// Invalid UTF-8 sequences become correct encodings of U+FFF8.
func explode(s string, n int) []string {
	l := utf8.RuneCountInString(s)
	if n <= 0 || n > l {
		n = l
	}
	a := make([]string, n)
	var size, rune int
	i, cur := 0, 0
	for ; i+1 < n; i++ {
		rune, size = utf8.DecodeRuneInString(s[cur:])
		a[i] = string(rune)
		cur += size
	}
	// add the rest
	a[i] = s[cur:]
	return a
}

// Count counts the number of non-overlapping instances of sep in s.
func Count(s, sep string) int {
	if sep == "" {
		return utf8.RuneCountInString(s) + 1
	}
	c := sep[0]
	l := len(sep)
	n := 0
	if l == 1 {
		// special case worth making fast
		for i := 0; i < len(s); i++ {
			if s[i] == c {
				n++
			}
		}
		return n
	}
	for i := 0; i+l <= len(s); i++ {
		if s[i] == c && s[i:i+l] == sep {
			n++
			i += l - 1
		}
	}
	return n
}

// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s.
func Index(s, sep string) int {
	n := len(sep)
	if n == 0 {
		return 0
	}
	c := sep[0]
	if n == 1 {
		// special case worth making fast
		for i := 0; i < len(s); i++ {
			if s[i] == c {
				return i
			}
		}
		return -1
	}
	// n > 1
	for i := 0; i+n <= len(s); i++ {
		if s[i] == c && s[i:i+n] == sep {
			return i
		}
	}
	return -1
}

// LastIndex returns the index of the last instance of sep in s, or -1 if sep is not present in s.
func LastIndex(s, sep string) int {
	n := len(sep)
	if n == 0 {
		return len(s)
	}
	c := sep[0]
	if n == 1 {
		// special case worth making fast
		for i := len(s) - 1; i >= 0; i-- {
			if s[i] == c {
				return i
			}
		}
		return -1
	}
	// n > 1
	for i := len(s) - n; i >= 0; i-- {
		if s[i] == c && s[i:i+n] == sep {
			return i
		}
	}
	return -1
}

// Generic split: splits after each instance of sep,
// including sepSave bytes of sep in the subarrays.
func genSplit(s, sep string, sepSave, n int) []string {
	if sep == "" {
		return explode(s, n)
	}
	if n <= 0 {
		n = Count(s, sep) + 1
	}
	c := sep[0]
	start := 0
	a := make([]string, n)
	na := 0
	for i := 0; i+len(sep) <= len(s) && na+1 < n; i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			a[na] = s[start : i+sepSave]
			na++
			start = i + len(sep)
			i += len(sep) - 1
		}
	}
	a[na] = s[start:]
	return a[0 : na+1]
}

// Split splits the string s around each instance of sep, returning an array of substrings of s.
// If sep is empty, Split splits s after each UTF-8 sequence.
// If n > 0, Split splits s into at most n substrings; the last substring will be the unsplit remainder.
func Split(s, sep string, n int) []string { return genSplit(s, sep, 0, n) }

// SplitAfter splits the string s after each instance of sep, returning an array of substrings of s.
// If sep is empty, SplitAfter splits s after each UTF-8 sequence.
// If n > 0, SplitAfter splits s into at most n substrings; the last substring will be the unsplit remainder.
func SplitAfter(s, sep string, n int) []string {
	return genSplit(s, sep, len(sep), n)
}

// Fields splits the string s around each instance of one or more consecutive white space
// characters, returning an array of substrings of s or an empty list if s contains only white space.
func Fields(s string) []string {
	n := 0
	inField := false
	for _, rune := range s {
		wasInField := inField
		inField = !unicode.IsSpace(rune)
		if inField && !wasInField {
			n++
		}
	}

	a := make([]string, n)
	na := 0
	fieldStart := -1
	for i, rune := range s {
		if unicode.IsSpace(rune) {
			if fieldStart >= 0 {
				a[na] = s[fieldStart:i]
				na++
				fieldStart = -1
			}
		} else if fieldStart == -1 {
			fieldStart = i
		}
	}
	if fieldStart != -1 {
		a[na] = s[fieldStart:]
		na++
	}
	return a[0:na]
}

// Join concatenates the elements of a to create a single string.   The separator string
// sep is placed between elements in the resulting string.
func Join(a []string, sep string) string {
	if len(a) == 0 {
		return ""
	}
	if len(a) == 1 {
		return a[0]
	}
	n := len(sep) * (len(a) - 1)
	for i := 0; i < len(a); i++ {
		n += len(a[i])
	}

	b := make([]byte, n)
	bp := 0
	for i := 0; i < len(a); i++ {
		s := a[i]
		for j := 0; j < len(s); j++ {
			b[bp] = s[j]
			bp++
		}
		if i+1 < len(a) {
			s = sep
			for j := 0; j < len(s); j++ {
				b[bp] = s[j]
				bp++
			}
		}
	}
	return string(b)
}

// HasPrefix tests whether the string s begins with prefix.
func HasPrefix(s, prefix string) bool {
	return len(s) >= len(prefix) && s[0:len(prefix)] == prefix
}

// HasSuffix tests whether the string s ends with suffix.
func HasSuffix(s, suffix string) bool {
	return len(s) >= len(suffix) && s[len(s)-len(suffix):] == suffix
}

// Map returns a copy of the string s with all its characters modified
// according to the mapping function. If mapping returns a negative value, the character is
// dropped from the string with no replacement.
func Map(mapping func(rune int) int, s string) string {
	// In the worst case, the string can grow when mapped, making
	// things unpleasant.  But it's so rare we barge in assuming it's
	// fine.  It could also shrink but that falls out naturally.
	maxbytes := len(s) // length of b
	nbytes := 0        // number of bytes encoded in b
	b := make([]byte, maxbytes)
	for _, c := range s {
		rune := mapping(c)
		if rune >= 0 {
			wid := 1
			if rune >= utf8.RuneSelf {
				wid = utf8.RuneLen(rune)
			}
			if nbytes+wid > maxbytes {
				// Grow the buffer.
				maxbytes = maxbytes*2 + utf8.UTFMax
				nb := make([]byte, maxbytes)
				for i, c := range b[0:nbytes] {
					nb[i] = c
				}
				b = nb
			}
			nbytes += utf8.EncodeRune(rune, b[nbytes:maxbytes])
		}
	}
	return string(b[0:nbytes])
}

// Repeat returns a new string consisting of count copies of the string s.
func Repeat(s string, count int) string {
	b := make([]byte, len(s)*count)
	bp := 0
	for i := 0; i < count; i++ {
		for j := 0; j < len(s); j++ {
			b[bp] = s[j]
			bp++
		}
	}
	return string(b)
}

// ToUpper returns a copy of the string s with all Unicode letters mapped to their upper case.
func ToUpper(s string) string { return Map(unicode.ToUpper, s) }

// ToLower returns a copy of the string s with all Unicode letters mapped to their lower case.
func ToLower(s string) string { return Map(unicode.ToLower, s) }

// ToTitle returns a copy of the string s with all Unicode letters mapped to their title case.
func ToTitle(s string) string { return Map(unicode.ToTitle, s) }

// Trim returns a slice of the string s, with all leading and trailing white space
// removed, as defined by Unicode.
func TrimSpace(s string) string {
	start, end := 0, len(s)
	for start < end {
		wid := 1
		rune := int(s[start])
		if rune >= utf8.RuneSelf {
			rune, wid = utf8.DecodeRuneInString(s[start:end])
		}
		if !unicode.IsSpace(rune) {
			break
		}
		start += wid
	}
	for start < end {
		wid := 1
		rune := int(s[end-1])
		if rune >= utf8.RuneSelf {
			// Back up carefully looking for beginning of rune. Mustn't pass start.
			for wid = 2; start <= end-wid && !utf8.RuneStart(s[end-wid]); wid++ {
			}
			if start > end-wid { // invalid UTF-8 sequence; stop processing
				return s[start:end]
			}
			rune, wid = utf8.DecodeRuneInString(s[end-wid : end])
		}
		if !unicode.IsSpace(rune) {
			break
		}
		end -= wid
	}
	return s[start:end]
}
<|endoftext|>


<commit_before>package handlers

import (
	"fmt"
	"github.com/APTrust/easy-store/bagit"
	"github.com/APTrust/easy-store/db/models"
	"github.com/APTrust/easy-store/util/fileutil"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	_ "github.com/mattn/go-sqlite3"
	"github.com/minio/minio-go"
	"github.com/pkg/errors"
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
)

// TODO: This should now handle JobWorkflowChanged as well.
func JobGetForm(env *Environment, w http.ResponseWriter, r *http.Request) error {
	data := make(map[string]interface{})
	job, err := ParseJobRequest(env, http.MethodGet, r)
	if err != nil {
		return WrapErr(err)
	}
	form, err := job.Form(env.DB)
	if err != nil {
		return WrapErr(err)
	}
	data["form"] = form
	data["obj"] = job
	return env.ExecTemplate(w, "job-form", data)
}

func ParseJobRequest(env *Environment, method string, r *http.Request) (*models.Job, error) {
	id, formValues, err := ParseRequest(r)
	if err != nil {
		return nil, err
	}
	return models.JobFromRequest(env.DB, method, id, formValues)
}

// func JobNewGet(env *Environment, w http.ResponseWriter, r *http.Request) error {
//	job := models.Job{}
//	postUrl := "/job/run"
//	data := make(map[string]interface{})
//	form := forms.BootstrapFormFromModel(job, forms.POST, postUrl)
//	form.Field("WorkflowID").SetSelectChoices(models.WorkflowOptions(env.DB))
//	sourceDirField := fields.HiddenField("SourceDir")
//	sourceDirField.SetId("SourceDir")
//	form.Elements(sourceDirField)
//	data["form"] = form
//	return env.ExecTemplate(w, "job", data)
// }

// func JobWorkflowChanged(env *Environment, w http.ResponseWriter, r *http.Request) error {
//	var err error
//	vars := mux.Vars(r)
//	id, _ := strconv.Atoi(vars["id"])
//	job := &models.Job{}
//	if id != 0 {
//		job, err = models.JobLoadWithRelations(env.DB, uint(id))
//		if err != nil {
//			return errors.WithStack(err)
//		}
//	} else {
//		err = r.ParseForm()
//		if err != nil {
//			return errors.WithStack(err)
//		}
//		workflowId, err := strconv.Atoi(r.Form.Get("WorkflowID"))
//		if err != nil {
//			return errors.WithStack(err)
//		}
//		job.WorkflowID = uint(workflowId)
//		workflow, err := models.WorkflowLoadWithRelations(env.DB, job.WorkflowID)
//		if err != nil {
//			return errors.WithStack(err)
//		}
//		job.Workflow = *workflow
//	}
//	data := make(map[string]interface{})
//	form, err := JobForm(job)
//	if err != nil {
//		return errors.WithStack(err)
//	}
//	form.Field("WorkflowID").SetSelectChoices(models.WorkflowOptions(env.DB))
//	form.Field("WorkflowID").SetValue(fmt.Sprintf("%d", job.WorkflowID))
//	sourceDirField := fields.HiddenField("SourceDir")
//	sourceDirField.SetId("SourceDir")
//	form.Elements(sourceDirField)
//	data["form"] = form
//	return env.ExecTemplate(w, "job", data)
// }

// Returns a Job form.
// func JobForm(job *models.Job) (*forms.Form, error) {
//	postUrl := fmt.Sprintf("/job/new")
//	if job.ID > uint(0) {
//		postUrl = fmt.Sprintf("/job/%d/edit", job.ID)
//	}
//	form := forms.BootstrapFormFromModel(*job, forms.POST, postUrl)

//	// Remove the submit button from the end of the form,
//	// add our new elements, and then replace the submit button
//	// at the end.
//	submitButton := form.Field("submitButton")
//	form.RemoveElement("submitButton")

//	// Add the tag value fields we need to display.
//	// Last param, true, means hide fields that already have
//	// default values.
//	if &job.Workflow != nil && &job.Workflow.BagItProfile != nil {
//		AddTagValueFields(job.Workflow.BagItProfile, form, true)
//		showHideButton := fields.Button("showHideTagFields", "Show/Hide Tag Fields")
//		showHideButton.SetId("showHideTagFields")
//		form.Elements(showHideButton)
//	}
//	form.Elements(submitButton)

//	// TODO: Add run button after redoing form.

//	return form, nil
// }

// This is a crude job runner for our demo. Break this out later,
// add proper error handling, etc. And fix the models too.
// We should be able to preload, instead of getting related ids
// and issuing new queries.
func JobRun(env *Environment, w http.ResponseWriter, r *http.Request) error {
	data := make(map[string]interface{})
	errMsg := ""
	err := r.ParseForm()
	if err != nil {
		return errors.WithStack(err)
	}

	// Gather the info we need to create the bag
	workflowId, _ := strconv.Atoi(r.PostFormValue("WorkflowID"))
	workflow := &models.Workflow{}
	env.DB.Find(&workflow, workflowId)

	profile := &models.BagItProfile{}
	env.DB.Preload("DefaultTagValues").Find(&profile, workflow.BagItProfileID)

	storageService := &models.StorageService{}
	env.DB.Find(&storageService, workflow.StorageServiceID)

	stagingDir := models.AppSetting{}
	env.DB.Where("name = ?", "Staging Directory").First(&stagingDir)

	sourceDir := r.PostFormValue("SourceDir")
	// HACK for demo: add virginia.edu. as bag name prefix
	bagName := "virginia.edu." + filepath.Base(sourceDir)
	bagPath := filepath.Join(stagingDir.Value, bagName)
	bagCreated := ("Creating bag " + bagName + " in " + stagingDir.Value + " using profile " +
		profile.Name + "
")
	bagItProfile, err := profile.Profile()
	if err != nil {
		errors.WithStack(err)
	}

	// Create the bag by copying the files into a staging dir,
	// adding tags and manifests.
	bagCreated += "
Files
"
	bagger, err := bagit.NewBagger(bagPath, bagItProfile)
	sourceFiles, _ := fileutil.RecursiveFileList(sourceDir)
	relDestPaths := make([]string, len(sourceFiles))
	for i, absSrcPath := range sourceFiles {
		// Use forward slash, even on Windows, for path of file inside bag
		origPathMinusRootDir := strings.Replace(absSrcPath, sourceDir+"/", "", 1)
		relDestPath := fmt.Sprintf("data/%s", origPathMinusRootDir)
		bagger.AddFile(absSrcPath, relDestPath)
		relDestPaths[i] = relDestPath
		bagCreated += ("Adding file " + absSrcPath + " at " + relDestPath + "
")
	}
	bagCreated += "
"

	// This adds the tags from the profile's default tag values.
	// In reality, several of the tags cannot come from general
	// defaults, because they're bag-specific. E.g. Title,
	// Description, Access, etc.
	bagCreated += "
Tags
"
	for _, dtv := range profile.DefaultTagValues {
		keyValuePair := &bagit.KeyValuePair{
			Key:   dtv.TagName,
			Value: dtv.TagValue,
		}
		bagger.AddTag(dtv.TagFile, keyValuePair)
		bagCreated += ("Adding tag " + dtv.TagName + " to file " + dtv.TagFile + "
")
	}
	bagCreated += "
"

	// Now that the bagger knows what to do, WriteBag() tells it to
	// go ahead and write out all the contents to the staging area.
	overwriteExistingBag := true
	checkRequiredTags := true
	bagger.WriteBag(overwriteExistingBag, checkRequiredTags)
	if len(bagger.Errors()) > 0 {
		for _, errMsg := range bagger.Errors() {
			errMsg += fmt.Sprintf("%s
", err.Error())
		}
	} else {
		bagCreated += ("Bag was written to " + bagPath + "\n\n")
	}

	manifestPath := filepath.Join(bagPath, "manifest-md5.txt")
	if fileutil.FileExists(manifestPath) {
		manifestData, err := ioutil.ReadFile(manifestPath)
		if err != nil {
			return errors.WithStack(err)
		} else {
			data["Manifest"] = string(manifestData)
		}
	}

	aptPath := filepath.Join(bagPath, "aptrust-info.txt")
	if fileutil.FileExists(aptPath) {
		aptData, err := ioutil.ReadFile(aptPath)
		if err != nil {
			return errors.WithStack(err)
		} else {
			data["APTrustInfo"] = string(aptData)
		}
	}

	bagInfoPath := filepath.Join(bagPath, "bag-info.txt")
	if fileutil.FileExists(bagInfoPath) {
		bagInfoData, err := ioutil.ReadFile(bagInfoPath)
		if err != nil {
			return errors.WithStack(err)
		} else {
			data["BagInfo"] = string(bagInfoData)
		}
	}

	// Tar the bag, if the Workflow config says to do that.
	// In the future, we may support formats other than tar.
	// Also, we will likely have to supply our own tar program
	// because Windows users won't have one.
	bagPathForValidation := bagPath
	tarFileName := fmt.Sprintf("%s.tar", bagName)
	if workflow.SerializationFormat == "tar" {
		cleanBagPath := bagPath
		if strings.HasSuffix(bagPath, string(os.PathSeparator)) {
			// Chop off final path separator, so call to filepath.Dir
			// below will return the parent dir name.
			cleanBagPath = bagPath[0 : len(bagPath)-1]
		}
		workingDir := filepath.Dir(cleanBagPath)
		tarFileAbsPath := filepath.Join(workingDir, tarFileName)
		bagCreated += ("Tarring bag to " + tarFileAbsPath + "\n")
		//cmd := exec.Command("tar", "cf", tarFileName, "--directory", bagName)
		cmd := exec.Command("tar", "cf", tarFileName, bagName)
		cmd.Dir = workingDir
		commandOutput, err := cmd.CombinedOutput()
		if err != nil {
			return errors.WithStack(err)
		}
		bagCreated += fmt.Sprintf("%s

", string(commandOutput))
		bagPathForValidation = tarFileAbsPath
	}

	// Validate the bag, just to make sure...
	bagCreated += "
Validation
"
	bagCreated += "Validating bag...
"
	bag := bagit.NewBag(bagPathForValidation)
	validator := bagit.NewValidator(bag, bagItProfile)
	validator.ReadBag()
	if len(validator.Errors()) > 0 {
		for _, e := range validator.Errors() {
			errMsg += fmt.Sprintf("%s
", e)
		}
	} else {
		bagCreated += ("Bag is valid
")
		if fileutil.LooksSafeToDelete(bagPath, 15, 3) {
			os.RemoveAll(bagPath)
			bagCreated += "Deleting working directory, kept tar file.
"
		}
	}

	data["BagName"] = bagName
	data["BagCreated"] = template.HTML(bagCreated)

	// If the config includes an S3 upload, do that now.
	// Use the minio client from https://minio.io/downloads.html#minio-client
	// Doc is at https://docs.minio.io/docs/minio-client-complete-guide
	// We're checking for S3, because this step is for demo only, and
	// s3 is the only storage service we've implemented.
	if storageService != nil && storageService.Protocol == "s3" {
		// Hack for demo: get S3 keys out of the environment.
		accessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
		secretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
		useSSL := true
		minioClient, err := minio.New(storageService.URL, accessKeyID, secretAccessKey, useSSL)
		if err != nil {
			return errors.WithStack(err)
		}
		n, err := minioClient.FPutObject(storageService.BucketOrFolder,
			tarFileName, // we're assuming the tar file was made for this demo
			bagPathForValidation,
			minio.PutObjectOptions{ContentType: "application/x-tar"})
		if err != nil {
			errMsg += ("Error uploading tar file to S3: " + err.Error() + "
")
		} else {
			msg := fmt.Sprintf("Successfully uploaded %s of size %d to receiving bucket.", tarFileName, n)
			data["UploadResult"] = template.HTML(msg + "
")
		}
	}

	if errMsg != "" {
		data["Error"] = template.HTML(errMsg)
	}

	return env.ExecTemplate(w, "job-result", data)
}
<commit_msg>Removed old code<commit_after>package handlers

import (
	"fmt"
	"github.com/APTrust/easy-store/bagit"
	"github.com/APTrust/easy-store/db/models"
	"github.com/APTrust/easy-store/util/fileutil"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	_ "github.com/mattn/go-sqlite3"
	"github.com/minio/minio-go"
	"github.com/pkg/errors"
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
)

func JobGetForm(env *Environment, w http.ResponseWriter, r *http.Request) error {
	data := make(map[string]interface{})
	job, err := ParseJobRequest(env, http.MethodGet, r)
	if err != nil {
		return WrapErr(err)
	}
	form, err := job.Form(env.DB)
	if err != nil {
		return WrapErr(err)
	}
	data["form"] = form
	data["obj"] = job
	return env.ExecTemplate(w, "job-form", data)
}

func ParseJobRequest(env *Environment, method string, r *http.Request) (*models.Job, error) {
	id, formValues, err := ParseRequest(r)
	if err != nil {
		return nil, err
	}
	return models.JobFromRequest(env.DB, method, id, formValues)
}

// This is a crude job runner for our demo. Break this out later,
// add proper error handling, etc. And fix the models too.
// We should be able to preload, instead of getting related ids
// and issuing new queries.
func JobRun(env *Environment, w http.ResponseWriter, r *http.Request) error {
	data := make(map[string]interface{})
	errMsg := ""
	err := r.ParseForm()
	if err != nil {
		return errors.WithStack(err)
	}

	// Gather the info we need to create the bag
	workflowId, _ := strconv.Atoi(r.PostFormValue("WorkflowID"))
	workflow := &models.Workflow{}
	env.DB.Find(&workflow, workflowId)

	profile := &models.BagItProfile{}
	env.DB.Preload("DefaultTagValues").Find(&profile, workflow.BagItProfileID)

	storageService := &models.StorageService{}
	env.DB.Find(&storageService, workflow.StorageServiceID)

	stagingDir := models.AppSetting{}
	env.DB.Where("name = ?", "Staging Directory").First(&stagingDir)

	sourceDir := r.PostFormValue("SourceDir")
	// HACK for demo: add virginia.edu. as bag name prefix
	bagName := "virginia.edu." + filepath.Base(sourceDir)
	bagPath := filepath.Join(stagingDir.Value, bagName)
	bagCreated := ("Creating bag " + bagName + " in " + stagingDir.Value + " using profile " +
		profile.Name + "
")
	bagItProfile, err := profile.Profile()
	if err != nil {
		errors.WithStack(err)
	}

	// Create the bag by copying the files into a staging dir,
	// adding tags and manifests.
	bagCreated += "
Files
"
	bagger, err := bagit.NewBagger(bagPath, bagItProfile)
	sourceFiles, _ := fileutil.RecursiveFileList(sourceDir)
	relDestPaths := make([]string, len(sourceFiles))
	for i, absSrcPath := range sourceFiles {
		// Use forward slash, even on Windows, for path of file inside bag
		origPathMinusRootDir := strings.Replace(absSrcPath, sourceDir+"/", "", 1)
		relDestPath := fmt.Sprintf("data/%s", origPathMinusRootDir)
		bagger.AddFile(absSrcPath, relDestPath)
		relDestPaths[i] = relDestPath
		bagCreated += ("Adding file " + absSrcPath + " at " + relDestPath + "
")
	}
	bagCreated += "
"

	// This adds the tags from the profile's default tag values.
	// In reality, several of the tags cannot come from general
	// defaults, because they're bag-specific. E.g. Title,
	// Description, Access, etc.
	bagCreated += "
Tags
"
	for _, dtv := range profile.DefaultTagValues {
		keyValuePair := &bagit.KeyValuePair{
			Key:   dtv.TagName,
			Value: dtv.TagValue,
		}
		bagger.AddTag(dtv.TagFile, keyValuePair)
		bagCreated += ("Adding tag " + dtv.TagName + " to file " + dtv.TagFile + "
")
	}
	bagCreated += "
"

	// Now that the bagger knows what to do, WriteBag() tells it to
	// go ahead and write out all the contents to the staging area.
	overwriteExistingBag := true
	checkRequiredTags := true
	bagger.WriteBag(overwriteExistingBag, checkRequiredTags)
	if len(bagger.Errors()) > 0 {
		for _, errMsg := range bagger.Errors() {
			errMsg += fmt.Sprintf("%s
", err.Error())
		}
	} else {
		bagCreated += ("Bag was written to " + bagPath + "\n\n")
	}

	manifestPath := filepath.Join(bagPath, "manifest-md5.txt")
	if fileutil.FileExists(manifestPath) {
		manifestData, err := ioutil.ReadFile(manifestPath)
		if err != nil {
			return errors.WithStack(err)
		} else {
			data["Manifest"] = string(manifestData)
		}
	}

	aptPath := filepath.Join(bagPath, "aptrust-info.txt")
	if fileutil.FileExists(aptPath) {
		aptData, err := ioutil.ReadFile(aptPath)
		if err != nil {
			return errors.WithStack(err)
		} else {
			data["APTrustInfo"] = string(aptData)
		}
	}

	bagInfoPath := filepath.Join(bagPath, "bag-info.txt")
	if fileutil.FileExists(bagInfoPath) {
		bagInfoData, err := ioutil.ReadFile(bagInfoPath)
		if err != nil {
			return errors.WithStack(err)
		} else {
			data["BagInfo"] = string(bagInfoData)
		}
	}

	// Tar the bag, if the Workflow config says to do that.
	// In the future, we may support formats other than tar.
	// Also, we will likely have to supply our own tar program
	// because Windows users won't have one.
	bagPathForValidation := bagPath
	tarFileName := fmt.Sprintf("%s.tar", bagName)
	if workflow.SerializationFormat == "tar" {
		cleanBagPath := bagPath
		if strings.HasSuffix(bagPath, string(os.PathSeparator)) {
			// Chop off final path separator, so call to filepath.Dir
			// below will return the parent dir name.
			cleanBagPath = bagPath[0 : len(bagPath)-1]
		}
		workingDir := filepath.Dir(cleanBagPath)
		tarFileAbsPath := filepath.Join(workingDir, tarFileName)
		bagCreated += ("Tarring bag to " + tarFileAbsPath + "\n")
		//cmd := exec.Command("tar", "cf", tarFileName, "--directory", bagName)
		cmd := exec.Command("tar", "cf", tarFileName, bagName)
		cmd.Dir = workingDir
		commandOutput, err := cmd.CombinedOutput()
		if err != nil {
			return errors.WithStack(err)
		}
		bagCreated += fmt.Sprintf("%s

", string(commandOutput))
		bagPathForValidation = tarFileAbsPath
	}

	// Validate the bag, just to make sure...
	bagCreated += "
Validation
"
	bagCreated += "Validating bag...
"
	bag := bagit.NewBag(bagPathForValidation)
	validator := bagit.NewValidator(bag, bagItProfile)
	validator.ReadBag()
	if len(validator.Errors()) > 0 {
		for _, e := range validator.Errors() {
			errMsg += fmt.Sprintf("%s
", e)
		}
	} else {
		bagCreated += ("Bag is valid
")
		if fileutil.LooksSafeToDelete(bagPath, 15, 3) {
			os.RemoveAll(bagPath)
			bagCreated += "Deleting working directory, kept tar file.
"
		}
	}

	data["BagName"] = bagName
	data["BagCreated"] = template.HTML(bagCreated)

	// If the config includes an S3 upload, do that now.
	// Use the minio client from https://minio.io/downloads.html#minio-client
	// Doc is at https://docs.minio.io/docs/minio-client-complete-guide
	// We're checking for S3, because this step is for demo only, and
	// s3 is the only storage service we've implemented.
	if storageService != nil && storageService.Protocol == "s3" {
		// Hack for demo: get S3 keys out of the environment.
		accessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
		secretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
		useSSL := true
		minioClient, err := minio.New(storageService.URL, accessKeyID, secretAccessKey, useSSL)
		if err != nil {
			return errors.WithStack(err)
		}
		n, err := minioClient.FPutObject(storageService.BucketOrFolder,
			tarFileName, // we're assuming the tar file was made for this demo
			bagPathForValidation,
			minio.PutObjectOptions{ContentType: "application/x-tar"})
		if err != nil {
			errMsg += ("Error uploading tar file to S3: " + err.Error() + "
")
		} else {
			msg := fmt.Sprintf("Successfully uploaded %s of size %d to receiving bucket.", tarFileName, n)
			data["UploadResult"] = template.HTML(msg + "
")
		}
	}

	if errMsg != "" {
		data["Error"] = template.HTML(errMsg)
	}

	return env.ExecTemplate(w, "job-result", data)
}
<|endoftext|>


<commit_before>package server

import (
	"github.com/zubairhamed/betwixt/server/pages"
	. "github.com/zubairhamed/go-commons/network"
)

func SetupHttpRoutes(server *DefaultServer) {
	http := server.httpServer

	http.NewRoute("/", METHOD_GET, handleHttpHome(server))
	http.NewRoute("/reg/{client}", METHOD_GET, handleHttpViewClient(server))
}

func handleHttpViewClient(server *DefaultServer) RouteHandler {
	return func(r Request) Response {
		page := &pages.ClientDetailPage{}

		type clientdetails struct {
		}

		model := clientdetails{}

		return &HttpResponse{
			TemplateModel: model,
			Payload:       NewBytesPayload(page.GetContent()),
		}
	}
}

func handleHttpHome(server *DefaultServer) RouteHandler {
	return func(r Request) Response {

		page := &pages.HomePage{}

		type client struct {
			Endpoint         string
			RegistrationID   string
			RegistrationDate string
			LastUpdate       string
		}

		model := []client{}

		for _, v := range server.clients {
			c := client{
				Endpoint:         v.GetName(),
				RegistrationID:   v.GetId(),
				RegistrationDate: v.GetRegistrationDate().String(),
				LastUpdate:       v.LastUpdate().String(),
			}
			model = append(model, c)
		}

		return &HttpResponse{
			TemplateModel: model,
			Payload:       NewBytesPayload(page.GetContent()),
		}
	}
}
<commit_msg>Reformat date output string in client registration and last updated dates<commit_after>package server

import (
	"github.com/zubairhamed/betwixt/server/pages"
	. "github.com/zubairhamed/go-commons/network"
)

func SetupHttpRoutes(server *DefaultServer) {
	http := server.httpServer

	http.NewRoute("/", METHOD_GET, handleHttpHome(server))
	http.NewRoute("/reg/{client}", METHOD_GET, handleHttpViewClient(server))
}

func handleHttpViewClient(server *DefaultServer) RouteHandler {
	return func(r Request) Response {
		page := &pages.ClientDetailPage{}

		type clientdetails struct {
		}

		model := clientdetails{}

		return &HttpResponse{
			TemplateModel: model,
			Payload:       NewBytesPayload(page.GetContent()),
		}
	}
}

func handleHttpHome(server *DefaultServer) RouteHandler {
	return func(r Request) Response {

		page := &pages.HomePage{}

		type client struct {
			Endpoint         string
			RegistrationID   string
			RegistrationDate string
			LastUpdate       string
		}

		model := []client{}

		for _, v := range server.clients {
			c := client{
				Endpoint:         v.GetName(),
				RegistrationID:   v.GetId(),
				RegistrationDate: v.GetRegistrationDate().Format("Jan 2, 2006, 3:04pm (SGT)"),
				LastUpdate:       v.LastUpdate().Format("Jan 2, 2006, 3:04pm (SGT)"),
			}

			model = append(model, c)
		}

		return &HttpResponse{
			TemplateModel: model,
			Payload:       NewBytesPayload(page.GetContent()),
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>konnectivity-client: close readCh for connections that never receive CLOSE_RSP<commit_after><|endoftext|>


<commit_before>package lars

import (
	"net/http"
	"net/http/httptest"
	"testing"

	. "gopkg.in/go-playground/assert.v1"
)

// NOTES:
// - Run "go test" to run tests
// - Run "gocov test | gocov report" to report on test converage by file
// - Run "gocov test | gocov annotate -" to report on all code and functions, those ,marked with "MISS" were never called
//
// or
//
// -- may be a good idea to change to output path to somewherelike /tmp
// go test -coverprofile cover.out && go tool cover -html=cover.out -o cover.html
//

func TestResponse(t *testing.T) {

	l := New()
	w := httptest.NewRecorder()
	r := NewResponse(w, l)

	// SetWriter
	r.SetWriter(w)

	// Assert Write
	Equal(t, w, r.Writer())

	// Assert Header
	NotEqual(t, nil, r.Header())

	// WriteHeader
	r.WriteHeader(http.StatusOK)
	Equal(t, http.StatusOK, r.status)

	Equal(t, true, r.committed)

	// Already committed
	r.WriteHeader(http.StatusTeapot)
	NotEqual(t, http.StatusTeapot, r.Status())

	// Status
	r.status = http.StatusOK
	Equal(t, http.StatusOK, r.Status())

	// Write
	info := "Information"
	_, err := r.Write([]byte(info))
	Equal(t, nil, err)

	// Flush
	r.Flush()

	// Size
	IsEqual(len(info), r.Size())

	//committed
	Equal(t, true, r.Committed())

	panicStr := "interface conversion: *httptest.ResponseRecorder is not http.Hijacker: missing method Hijack"
	fnPanic := func() {
		r.Hijack()
	}
	PanicMatches(t, fnPanic, panicStr)

	panicStr = "interface conversion: *httptest.ResponseRecorder is not http.CloseNotifier: missing method CloseNotify"
	fnPanic = func() {
		r.CloseNotify()
	}
	PanicMatches(t, fnPanic, panicStr)

	// reset
	r.reset(httptest.NewRecorder())
}
<commit_msg>Add Missing WriteString test in Response<commit_after>package lars

import (
	"net/http"
	"net/http/httptest"
	"testing"

	. "gopkg.in/go-playground/assert.v1"
)

// NOTES:
// - Run "go test" to run tests
// - Run "gocov test | gocov report" to report on test converage by file
// - Run "gocov test | gocov annotate -" to report on all code and functions, those ,marked with "MISS" were never called
//
// or
//
// -- may be a good idea to change to output path to somewherelike /tmp
// go test -coverprofile cover.out && go tool cover -html=cover.out -o cover.html
//

func TestResponse(t *testing.T) {

	l := New()
	w := httptest.NewRecorder()
	r := NewResponse(w, l)

	// SetWriter
	r.SetWriter(w)

	// Assert Write
	Equal(t, w, r.Writer())

	// Assert Header
	NotEqual(t, nil, r.Header())

	// WriteHeader
	r.WriteHeader(http.StatusOK)
	Equal(t, http.StatusOK, r.status)

	Equal(t, true, r.committed)

	// Already committed
	r.WriteHeader(http.StatusTeapot)
	NotEqual(t, http.StatusTeapot, r.Status())

	// Status
	r.status = http.StatusOK
	Equal(t, http.StatusOK, r.Status())

	// Write
	info := "Information"
	_, err := r.Write([]byte(info))
	Equal(t, nil, err)

	// Flush
	r.Flush()

	// Size
	IsEqual(len(info), r.Size())

	// WriteString
	s := "lars"
	n, err := r.WriteString(s)
	Equal(t, err, nil)
	Equal(t, n, 4)

	//committed
	Equal(t, true, r.Committed())

	panicStr := "interface conversion: *httptest.ResponseRecorder is not http.Hijacker: missing method Hijack"
	fnPanic := func() {
		r.Hijack()
	}
	PanicMatches(t, fnPanic, panicStr)

	panicStr = "interface conversion: *httptest.ResponseRecorder is not http.CloseNotifier: missing method CloseNotify"
	fnPanic = func() {
		r.CloseNotify()
	}
	PanicMatches(t, fnPanic, panicStr)

	// reset
	r.reset(httptest.NewRecorder())
}
<|endoftext|>


<commit_before>package gremgo

import (
	"reflect"
	"testing"
)

var dummySuccessfulResponse = []byte(`{"result":{"data":[{"id": 2,"label": "person","type": "vertex","properties": [
  {"id": 2, "value": "vadas", "label": "name"},
  {"id": 3, "value": 27, "label": "age"}]}
  ], "meta":{}},
 "requestId":"1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
 "status":{"code":200,"attributes":{},"message":""}}`)

var dummyPartialResponse1 = []byte(`{"result":{"data":[{"id": 2,"label": "person","type": "vertex","properties": [
  {"id": 2, "value": "vadas", "label": "name"},
  {"id": 3, "value": 27, "label": "age"}]},
  ], "meta":{}},
 "requestId":"1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
 "status":{"code":206,"attributes":{},"message":""}}`)

var dummyPartialResponse2 = []byte(`{"result":{"data":[{"id": 4,"label": "person","type": "vertex","properties": [
  {"id": 5, "value": "quant", "label": "name"},
  {"id": 6, "value": 54, "label": "age"}]},
  ], "meta":{}},
 "requestId":"1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
 "status":{"code":200,"attributes":{},"message":""}}`)

var dummySuccessfulResponseMarshalled = response{
	requestid: "1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
	code:      200,
	data:      "testData",
}

var dummyPartialResponse1Marshalled = response{
	requestid: "1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
	code:      206,
	data:      "testPartialData1",
}

var dummyPartialResponse2Marshalled = response{
	requestid: "1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
	code:      200,
	data:      "testPartialData2",
}

// func TestResponseHandling(t *testing.T) {
// 	c := newClient()
//
// 	c.handleResponse(dummySuccessfulResponse)
//
// 	var expected []interface{}
// 	expected = append(expected, dummySuccessfulResponseMarshalled.data)
//
// 	fmt.Println(expected)
// 	fmt.Println(c.retrieveResponse(dummySuccessfulResponseMarshalled.requestid))
//
// 	if reflect.DeepEqual(c.retrieveResponse(dummySuccessfulResponseMarshalled.requestid), expected) != true {
// 		t.Fail()
// 	}
// }

func TestResponseMarshalling(t *testing.T) {
	resp, err := marshalResponse(dummySuccessfulResponse)
	if err != nil {
		t.Error(err)
	}
	if dummySuccessfulResponseMarshalled.requestid != resp.requestid || dummySuccessfulResponseMarshalled.code != resp.code {
		t.Error("Expected requestid and code does not match actual.")
	} else if reflect.TypeOf(resp.data).String() != "[]interface {}" {
		t.Error("Expected data type does not match actual.")
	}
}

func TestResponseSortingSingleResponse(t *testing.T) {
	c := newClient()

	c.sortResponse(dummySuccessfulResponseMarshalled)

	var expected []interface{}
	expected = append(expected, dummySuccessfulResponseMarshalled.data)

	if reflect.DeepEqual(c.results[dummySuccessfulResponseMarshalled.requestid], expected) != true {
		t.Fail()
	}
}

func TestResponseSortingMultipleResponse(t *testing.T) {
	c := newClient()

	c.sortResponse(dummyPartialResponse1Marshalled)
	c.sortResponse(dummyPartialResponse2Marshalled)

	var expected []interface{}
	expected = append(expected, dummyPartialResponse1Marshalled.data)
	expected = append(expected, dummyPartialResponse2Marshalled.data)

	if reflect.DeepEqual(c.results[dummyPartialResponse1Marshalled.requestid], expected) != true {
		t.Fail()
	}
}

func TestResponseRetrieval(t *testing.T) {
	c := newClient()

	c.sortResponse(dummyPartialResponse1Marshalled)
	c.sortResponse(dummyPartialResponse2Marshalled)

	resp := c.retrieveResponse(dummyPartialResponse1Marshalled.requestid)

	var expected []interface{}
	expected = append(expected, dummyPartialResponse1Marshalled.data)
	expected = append(expected, dummyPartialResponse2Marshalled.data)

	if reflect.DeepEqual(resp, expected) != true {
		t.Fail()
	}
}

func TestResponseDeletion(t *testing.T) {
	c := newClient()

	c.sortResponse(dummyPartialResponse1Marshalled)
	c.sortResponse(dummyPartialResponse2Marshalled)

	c.deleteResponse(dummyPartialResponse1Marshalled.requestid)

	if len(c.results[dummyPartialResponse1Marshalled.requestid]) != 0 {
		t.Fail()
	}
}
<commit_msg>Added handleResponse test<commit_after>package gremgo

import (
	"reflect"
	"testing"
)

var dummySuccessfulResponse = []byte(`{"result":{"data":[{"id": 2,"label": "person","type": "vertex","properties": [
  {"id": 2, "value": "vadas", "label": "name"},
  {"id": 3, "value": 27, "label": "age"}]}
  ], "meta":{}},
 "requestId":"1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
 "status":{"code":200,"attributes":{},"message":""}}`)

var dummyPartialResponse1 = []byte(`{"result":{"data":[{"id": 2,"label": "person","type": "vertex","properties": [
  {"id": 2, "value": "vadas", "label": "name"},
  {"id": 3, "value": 27, "label": "age"}]},
  ], "meta":{}},
 "requestId":"1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
 "status":{"code":206,"attributes":{},"message":""}}`)

var dummyPartialResponse2 = []byte(`{"result":{"data":[{"id": 4,"label": "person","type": "vertex","properties": [
  {"id": 5, "value": "quant", "label": "name"},
  {"id": 6, "value": 54, "label": "age"}]},
  ], "meta":{}},
 "requestId":"1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
 "status":{"code":200,"attributes":{},"message":""}}`)

var dummySuccessfulResponseMarshalled = response{
	requestid: "1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
	code:      200,
	data:      "testData",
}

var dummyPartialResponse1Marshalled = response{
	requestid: "1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
	code:      206,
	data:      "testPartialData1",
}

var dummyPartialResponse2Marshalled = response{
	requestid: "1d6d02bd-8e56-421d-9438-3bd6d0079ff1",
	code:      200,
	data:      "testPartialData2",
}

func TestResponseHandling(t *testing.T) {
	c := newClient()

	c.handleResponse(dummySuccessfulResponse)

	var expected []interface{}
	expected = append(expected, dummySuccessfulResponseMarshalled.data)

	if reflect.TypeOf(expected).String() != reflect.TypeOf(c.retrieveResponse(dummySuccessfulResponseMarshalled.requestid)).String() {
		t.Error("Expected data type does not match actual.")
	}
}

func TestResponseMarshalling(t *testing.T) {
	resp, err := marshalResponse(dummySuccessfulResponse)
	if err != nil {
		t.Error(err)
	}
	if dummySuccessfulResponseMarshalled.requestid != resp.requestid || dummySuccessfulResponseMarshalled.code != resp.code {
		t.Error("Expected requestid and code does not match actual.")
	} else if reflect.TypeOf(resp.data).String() != "[]interface {}" {
		t.Error("Expected data type does not match actual.")
	}
}

func TestResponseSortingSingleResponse(t *testing.T) {
	c := newClient()

	c.sortResponse(dummySuccessfulResponseMarshalled)

	var expected []interface{}
	expected = append(expected, dummySuccessfulResponseMarshalled.data)

	if reflect.DeepEqual(c.results[dummySuccessfulResponseMarshalled.requestid], expected) != true {
		t.Fail()
	}
}

func TestResponseSortingMultipleResponse(t *testing.T) {
	c := newClient()

	c.sortResponse(dummyPartialResponse1Marshalled)
	c.sortResponse(dummyPartialResponse2Marshalled)

	var expected []interface{}
	expected = append(expected, dummyPartialResponse1Marshalled.data)
	expected = append(expected, dummyPartialResponse2Marshalled.data)

	if reflect.DeepEqual(c.results[dummyPartialResponse1Marshalled.requestid], expected) != true {
		t.Fail()
	}
}

func TestResponseRetrieval(t *testing.T) {
	c := newClient()

	c.sortResponse(dummyPartialResponse1Marshalled)
	c.sortResponse(dummyPartialResponse2Marshalled)

	resp := c.retrieveResponse(dummyPartialResponse1Marshalled.requestid)

	var expected []interface{}
	expected = append(expected, dummyPartialResponse1Marshalled.data)
	expected = append(expected, dummyPartialResponse2Marshalled.data)

	if reflect.DeepEqual(resp, expected) != true {
		t.Fail()
	}
}

func TestResponseDeletion(t *testing.T) {
	c := newClient()

	c.sortResponse(dummyPartialResponse1Marshalled)
	c.sortResponse(dummyPartialResponse2Marshalled)

	c.deleteResponse(dummyPartialResponse1Marshalled.requestid)

	if len(c.results[dummyPartialResponse1Marshalled.requestid]) != 0 {
		t.Fail()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/pborman/getopt"
)

const currentVersion = "0.5"

var lastCheck = time.Now()
var newLastCheck = time.Now()

// Config for the application
type Config struct {
	webhookURL string
	path       string
	watch      int
	username   string
}

func main() {

	config := parseOptions()

	checkPath(config.path)
	checkUpdates()

	log.Print("Waiting for images to appear in ", config.path)
	// wander the path, forever
	for {
		err := filepath.Walk(config.path,
			func(path string, f os.FileInfo, err error) error { return checkFile(path, f, err, config) })
		if err != nil {
			log.Fatal("could not watch path", err)
		}
		lastCheck = newLastCheck
		time.Sleep(time.Duration(config.watch) * time.Second)
	}
}

func checkPath(path string) {
	src, err := os.Stat(path)
	if err != nil {
		log.Fatal(err)
	}
	if !src.IsDir() {
		log.Fatal(path, " is not a directory")
		os.Exit(1)
	}
}

func checkUpdates() {

	type GithubRelease struct {
		HTMLURL string
		TagName string
		Name    string
		Body    string
	}

	client := &http.Client{Timeout: time.Second * 5}
	resp, err := client.Get("https://api.github.com/repos/tardisx/discord-auto-upload/releases/latest")
	if err != nil {
		log.Fatal("could not check for updates:", err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("could not check read update response")
	}

	var latest GithubRelease
	err = json.Unmarshal(body, &latest)

	if err != nil {
		log.Fatal("could not parse JSON: ", err)
	}

	if currentVersion < latest.TagName {
		fmt.Printf("You are currently on version %s, but version %s is available\n", currentVersion, latest.TagName)
		fmt.Println("----------- Release Info -----------")
		fmt.Println(latest.Body)
		fmt.Println("------------------------------------")
	}

}

func parseOptions() Config {

	var newConfig Config
	// Declare the flags to be used
	webhookFlag := getopt.StringLong("webhook", 'w', "", "discord webhook URL")
	pathFlag := getopt.StringLong("directory", 'd', "", "directory to scan, optional, defaults to current directory")
	watchFlag := getopt.Int16Long("watch", 's', 10, "time between scans")
	usernameFlag := getopt.StringLong("username", 'u', "", "username for the bot upload")
	helpFlag := getopt.BoolLong("help", 'h', "help")
	versionFlag := getopt.BoolLong("version", 'v', "show version")
	getopt.SetParameters("")

	getopt.Parse()

	if *helpFlag {
		getopt.PrintUsage(os.Stderr)
		os.Exit(0)
	}

	if *versionFlag {
		fmt.Println("dau - https://github.com/tardisx/discord-auto-upload")
		fmt.Printf("Version: %s\n", currentVersion)
		os.Exit(0)
	}

	if !getopt.IsSet("directory") {
		*pathFlag = "./"
		log.Println("Defaulting to current directory")
	}

	if !getopt.IsSet("webhook") {
		log.Fatal("ERROR: You must specify a --webhook URL")
	}

	newConfig.path = *pathFlag
	newConfig.webhookURL = *webhookFlag
	newConfig.watch = int(*watchFlag)
	newConfig.username = *usernameFlag

	return newConfig
}

func checkFile(path string, f os.FileInfo, err error, config Config) error {

	if f.ModTime().After(lastCheck) && f.Mode().IsRegular() {

		if fileEligible(config, path) {
			// process file
			processFile(config, path)
		}

		if newLastCheck.Before(f.ModTime()) {
			newLastCheck = f.ModTime()
		}
	}

	return nil
}

func fileEligible(config Config, file string) bool {
	extension := strings.ToLower(filepath.Ext(file))
	if extension == ".png" || extension == ".jpg" || extension == ".gif" {
		return true
	}
	return false
}

func processFile(config Config, file string) {
	log.Print("Uploading ", file)

	extraParams := map[string]string{}

	if config.username != "" {
		extraParams["username"] = config.username
	}

	type DiscordAPIResponseAttachment struct {
		URL      string
		ProxyURL string
		Size     int
		Width    int
		Height   int
		Filename string
	}

	type DiscordAPIResponse struct {
		Attachments []DiscordAPIResponseAttachment
		ID          int64 `json:",string"`
	}

	request, err := newfileUploadRequest(config.webhookURL, extraParams, "file", file)
	if err != nil {
		log.Fatal(err)
	}
	start := time.Now()
	client := &http.Client{Timeout: time.Second * 30}
	resp, err := client.Do(request)
	if err != nil {

		log.Fatal("Error performing request:", err)

	} else {

		if resp.StatusCode != 200 {
			log.Print("Bad response from server:", resp.StatusCode)
			return
		}

		resBody, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Fatal("could not deal with body", err)
		}
		resp.Body.Close()

		var res DiscordAPIResponse
		err = json.Unmarshal(resBody, &res)

		if err != nil {
			log.Print("could not parse JSON: ", err)
			fmt.Println("Response was:", string(resBody[:]))
			return
		}
		if len(res.Attachments) < 1 {
			log.Print("bad response - no attachments?")
			return
		}
		var a = res.Attachments[0]
		elapsed := time.Since(start)
		rate := float64(a.Size) / elapsed.Seconds() / 1024.0

		log.Printf("Uploaded to %s %dx%d", a.URL, a.Width, a.Height)
		log.Printf("id: %d, %d bytes transferred in %.2f seconds (%.2f KiB/s)", res.ID, a.Size, elapsed.Seconds(), rate)
	}

}

func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, filepath.Base(path))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		log.Fatal("Could not copy: ", err)
	}

	for key, val := range params {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", uri, body)
	req.Header.Set("Content-Type", writer.FormDataContentType())
	return req, err
}
<commit_msg>Hacky image watermarking (not yet complete)<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"image"

	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"

	"github.com/fogleman/gg"
	"github.com/pborman/getopt"
	"golang.org/x/image/font/inconsolata"
)

const currentVersion = "0.5"

var lastCheck = time.Now()
var newLastCheck = time.Now()

// Config for the application
type Config struct {
	webhookURL string
	path       string
	watch      int
	username   string
}

func main() {

	config := parseOptions()

	checkPath(config.path)
	checkUpdates()

	log.Print("Waiting for images to appear in ", config.path)
	// wander the path, forever
	for {
		err := filepath.Walk(config.path,
			func(path string, f os.FileInfo, err error) error { return checkFile(path, f, err, config) })
		if err != nil {
			log.Fatal("could not watch path", err)
		}
		lastCheck = newLastCheck
		time.Sleep(time.Duration(config.watch) * time.Second)
	}
}

func checkPath(path string) {
	src, err := os.Stat(path)
	if err != nil {
		log.Fatal(err)
	}
	if !src.IsDir() {
		log.Fatal(path, " is not a directory")
		os.Exit(1)
	}
}

func checkUpdates() {

	type GithubRelease struct {
		HTMLURL string
		TagName string
		Name    string
		Body    string
	}

	client := &http.Client{Timeout: time.Second * 5}
	resp, err := client.Get("https://api.github.com/repos/tardisx/discord-auto-upload/releases/latest")
	if err != nil {
		log.Fatal("could not check for updates:", err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("could not check read update response")
	}

	var latest GithubRelease
	err = json.Unmarshal(body, &latest)

	if err != nil {
		log.Fatal("could not parse JSON: ", err)
	}

	if currentVersion < latest.TagName {
		fmt.Printf("You are currently on version %s, but version %s is available\n", currentVersion, latest.TagName)
		fmt.Println("----------- Release Info -----------")
		fmt.Println(latest.Body)
		fmt.Println("------------------------------------")
	}

}

func parseOptions() Config {

	var newConfig Config
	// Declare the flags to be used
	webhookFlag := getopt.StringLong("webhook", 'w', "", "discord webhook URL")
	pathFlag := getopt.StringLong("directory", 'd', "", "directory to scan, optional, defaults to current directory")
	watchFlag := getopt.Int16Long("watch", 's', 10, "time between scans")
	usernameFlag := getopt.StringLong("username", 'u', "", "username for the bot upload")
	helpFlag := getopt.BoolLong("help", 'h', "help")
	versionFlag := getopt.BoolLong("version", 'v', "show version")
	getopt.SetParameters("")

	getopt.Parse()

	if *helpFlag {
		getopt.PrintUsage(os.Stderr)
		os.Exit(0)
	}

	if *versionFlag {
		fmt.Println("dau - https://github.com/tardisx/discord-auto-upload")
		fmt.Printf("Version: %s\n", currentVersion)
		os.Exit(0)
	}

	if !getopt.IsSet("directory") {
		*pathFlag = "./"
		log.Println("Defaulting to current directory")
	}

	if !getopt.IsSet("webhook") {
		log.Fatal("ERROR: You must specify a --webhook URL")
	}

	newConfig.path = *pathFlag
	newConfig.webhookURL = *webhookFlag
	newConfig.watch = int(*watchFlag)
	newConfig.username = *usernameFlag

	return newConfig
}

func checkFile(path string, f os.FileInfo, err error, config Config) error {

	if f.ModTime().After(lastCheck) && f.Mode().IsRegular() {

		if fileEligible(config, path) {
			// process file
			processFile(config, path)
		}

		if newLastCheck.Before(f.ModTime()) {
			newLastCheck = f.ModTime()
		}
	}

	return nil
}

func fileEligible(config Config, file string) bool {
	extension := strings.ToLower(filepath.Ext(file))
	if extension == ".png" || extension == ".jpg" || extension == ".gif" {
		return true
	}
	return false
}

func processFile(config Config, file string) {

	log.Print("Munging ", file)

	mungeFile(file)

	log.Print("Uploading ", file)

	extraParams := map[string]string{}

	if config.username != "" {
		extraParams["username"] = config.username
	}

	type DiscordAPIResponseAttachment struct {
		URL      string
		ProxyURL string
		Size     int
		Width    int
		Height   int
		Filename string
	}

	type DiscordAPIResponse struct {
		Attachments []DiscordAPIResponseAttachment
		ID          int64 `json:",string"`
	}

	request, err := newfileUploadRequest(config.webhookURL, extraParams, "file", file)
	if err != nil {
		log.Fatal(err)
	}
	start := time.Now()
	client := &http.Client{Timeout: time.Second * 30}
	resp, err := client.Do(request)
	if err != nil {

		log.Fatal("Error performing request:", err)

	} else {

		if resp.StatusCode != 200 {
			log.Print("Bad response from server:", resp.StatusCode)
			return
		}

		resBody, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Fatal("could not deal with body", err)
		}
		resp.Body.Close()

		var res DiscordAPIResponse
		err = json.Unmarshal(resBody, &res)

		if err != nil {
			log.Print("could not parse JSON: ", err)
			fmt.Println("Response was:", string(resBody[:]))
			return
		}
		if len(res.Attachments) < 1 {
			log.Print("bad response - no attachments?")
			return
		}
		var a = res.Attachments[0]
		elapsed := time.Since(start)
		rate := float64(a.Size) / elapsed.Seconds() / 1024.0

		log.Printf("Uploaded to %s %dx%d", a.URL, a.Width, a.Height)
		log.Printf("id: %d, %d bytes transferred in %.2f seconds (%.2f KiB/s)", res.ID, a.Size, elapsed.Seconds(), rate)
	}

}

func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, filepath.Base(path))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		log.Fatal("Could not copy: ", err)
	}

	for key, val := range params {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", uri, body)
	req.Header.Set("Content-Type", writer.FormDataContentType())
	return req, err
}

func mungeFile(path string) {

	reader, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	im, _, err := image.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}
	bounds := im.Bounds()
	// var S float64 = float64(bounds.Max.X)

	dc := gg.NewContext(bounds.Max.X, bounds.Max.Y)
	dc.Clear()
	dc.SetRGB(0, 0, 0)

	dc.SetFontFace(inconsolata.Regular8x16)

	dc.DrawImage(im, 0, 0)

	dc.DrawRoundedRectangle(0, float64(bounds.Max.Y-18.0), 320, float64(bounds.Max.Y), 0)
	dc.SetRGB(0, 0, 0)
	dc.Fill()

	dc.SetRGB(1, 1, 1)

	dc.DrawString("github.com/tardisx/discord-auto-upload", 5.0, float64(bounds.Max.Y)-5.0)

	dc.SavePNG("../out.png")
}
<|endoftext|>


<commit_before>/*
Package go-restful, a lean package for creating REST-style WebServices without magic.

Example WebService:

	package landscapeservice

	import (
	    "github.com/emicklei/go-restful"
	)

	func New() *restful.WebService {
		ws := new(restful.WebService)
	   	ws.Path("/applications").
			Consumes(restful.MIME_XML, restful.MIME_JSON).
			Produces(restful.MIME_XML, restful.MIME_JSON)

		ws.Route(ws.GET("/{id}").To(getApplication).
			// for documentation
			Doc("Get the Application node by its id").
			Param(ws.PathParameter("id" , "the identifier for an application node")).
			Param(ws.QueryParameter("environment" , "the scope in which the application node lives")).
			Writes(Application{})) // to the response

		ws.Route(ws.POST("/").To(saveApplication).
			// for documentation
			Doc("Create or update the Application node").
			Reads(Application{})) // from the request
		return ws
	}
	func getApplication(request *Request, response *Response) {
			id := request.PathParameter("id")
			env := request.QueryParameter("environment")
			...
	}
	func saveApplication(request *Request, response *Response) {
		// response.AddHeader("X-Something","other")
		// response.WriteEntity(anApp) , uses Accept header to detect XML/JSON
		// response.WriterError(http.StatusInternalServerError,err)
	}

Example main:

	func main() {
		restful.Add(landscapeservice.New())
		log.Fatal(http.ListenAndServe(":8080", nil))
	}

WebServices

A WebService has a collection of Route objects that dispatch incoming Http Requests to a function calls.

	type RouteFunction func(*Request, *Response)

A Route is defined by a HTTP method, an URL path and (optionally) the MIME types it consumes (Content-Type) and produces (Accept).
This package has the logic to find the best matching Route and if found, call its Function.
The (*Request, *Response) arguments provide functions for reading information from the request and writing information back to the response.

Filters

A filter dynamically intercepts requests and responses to transform or use the information contained in the requests or responses.
You can use filters to perform generic logging, measurement, authentication, redirect, set response headers etc.
In the restful package there are three hooks into the request,response flow where filters can be added.
Each filter must define a FilterFunction:

	func (req *restful.Request, resp *restful.Response, chain *restful.FilterChain)

Use the following statement to pass the request,response pair to the next filter or RouteFunction

	chain.ProcessFilter(req, resp)

Global Filters

These are processed before any registered WebService.

	// install a global filter (processed before any webservice)
	restful.Filter(globalLogging)

WebService Filters

These are processed before any Route of a WebService.

	// install a webservice filter (processed before any route)
	ws.Filter(webserviceLogging).Filter(measureTime)

Route Filters

These are processed before calling the function associated with the Route.

	// install 2 chained route filters (processed before calling findUser)
	ws.Route(ws.GET("/{user-id}").Filter(routeLogging).Filter(NewCountFilter().routeCounter).To(findUser))

See the example https://github.com/emicklei/go-restful/blob/master/examples/restful-filters.go with full implementations.

Serving files

Use the Go standard http.ServeFile function to serve file system assets.

	ws.Route(ws.GET("/static/{resource}").To(staticFromPathParam))
	...
	// http://localhost:8080/static/test.xml
	// http://localhost:8080/static/
	func staticFromPathParam(req *restful.Request, resp *restful.Response) {
		http.ServeFile(
			resp.ResponseWriter,
			req.Request,
			path.Join(rootdir, req.PathParameter("resource")))
	}

See the example https://github.com/emicklei/go-restful/blob/master/examples/restful-serve-static.go with full implementations.

Error Handling

Unexpected things happen. If a request cannot be processed because of a failure, your service needs to tell the response what happened and why.
For this reason HTTP status codes exist and it is important to use the correct code in every exceptional situation.

400: Bad Request

If path or query parameters are not valid (content or type) then use http.StatusBadRequest.

	id, err := strconv.Atoi(req.PathParameter("id"))
	if err != nil {
		resp.WriteHeader(http.StatusBadRequest)
		return
	}

404: Not Found

Despite a valid URI, the resource requested may not be available

	resp.WriteHeader(http.StatusNotFound)

500: Internal Server Error

If the application logic could not process the request (or write the response) then use http.StatusInternalServerError.

	question, err := application.SharedLogic.GetQuestionById(id)
	if err != nil {
		log.Printf("GetQuestionById failed:", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

ServiceError

In addition to setting the correct (error) Http status code, you can choose to write a ServiceError message on the response:

	resp.WriteEntity(restful.NewError(http.StatusBadRequest, "Non-integer {id} path parameter"))

	resp.WriteEntity(restful.NewError(http.StatusInternalServerError, err.Error()))

Resources

[project]: https://github.com/emicklei/go-restful

[example]: https://github.com/emicklei/go-restful/blob/master/examples/restful-user-service.go

[design]:  http://ernestmicklei.com/2012/11/11/go-restful-api-design/

[showcase]: https://github.com/emicklei/landskape

(c) 2012,2013, http://ernestmicklei.com. MIT License
*/
package restful
<commit_msg>Fix application signatures.<commit_after>/*
Package go-restful, a lean package for creating REST-style WebServices without magic.

Example WebService:

	package landscapeservice

	import (
	    "github.com/emicklei/go-restful"
	)

	func New() *restful.WebService {
		ws := new(restful.WebService)
	   	ws.Path("/applications").
			Consumes(restful.MIME_XML, restful.MIME_JSON).
			Produces(restful.MIME_XML, restful.MIME_JSON)

		ws.Route(ws.GET("/{id}").To(getApplication).
			// for documentation
			Doc("Get the Application node by its id").
			Param(ws.PathParameter("id" , "the identifier for an application node")).
			Param(ws.QueryParameter("environment" , "the scope in which the application node lives")).
			Writes(Application{})) // to the response

		ws.Route(ws.POST("/").To(saveApplication).
			// for documentation
			Doc("Create or update the Application node").
			Reads(Application{})) // from the request
		return ws
	}
	func getApplication(request *restful.Request, response *restful.Response) {
			id := request.PathParameter("id")
			env := request.QueryParameter("environment")
			...
	}
	func saveApplication(request *restful.Request, response *restful.Response) {
		// response.AddHeader("X-Something","other")
		// response.WriteEntity(anApp) , uses Accept header to detect XML/JSON
		// response.WriterError(http.StatusInternalServerError,err)
	}

Example main:

	func main() {
		restful.Add(landscapeservice.New())
		log.Fatal(http.ListenAndServe(":8080", nil))
	}

WebServices

A WebService has a collection of Route objects that dispatch incoming Http Requests to a function calls.

	type RouteFunction func(*restful.Request, *restful.Response)

A Route is defined by a HTTP method, an URL path and (optionally) the MIME types it consumes (Content-Type) and produces (Accept).
This package has the logic to find the best matching Route and if found, call its Function.
The (*Request, *Response) arguments provide functions for reading information from the request and writing information back to the response.

Filters

A filter dynamically intercepts requests and responses to transform or use the information contained in the requests or responses.
You can use filters to perform generic logging, measurement, authentication, redirect, set response headers etc.
In the restful package there are three hooks into the request,response flow where filters can be added.
Each filter must define a FilterFunction:

	func (req *restful.Request, resp *restful.Response, chain *restful.FilterChain)

Use the following statement to pass the request,response pair to the next filter or RouteFunction

	chain.ProcessFilter(req, resp)

Global Filters

These are processed before any registered WebService.

	// install a global filter (processed before any webservice)
	restful.Filter(globalLogging)

WebService Filters

These are processed before any Route of a WebService.

	// install a webservice filter (processed before any route)
	ws.Filter(webserviceLogging).Filter(measureTime)

Route Filters

These are processed before calling the function associated with the Route.

	// install 2 chained route filters (processed before calling findUser)
	ws.Route(ws.GET("/{user-id}").Filter(routeLogging).Filter(NewCountFilter().routeCounter).To(findUser))

See the example https://github.com/emicklei/go-restful/blob/master/examples/restful-filters.go with full implementations.

Serving files

Use the Go standard http.ServeFile function to serve file system assets.

	ws.Route(ws.GET("/static/{resource}").To(staticFromPathParam))
	...
	// http://localhost:8080/static/test.xml
	// http://localhost:8080/static/
	func staticFromPathParam(req *restful.Request, resp *restful.Response) {
		http.ServeFile(
			resp.ResponseWriter,
			req.Request,
			path.Join(rootdir, req.PathParameter("resource")))
	}

See the example https://github.com/emicklei/go-restful/blob/master/examples/restful-serve-static.go with full implementations.

Error Handling

Unexpected things happen. If a request cannot be processed because of a failure, your service needs to tell the response what happened and why.
For this reason HTTP status codes exist and it is important to use the correct code in every exceptional situation.

400: Bad Request

If path or query parameters are not valid (content or type) then use http.StatusBadRequest.

	id, err := strconv.Atoi(req.PathParameter("id"))
	if err != nil {
		resp.WriteHeader(http.StatusBadRequest)
		return
	}

404: Not Found

Despite a valid URI, the resource requested may not be available

	resp.WriteHeader(http.StatusNotFound)

500: Internal Server Error

If the application logic could not process the request (or write the response) then use http.StatusInternalServerError.

	question, err := application.SharedLogic.GetQuestionById(id)
	if err != nil {
		log.Printf("GetQuestionById failed:", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

ServiceError

In addition to setting the correct (error) Http status code, you can choose to write a ServiceError message on the response:

	resp.WriteEntity(restful.NewError(http.StatusBadRequest, "Non-integer {id} path parameter"))

	resp.WriteEntity(restful.NewError(http.StatusInternalServerError, err.Error()))

Resources

[project]: https://github.com/emicklei/go-restful

[example]: https://github.com/emicklei/go-restful/blob/master/examples/restful-user-service.go

[design]:  http://ernestmicklei.com/2012/11/11/go-restful-api-design/

[showcase]: https://github.com/emicklei/landskape

(c) 2012,2013, http://ernestmicklei.com. MIT License
*/
package restful
<|endoftext|>


<commit_before>/*
go-restful, a petit package for creating REST-style WebServices without magic. (Work-in-Progress)

Design discussed on http://ernestmicklei.com/2012/11/11/go-restful-api-design/

Example WebService:

	package landscapeservice

	import (
	    "github.com/emicklei/go-restful"
	)

	type LandscapeService struct {
		restful.WebService
	}
	func New() *LandscapeService {
		ws := new(LandscapeService)
		ws.Path("/applications").Accept("application/xml").ContentType("application/xml")
		ws.Route(ws.Method("GET").Path("/{id}").To(GetApplication))
		return ws
	}
	func GetApplication(request *Request, writer http.ResponseWriter) {
		id := request.PathParameter("id")
		...
	}

Example main:

	func main() {
		restful.Add(landscapeservice.New())
		log.Fatal(http.ListenAndServe(":8080", nil))
	}

*/
package restful
<commit_msg>updated doc<commit_after>/*
go-restful, a petit package for creating REST-style WebServices without magic. (Work-in-Progress)

Design discussed on http://ernestmicklei.com/2012/11/11/go-restful-api-design/

Example WebService:

	package landscapeservice

	import (
	    "github.com/emicklei/go-restful"
	)

	type LandscapeService struct {
		restful.WebService
	}
	func New() *LandscapeService {
		ws := new(LandscapeService)
		ws.Path("/applications").Accept("application/xml").ContentType("application/xml")

		ws.Route(ws.GET("/{id}").To(GetApplication))
		ws.Route(ws.POST("/").To(SaveApplication))
		return ws
	}
	func GetApplication(request *Request, response *Response) {
		id := request.PathParameter("id")
		...
	}
	func SaveApplication(request *Request, response *Response) {
		...
	}

Example main:

	func main() {
		restful.Add(landscapeservice.New())
		log.Fatal(http.ListenAndServe(":8080", nil))
	}

*/
package restful
<|endoftext|>


<commit_before>// Package M3U8 is parser & generator library for Apple HLS.

// Copyleft 2013-2015 Alexander I.Grafov aka Axel <grafov@gmail.com>

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// ॐ तारे तुत्तारे तुरे स्व

/*

This is a most complete opensource library for parsing and generating of M3U8 playlists used in HTTP Live Streaming (Apple HLS) for internet video translations.

M3U8 is simple text format and parsing library for it must be simple too. It did not offer ways to play HLS or handle playlists over HTTP. Library features are:

  * Support HLS specs up to version 5 of the protocol.
  * Parsing and generation of master-playlists and media-playlists.
  * Autodetect input streams as master or media playlists.
  * Offer structures for keeping playlists metadata.
  * Encryption keys support for usage with DRM systems like Verimatrix (http://verimatrix.com) etc.
  * Support for non standard Google Widevine (http://www.widevine.com) tags.

Library coded acordingly with IETF draft http://tools.ietf.org/html/draft-pantos-http-live-streaming

Examples of usage may be found in *_test.go files of a package. Also see below some simple examples.

Create simple media playlist with sliding window of 3 segments and maximum of 50 segments.

        p, e := NewMediaPlaylist(3, 50)
        if e != nil {
          panic(fmt.Sprintf("Create media playlist failed: %s", e))
        }
        for i := 0; i < 5; i++ {
          e = p.Add(fmt.Sprintf("test%d.ts", i), 5.0)
          if e != nil {
            panic(fmt.Sprintf("Add segment #%d to a media playlist failed: %s", i, e))
          }
        }
        fmt.Println(p.Encode(true).String())

We add 5 testX.ts segments to playlist then encode it to M3U8 format and convert to string.

Next example shows parsing of master playlist:

        f, err := os.Open("sample-playlists/master.m3u8")
        if err != nil {
          fmt.Println(err)
        }
        p := NewMasterPlaylist()
        err = p.Decode(bufio.NewReader(f), false)
        if err != nil {
          fmt.Println(err)
        }

        fmt.Printf("Playlist object: %+v\n", p)

We are open playlist from the file and parse it as master playlist.

*/
package m3u8
<commit_msg>fix typo<commit_after>// Package M3U8 is parser & generator library for Apple HLS.

// Copyleft 2013-2015 Alexander I.Grafov aka Axel <grafov@gmail.com>

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// ॐ तारे तुत्तारे तुरे स्व

/*

This is a most complete opensource library for parsing and generating of M3U8 playlists used in HTTP Live Streaming (Apple HLS) for internet video translations.

M3U8 is simple text format and parsing library for it must be simple too. It did not offer ways to play HLS or handle playlists over HTTP. Library features are:

  * Support HLS specs up to version 5 of the protocol.
  * Parsing and generation of master-playlists and media-playlists.
  * Autodetect input streams as master or media playlists.
  * Offer structures for keeping playlists metadata.
  * Encryption keys support for usage with DRM systems like Verimatrix (http://verimatrix.com) etc.
  * Support for non standard Google Widevine (http://www.widevine.com) tags.

Library coded accordingly with IETF draft http://tools.ietf.org/html/draft-pantos-http-live-streaming

Examples of usage may be found in *_test.go files of a package. Also see below some simple examples.

Create simple media playlist with sliding window of 3 segments and maximum of 50 segments.

        p, e := NewMediaPlaylist(3, 50)
        if e != nil {
          panic(fmt.Sprintf("Create media playlist failed: %s", e))
        }
        for i := 0; i < 5; i++ {
          e = p.Add(fmt.Sprintf("test%d.ts", i), 5.0)
          if e != nil {
            panic(fmt.Sprintf("Add segment #%d to a media playlist failed: %s", i, e))
          }
        }
        fmt.Println(p.Encode(true).String())

We add 5 testX.ts segments to playlist then encode it to M3U8 format and convert to string.

Next example shows parsing of master playlist:

        f, err := os.Open("sample-playlists/master.m3u8")
        if err != nil {
          fmt.Println(err)
        }
        p := NewMasterPlaylist()
        err = p.Decode(bufio.NewReader(f), false)
        if err != nil {
          fmt.Println(err)
        }

        fmt.Printf("Playlist object: %+v\n", p)

We are open playlist from the file and parse it as master playlist.

*/
package m3u8
<|endoftext|>


<commit_before>// Package c4 contains Go implementations for core C4 concepts.
//
// Creating IDs
//
// To create a C4 ID, use the IDEncoder type as a hash writer.
//
//     e := c4.NewIDEncoder()
//     io.Copy(e, src)
//     id := e.ID()
//
// Parsing IDs
//
// To parse an ID from a string, use c4.ParseID(src)
package c4
<commit_msg>tweaked docs<commit_after>// Package c4 contains Go implementations for core C4 concepts.
//
// Creating IDs
//
// To create a C4 ID, use the IDEncoder type as a hash writer.
//
//     e := c4.NewIDEncoder()
//     io.Copy(e, src)
//     id := e.ID()
//
// Parsing IDs
//
// To parse an ID from a string, use the ParseID function.
//
//     c4.ParseID(src)
package c4
<|endoftext|>


<commit_before>// Goque provides embedded, disk-based implementations of stack, queue, and priority queue data structures.
//
// Motivation for creating this project was the need for a persistent priority queue that remained performant while growing well beyond the available memory of a given machine. While there are many packages for Go offering queues, they all seem to be memory based and/or standalone solutions that are not embeddable within an application.
//
// Instead of using an in-memory heap structure to store data, everything is stored using the Go port of LevelDB (https://github.com/syndtr/goleveldb). This results in very little memory being used no matter the size of the database, while read and write performance remains near constant.
//
// See README.md or visit https://github.com/beeker1121/goque for more info.
package goque
<commit_msg>Fixed golint warning<commit_after>// Package goque provides embedded, disk-based implementations of stack, queue, and priority queue data structures.
//
// Motivation for creating this project was the need for a persistent priority queue that remained performant while growing well beyond the available memory of a given machine. While there are many packages for Go offering queues, they all seem to be memory based and/or standalone solutions that are not embeddable within an application.
//
// Instead of using an in-memory heap structure to store data, everything is stored using the Go port of LevelDB (https://github.com/syndtr/goleveldb). This results in very little memory being used no matter the size of the database, while read and write performance remains near constant.
//
// See README.md or visit https://github.com/beeker1121/goque for more info.
package goque
<|endoftext|>


<commit_before>/*
Package workerpool queues work to a limited number of goroutines.

The purpose of the worker pool is to limit the concurrency of the task
performed by the workers.  This is useful when performing a task that requires
sufficient resources (CPU, memory, etc.), that running too many tasks at the
same time would exhaust resources.

Non-blocking task submission

A task is a function submitted to the worker pool for execution.  Submitting
tasks to this worker pool will not block, regardless of the number of tasks.
Tasks read from the input task queue are immediately dispatched to an available
worker.  If no worker is immediately available, or there are already tasks
waiting for an available worker, then the task is put on a waiting queue to
wait for an available worker.  This clears the incoming task from the task
queue immediately, whether or not a worker is currently available, and will not
block the submission of tasks.

The intent of the worker pool is to limit the concurrency of task execution,
not limit the number of tasks queued to be executed.  Therefore, this unbounded
input of tasks is acceptable as the tasks cannot be discarded.  If the number
of inbound tasks is too many to even queue for pending processing, then the
solution is outside the scope of workerpool, and should be solved by
distributing load over multiple systems, and/or storing input for pending
processing in intermediate storage such as a database, file system, distributed
message queue, etc.

Dispatcher

This worker pool uses a single dispatcher goroutine to read tasks from the
input task queue and dispatch them to a worker goroutine.  This allows for a
small input channel, and lets the dispatcher queue as many tasks as are
submitted when there are no available workers.  Additionally, the dispatcher
can adjust the number of workers as appropriate for the work load, without
having to utilize locked counters and checks incurred on task submission.

When no tasks have been submitted for a period of time, a worker is removed by
the dispatcher.  This is done until there are no more workers to remove.  The
minimum number of workers is always zero, because the time to start new workers
is insignificant.

Usage note

It is advisable to use different worker pools for tasks that are bound by
different resources, or that have different resource use patterns.  For
example, tasks that use X Mb of memory may need different concurrency limits
than tasks that use Y Mb of memory.

Waiting queue vs goroutines

When there are no available workers to handle incoming tasks, the tasks are put
on a waiting queue.  In previous versions of workerpool, these tasks were
passed to goroutines.  Using a queue is faster and has less memory overhead
than creating a separate goroutine for each waiting task, allowing a much
higher number of waiting tasks.

Credits

This implementation builds on ideas from the following:

http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang
http://nesv.github.io/golang/2014/02/25/worker-queues-in-go.html

*/
package workerpool
<commit_msg>update doc<commit_after>/*
Package workerpool queues work to a limited number of goroutines.

The purpose of the worker pool is to limit the concurrency of tasks
executed by the workers.  This is useful when performing tasks that require
sufficient resources (CPU, memory, etc.), and running too many tasks at the
same time would exhaust resources.

Non-blocking task submission

A task is a function submitted to the worker pool for execution.  Submitting
tasks to this worker pool will not block, regardless of the number of tasks.
Tasks read from the input task queue are immediately dispatched to an available
worker.  If no worker is immediately available, or there are already tasks
waiting for an available worker, then the task is put on a waiting queue to
wait for an available worker.  This clears the incoming task from the input
task queue immediately, whether or not a worker is currently available, and
will not block the submission of tasks.

The intent of the worker pool is to limit the concurrency of task execution,
not limit the number of tasks queued to be executed.  Therefore, this unbounded
input of tasks is acceptable as the tasks cannot be discarded.  If the number
of inbound tasks is too many to even queue for pending processing, then the
solution is outside the scope of workerpool, and should be solved by
distributing load over multiple systems, and/or storing input for pending
processing in intermediate storage such as a database, file system, distributed
message queue, etc.

Dispatcher

This worker pool uses a single dispatcher goroutine to read tasks from the
input task queue and dispatch them to a worker goroutine.  This allows for a
small input channel, and lets the dispatcher queue as many tasks as are
submitted when there are no available workers.  Additionally, the dispatcher
can adjust the number of workers as appropriate for the work load, without
having to utilize locked counters and checks incurred on task submission.

When no tasks have been submitted for a period of time, a worker is removed by
the dispatcher.  This is done until there are no more workers to remove.  The
minimum number of workers is always zero, because the time to start new workers
is insignificant.

Usage note

It is advisable to use different worker pools for tasks that are bound by
different resources, or that have different resource use patterns.  For
example, tasks that use X Mb of memory may need different concurrency limits
than tasks that use Y Mb of memory.

Waiting queue vs goroutines

When there are no available workers to handle incoming tasks, the tasks are put
on a waiting queue.  In previous versions of workerpool, these tasks were
passed to goroutines.  Using a queue is faster and has less memory overhead
than creating a separate goroutine for each waiting task, allowing a much
higher number of waiting tasks.

Credits

This implementation builds on ideas from the following:

http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang
http://nesv.github.io/golang/2014/02/25/worker-queues-in-go.html

*/
package workerpool
<|endoftext|>


<commit_before>/*
Package profiler contains tools to help profile a running Go service.

Enabling Memory Profiling
-------------------------

To enable memory profiling, modify your main method like this:

	import (
		"net/http"
		"github.com/wblakecaldwell/profiler"
	)
	func main() {
		// add handlers to help us track memory usage - they don't track memory until they're told to
		profiler.AddMemoryProfilingHandlers()

		// listen on port 6060 (pick a port)
		http.ListenAndServe(":6060", nil)
	}

Using Memory Profiling
----------------------

Enabling Memory Profiling exposes the following endpoints:

- http://localhost:6060/profiler/stop :    Stop recording memory statistics
- http://localhost:6060/profiler/start :   Start recording memory statistics
- http://localhost:6060/profiler/info.html :   Main page you should visit
- http://localhost:6060/profiler/info :   JSON data that feeds profiler/info.html

Working With the Template Files
-------------------------------

We bundle the template files in the Go binary with the 'go-bindata' tool. Everything in
github.com/wblakecaldwell/profiler/profiler-web is bundled up into github.com/wblakecaldwell/profiler/profiler-web.go
with the command, assuming your repository is in $GOPATH/src.

Production Code Generation (Check this in):

	go get github.com/jteeuwen/go-bindata/...
	go install github.com/jteeuwen/go-bindata/go-bindata

	go-bindata -prefix "$GOPATH/src/github.com/wblakecaldwell/profiler/profiler-web/" -pkg "profiler" -nocompress -o "$GOPATH/src/github.com/wblakecaldwell/profiler/profiler-web.go" "$GOPATH/src/github.com/wblakecaldwell/profiler/profiler-web"

If you'd like to make changes to the templates, then use 'go-bindata' in debug mode. Instead of compiling
the contents of the template files into profiler-web.go, it generates code to read the content of the template
files as they exist at that moment. This way, you can start your service, view the page, make changes, then
refresh the browser to see them:

Development Code Generation:

	go-bindata -debug -prefix "$GOPATH/src/github.com/wblakecaldwell/profiler/profiler-web/" -pkg "profiler" -nocompress -o "$GOPATH/src/github.com/wblakecaldwell/profiler/profiler-web.go" "$GOPATH/src/github.com/wblakecaldwell/profiler/profiler-web"

When you've wrapped up development, make sure to rebuild profiler-web.go to contain the contents of the file with the first non-debug command.

*/
package profiler
<commit_msg>Removed the README.md content from doc.go<commit_after>/*
Package profiler contains tools to help profile a running Go service.
*/
package profiler
<|endoftext|>