<commit_before>// Package gcfg reads "gitconfig-like" text-based configuration files with
// "name=value" pairs grouped into sections (gcfg files).
// Support for writing gcfg files may be added later.
//
// See ReadInto and the examples to get an idea of how to use it.
//
// This package is still a work in progress, and both the supported syntax and
// the API is subject to change.
//
// The syntax is based on that used by git config:
// http://git-scm.com/docs/git-config#_syntax .
// Note that the gcfg syntax may diverge from that of git config in the future
// to a limited degree.
// Currently planned differences (apart from TODOs listed below) are:
//  - gcfg files must use UTF-8 encoding and must not contain the 0 byte
//  - include and "path" type is not supported at the package level
//  - `[sec.sub]` format is not allowed (deprecated in gitconfig)
//  - `[sec ""]` is not allowed
//    - `[sec]` is the equivalent (section name "sec" and empty subsection name)
//  - within a single file, definitions must be consecutive for each:
//    - section: '[secA]' -> '[secB]' -> '[secA]' is an error
//    - subsection: '[sec "A"]' -> '[sec "B"]' -> '[sec "A"]' is an error
//    - multivalued variable: 'multi=a' -> 'other=x' -> 'multi=b' is an error
//
// The package may be usable for handling some of the various "INI file" formats
// used by some programs and libraries, but achieving or maintaining
// compatibility with any of those is not a primary concern.
//
// TODO:
//  - docs
//    - format spec
//  - parsing
//    - define internal representation structure
//    - support multi-value variables
//    - non-regexp based parser
//    - support partially quoted strings
//    - support escaping in strings
//    - support multiple inputs (readers, strings, files)
//    - support declaring encoding (?)
//    - support pointer fields
//  - scanEnum
//    - should use longest match (?)
//    - support matching on unique prefix (?)
//  - writing gcfg files
//  - error handling
//    - include error context
//    - more helpful error messages
//    - error types / codes?
//    - limit input size?
//
package gcfg

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"reflect"
	"regexp"
	"strings"
)

var (
	reCmnt    = regexp.MustCompile(`^([^;#"]*)[;#].*$`)
	reCmntQ   = regexp.MustCompile(`^([^;#"]*"[^"]*"[^;#"]*)[;#].*$`)
	reBlank   = regexp.MustCompile(`^\s*$`)
	reSect    = regexp.MustCompile(`^\s*\[\s*([^"\s]*)\s*\]\s*$`)
	reSectSub = regexp.MustCompile(`^\s*\[\s*([^"\s]*)\s*"([^"]+)"\s*\]\s*$`)
	reVar     = regexp.MustCompile(`^\s*([^"=\s]+)\s*=\s*([^"\s]*)\s*$`)
	reVarQ    = regexp.MustCompile(`^\s*([^"=\s]+)\s*=\s*"([^"\n\\]*)"\s*$`)
	reVarDflt = regexp.MustCompile(`^\s*\b(.*)\b\s*$`)
)

const (
	// Default value string in case a value for a variable isn't provided.
	defaultValue = "true"
)

type gbool bool

var gboolValues = map[string]interface{}{
	"true": true, "yes": true, "on": true, "1": true,
	"false": false, "no": false, "off": false, "0": false}

func (b *gbool) Scan(state fmt.ScanState, verb rune) error {
	v, err := scanEnum(state, gboolValues, true)
	if err != nil {
		return err
	}
	bb, _ := v.(bool) // cannot be non-bool
	*b = gbool(bb)
	return nil
}

func fieldFold(v reflect.Value, name string) reflect.Value {
	n := strings.Replace(name, "-", "_", -1)
	return v.FieldByNameFunc(func(fieldName string) bool {
		return strings.EqualFold(n, fieldName)
	})
}

func set(cfg interface{}, sect, sub, name, value string) error {
	vDest := reflect.ValueOf(cfg).Elem()
	vSect := fieldFold(vDest, sect)
	if vSect.Kind() == reflect.Map {
		if vSect.IsNil() {
			vSect.Set(reflect.MakeMap(vSect.Type()))
		}
		k := reflect.ValueOf(sub)
		pv := vSect.MapIndex(k)
		if !pv.IsValid() {
			vType := vSect.Type().Elem().Elem()
			pv = reflect.New(vType)
			vSect.SetMapIndex(k, pv)
		}
		vSect = pv.Elem()
	} else if sub != "" {
		return fmt.Errorf("expected map; section %q subsection %q", sect, sub)
	}
	vName := fieldFold(vSect, name)
	vAddr := vName.Addr().Interface()
	switch v := vAddr.(type) {
	case *string:
		*v = value
		return nil
	case *bool:
		vAddr = (*gbool)(v)
	}
	// attempt to read an extra rune to make sure the value is consumed
	var r rune
	n, err := fmt.Sscanf(value, "%v%c", vAddr, &r)
	switch {
	case n < 1 || n == 1 && err != io.EOF:
		return fmt.Errorf("failed to parse %q as %#v: parse error %v", value,
			vName.Type(), err)
	case n > 1:
		return fmt.Errorf("failed to parse %q as %#v: extra characters", value,
			vName.Type())
	case n == 1 && err == io.EOF:
		return nil
	}
	panic("never reached")
}

// ReadInto reads gcfg formatted data from reader and sets the values into the
// corresponding fields in config.
//
// Config must be a pointer to a struct.
// Each section corresponds to a struct field in config, and each variable in a
// section corresponds to a data field in the section struct.
// The name of the field must match the name of the section or variable,
// ignoring case.
// Hyphens in variable names correspond to underscores in section or field
// names.
//
// For sections with subsections, the corresponding field in config must be a
// map, rather than a struct, with string keys and pointer-to-struct values.
// Values for subsection variables are stored in the map with the subsection
// name used as the map key.
// (Note that unlike section and variable names, subsection names are case
// sensitive.)
// When using a map, and there is a section with the same section name but
// without a subsection name, its values are stored with the empty string used
// as the key.
//
// The section structs in the config struct may contain arbitrary types.
// For string fields, the (unquoted and unescaped) value string is assigned to
// the field.
// For bool fields, the field is set to true if the value is "true", "yes", "on"
// or "1", and set to false if the value is "false", "no", "off" or "0",
// ignoring case.
// For all other types, fmt.Sscanf is used to parse the value and set it to the
// field.
// This means that built-in Go types are parseable using the standard format,
// and any user-defined type is parseable if it implements the fmt.Scanner
// interface.
//
// See ReadStringInto for examples.
//
func ReadInto(config interface{}, reader io.Reader) error {
	r := bufio.NewReader(reader)
	sect := (*string)(nil)
	sectsub := ""
	for line := 1; true; line++ {
		l, pre, err := r.ReadLine()
		if err != nil && err != io.EOF {
			return err
		} else if pre {
			return errors.New("line too long")
		}
		// exclude comments
		if c := reCmnt.FindSubmatch(l); c != nil {
			l = c[1]
		} else if c := reCmntQ.FindSubmatch(l); c != nil {
			l = c[1]
		}
		if !reBlank.Match(l) {
			// "switch" based on line contents
			if sec := reSect.FindSubmatch(l); sec != nil {
				strsec := string(sec[1])
				sect, sectsub = &strsec, ""
			} else if sec := reSectSub.FindSubmatch(l); sec != nil {
				strsec := string(sec[1])
				strsub := string(sec[2])
				if strsub == "" {
					return errors.New("empty subsection not allowed")
				}
				sect, sectsub = &strsec, strsub
			} else if v, vq, vd := reVar.FindSubmatch(l),
				reVarQ.FindSubmatch(l), reVarDflt.FindSubmatch(l); //
			v != nil || vq != nil || vd != nil {
				if sect == nil {
					return errors.New("no section")
				}
				var name, value string
				if v != nil {
					name, value = string(v[1]), string(v[2])
				} else if vq != nil {
					name, value = string(vq[1]), string(vq[2])
				} else { // vd != nil
					name, value = string(vd[1]), defaultValue
				}
				err := set(config, *sect, sectsub, name, value)
				if err != nil {
					return err
				}
			} else {
				return fmt.Errorf("invalid line %q", string(l))
			}
		}
		if err == io.EOF {
			break
		}
	}
	return nil
}

// ReadStringInto reads gcfg formatted data from str and sets the values into
// the corresponding fields in config.
// See ReadInto(config, reader) for a detailed description of how values are
// parsed and set into config.
func ReadStringInto(config interface{}, str string) error {
	r := strings.NewReader(str)
	return ReadInto(config, r)
}

// ReadFileInto reads gcfg formatted data from the file filename and sets the
// values into the corresponding fields in config.
// See ReadInto(config, reader) for a detailed description of how values are
// parsed and set into config.
func ReadFileInto(config interface{}, filename string) error {
	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	return ReadInto(config, f)
}
<commit_msg>improve docs<commit_after>// Package gcfg reads "gitconfig-like" text-based configuration files with
// "name=value" pairs grouped into sections (gcfg files).
// Support for writing gcfg files may be added later.
//
// See ReadInto and the examples to get an idea of how to use it.
//
// This package is still a work in progress, and both the supported syntax and
// the API is subject to change.
//
// The syntax is based on that used by git config:
// http://git-scm.com/docs/git-config#_syntax .
// There are some (planned) differences compared to the git config format:
//  - improve data portability:
//    - must be encoded in UTF-8 (for now) and must not contain the 0 byte
//    - include and "path" type is not supported
//      (path type may be implementable as a user-defined type)
//  - disallow potentially ambiguous or misleading definitions:
//    - `[sec.sub]` format is not allowed (deprecated in gitconfig)
//    - `[sec ""]` is not allowed
//      - use `[sec]` for section name "sec" and empty subsection name
//    - within a single file, definitions must be consecutive for each:
//      - section: '[secA]' -> '[secB]' -> '[secA]' is an error
//      - subsection: '[sec "A"]' -> '[sec "B"]' -> '[sec "A"]' is an error
//      - multivalued variable: 'multi=a' -> 'other=x' -> 'multi=b' is an error
//
// The package may be usable for handling some of the various "INI file" formats
// used by some programs and libraries, but achieving or maintaining
// compatibility with any of those is not a primary concern.
//
// TODO:
//  - format
//    - explain "why gcfg"
//    - define valid section and variable names
//    - define handling of default value
//    - complete syntax documentation
//  - reading
//    - define internal representation structure
//    - support multi-value variables
//    - non-regexp based parser
//    - support partially quoted strings
//    - support escaping in strings
//    - support multiple inputs (readers, strings, files)
//    - support declaring encoding (?)
//    - support pointer fields
//    - support varying fields sets for subsections (?)
//  - scanEnum
//    - should use longest match (?)
//    - support matching on unique prefix (?)
//  - writing gcfg files
//  - error handling
//    - include error context
//    - avoid reflection-related panics
//    - more helpful error messages
//    - error types / codes?
//    - limit input size?
//  - move TODOs to issue tracker (eventually)
//
package gcfg

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"reflect"
	"regexp"
	"strings"
)

var (
	reCmnt    = regexp.MustCompile(`^([^;#"]*)[;#].*$`)
	reCmntQ   = regexp.MustCompile(`^([^;#"]*"[^"]*"[^;#"]*)[;#].*$`)
	reBlank   = regexp.MustCompile(`^\s*$`)
	reSect    = regexp.MustCompile(`^\s*\[\s*([^"\s]*)\s*\]\s*$`)
	reSectSub = regexp.MustCompile(`^\s*\[\s*([^"\s]*)\s*"([^"]+)"\s*\]\s*$`)
	reVar     = regexp.MustCompile(`^\s*([^"=\s]+)\s*=\s*([^"\s]*)\s*$`)
	reVarQ    = regexp.MustCompile(`^\s*([^"=\s]+)\s*=\s*"([^"\n\\]*)"\s*$`)
	reVarDflt = regexp.MustCompile(`^\s*\b(.*)\b\s*$`)
)

const (
	// Default value string in case a value for a variable isn't provided.
	defaultValue = "true"
)

type gbool bool

var gboolValues = map[string]interface{}{
	"true": true, "yes": true, "on": true, "1": true,
	"false": false, "no": false, "off": false, "0": false}

func (b *gbool) Scan(state fmt.ScanState, verb rune) error {
	v, err := scanEnum(state, gboolValues, true)
	if err != nil {
		return err
	}
	bb, _ := v.(bool) // cannot be non-bool
	*b = gbool(bb)
	return nil
}

func fieldFold(v reflect.Value, name string) reflect.Value {
	n := strings.Replace(name, "-", "_", -1)
	return v.FieldByNameFunc(func(fieldName string) bool {
		return strings.EqualFold(n, fieldName)
	})
}

func set(cfg interface{}, sect, sub, name, value string) error {
	vDest := reflect.ValueOf(cfg).Elem()
	vSect := fieldFold(vDest, sect)
	if vSect.Kind() == reflect.Map {
		if vSect.IsNil() {
			vSect.Set(reflect.MakeMap(vSect.Type()))
		}
		k := reflect.ValueOf(sub)
		pv := vSect.MapIndex(k)
		if !pv.IsValid() {
			vType := vSect.Type().Elem().Elem()
			pv = reflect.New(vType)
			vSect.SetMapIndex(k, pv)
		}
		vSect = pv.Elem()
	} else if sub != "" {
		return fmt.Errorf("expected map; section %q subsection %q", sect, sub)
	}
	vName := fieldFold(vSect, name)
	vAddr := vName.Addr().Interface()
	switch v := vAddr.(type) {
	case *string:
		*v = value
		return nil
	case *bool:
		vAddr = (*gbool)(v)
	}
	// attempt to read an extra rune to make sure the value is consumed
	var r rune
	n, err := fmt.Sscanf(value, "%v%c", vAddr, &r)
	switch {
	case n < 1 || n == 1 && err != io.EOF:
		return fmt.Errorf("failed to parse %q as %#v: parse error %v", value,
			vName.Type(), err)
	case n > 1:
		return fmt.Errorf("failed to parse %q as %#v: extra characters", value,
			vName.Type())
	case n == 1 && err == io.EOF:
		return nil
	}
	panic("never reached")
}

// ReadInto reads gcfg formatted data from reader and sets the values into the
// corresponding fields in config.
//
// Config must be a pointer to a struct.
// Each section corresponds to a struct field in config, and each variable in a
// section corresponds to a data field in the section struct.
// The name of the field must match the name of the section or variable,
// ignoring case.
// Hyphens in variable names correspond to underscores in section or field
// names.
//
// For sections with subsections, the corresponding field in config must be a
// map, rather than a struct, with string keys and pointer-to-struct values.
// Values for subsection variables are stored in the map with the subsection
// name used as the map key.
// (Note that unlike section and variable names, subsection names are case
// sensitive.)
// When using a map, and there is a section with the same section name but
// without a subsection name, its values are stored with the empty string used
// as the key.
//
// The section structs in the config struct may contain arbitrary types.
// For string fields, the (unquoted and unescaped) value string is assigned to
// the field.
// For bool fields, the field is set to true if the value is "true", "yes", "on"
// or "1", and set to false if the value is "false", "no", "off" or "0",
// ignoring case.
// For all other types, fmt.Sscanf is used to parse the value and set it to the
// field.
// This means that built-in Go types are parseable using the standard format,
// and any user-defined type is parseable if it implements the fmt.Scanner
// interface.
// Note that the value is considered invalid unless fmt.Scanner fully consumes
// the value string without error.
//
// See ReadStringInto for examples.
//
func ReadInto(config interface{}, reader io.Reader) error {
	r := bufio.NewReader(reader)
	sect := (*string)(nil)
	sectsub := ""
	for line := 1; true; line++ {
		l, pre, err := r.ReadLine()
		if err != nil && err != io.EOF {
			return err
		} else if pre {
			return errors.New("line too long")
		}
		// exclude comments
		if c := reCmnt.FindSubmatch(l); c != nil {
			l = c[1]
		} else if c := reCmntQ.FindSubmatch(l); c != nil {
			l = c[1]
		}
		if !reBlank.Match(l) {
			// "switch" based on line contents
			if sec := reSect.FindSubmatch(l); sec != nil {
				strsec := string(sec[1])
				sect, sectsub = &strsec, ""
			} else if sec := reSectSub.FindSubmatch(l); sec != nil {
				strsec := string(sec[1])
				strsub := string(sec[2])
				if strsub == "" {
					return errors.New("empty subsection not allowed")
				}
				sect, sectsub = &strsec, strsub
			} else if v, vq, vd := reVar.FindSubmatch(l),
				reVarQ.FindSubmatch(l), reVarDflt.FindSubmatch(l); //
			v != nil || vq != nil || vd != nil {
				if sect == nil {
					return errors.New("no section")
				}
				var name, value string
				if v != nil {
					name, value = string(v[1]), string(v[2])
				} else if vq != nil {
					name, value = string(vq[1]), string(vq[2])
				} else { // vd != nil
					name, value = string(vd[1]), defaultValue
				}
				err := set(config, *sect, sectsub, name, value)
				if err != nil {
					return err
				}
			} else {
				return fmt.Errorf("invalid line %q", string(l))
			}
		}
		if err == io.EOF {
			break
		}
	}
	return nil
}

// ReadStringInto reads gcfg formatted data from str and sets the values into
// the corresponding fields in config.
// ReadStringInfo is a wrapper for ReadInfo; see ReadInto(config, reader) for
// detailed description of how data is read and set into config.
func ReadStringInto(config interface{}, str string) error {
	r := strings.NewReader(str)
	return ReadInto(config, r)
}

// ReadFileInto reads gcfg formatted data from the file filename and sets the
// values into the corresponding fields in config.
// ReadFileInto is a wrapper for ReadInfo; see ReadInto(config, reader) for
// detailed description of how data is read and set into config.
func ReadFileInto(config interface{}, filename string) error {
	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	return ReadInto(config, f)
}
<|endoftext|>


<commit_before>package ghch

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/Masterminds/semver"
	"github.com/octokit/go-octokit/octokit"
)

type ghch struct {
	repoPath string
	remote   string
	verbose  bool
	client   *octokit.Client
}

func (gh *ghch) initialize() *ghch {
	// XXX authentication
	gh.client = octokit.NewClient(nil)
	return gh
}

func (gh *ghch) cmd(argv ...string) (string, error) {
	arg := []string{"-C", gh.repoPath}
	arg = append(arg, argv...)
	cmd := exec.Command("git", arg...)
	cmd.Env = append(os.Environ(), "LANG=C")

	var b bytes.Buffer
	cmd.Stdout = &b
	cmd.Stderr = os.Stderr
	err := cmd.Run()
	return b.String(), err
}

var verReg = regexp.MustCompile(`^v?[0-9]+(?:\.[0-9]+){0,2}$`)

func (gh *ghch) versions() []string {
	out, _ := gh.cmd("tag")
	return parseVerions(out)
}

func parseVerions(out string) []string {
	rawTags := strings.Split(out, "\n")
	var versions []*semver.Version
	for _, tag := range rawTags {
		t := strings.TrimSpace(tag)
		if verReg.MatchString(t) {
			v, _ := semver.NewVersion(t)
			versions = append(versions, v)
		}
	}
	sort.Sort(sort.Reverse(semver.Collection(versions)))
	var vers = make([]string, len(versions))
	for i, v := range versions {
		vers[i] = v.Original()
	}
	return vers
}

func (gh *ghch) getRemote() string {
	if gh.remote != "" {
		return gh.remote
	}
	return "origin"
}

var repoURLReg = regexp.MustCompile(`([^/:]+)/([^/]+?)(?:\.git)?$`)

func (gh *ghch) ownerAndRepo() (owner, repo string) {
	out, _ := gh.cmd("remote", "-v")
	remotes := strings.Split(out, "\n")
	for _, r := range remotes {
		fields := strings.Fields(r)
		if len(fields) > 1 && fields[0] == gh.getRemote() {
			if matches := repoURLReg.FindStringSubmatch(fields[1]); len(matches) > 2 {
				return matches[1], matches[2]
			}
		}
	}
	return
}

func (gh *ghch) mergedPRs(from, to string) (prs []*octokit.PullRequest) {
	owner, repo := gh.ownerAndRepo()
	nums := gh.mergedPRNums(from, to)
	for _, num := range nums {
		url, _ := octokit.PullRequestsURL.Expand(octokit.M{"owner": owner, "repo": repo, "number": num})
		pr, r := gh.client.PullRequests(url).One()
		if r.HasError() {
			log.Print(r.Err)
			continue
		}
		if !gh.verbose {
			pr = reducePR(pr)
		}
		prs = append(prs, pr)
	}
	return
}

var prMergeReg = regexp.MustCompile(`^[a-f0-9]{7} Merge pull request #([0-9]+) from`)

func (gh *ghch) mergedPRNums(from, to string) (nums []int) {
	if from == "" {
		vers := gh.versions()
		if len(vers) < 1 {
			return
		}
		from = vers[0]
	}
	revisionRange := fmt.Sprintf("%s..%s", from, to)
	out, err := gh.cmd("log", revisionRange, "--merges", "--oneline")
	if err != nil {
		return
	}
	return parseMergedPRNums(out)
}

func parseMergedPRNums(out string) (nums []int) {
	lines := strings.Split(out, "\n")
	for _, line := range lines {
		if matches := prMergeReg.FindStringSubmatch(line); len(matches) > 1 {
			i, _ := strconv.Atoi(matches[1])
			nums = append(nums, i)
		}
	}
	return
}
<commit_msg>use sync.WaitGroup<commit_after>package ghch

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/Masterminds/semver"
	"github.com/octokit/go-octokit/octokit"
)

type ghch struct {
	repoPath string
	remote   string
	verbose  bool
	client   *octokit.Client
}

func (gh *ghch) initialize() *ghch {
	// XXX authentication
	gh.client = octokit.NewClient(nil)
	return gh
}

func (gh *ghch) cmd(argv ...string) (string, error) {
	arg := []string{"-C", gh.repoPath}
	arg = append(arg, argv...)
	cmd := exec.Command("git", arg...)
	cmd.Env = append(os.Environ(), "LANG=C")

	var b bytes.Buffer
	cmd.Stdout = &b
	cmd.Stderr = os.Stderr
	err := cmd.Run()
	return b.String(), err
}

var verReg = regexp.MustCompile(`^v?[0-9]+(?:\.[0-9]+){0,2}$`)

func (gh *ghch) versions() []string {
	out, _ := gh.cmd("tag")
	return parseVerions(out)
}

func parseVerions(out string) []string {
	rawTags := strings.Split(out, "\n")
	var versions []*semver.Version
	for _, tag := range rawTags {
		t := strings.TrimSpace(tag)
		if verReg.MatchString(t) {
			v, _ := semver.NewVersion(t)
			versions = append(versions, v)
		}
	}
	sort.Sort(sort.Reverse(semver.Collection(versions)))
	var vers = make([]string, len(versions))
	for i, v := range versions {
		vers[i] = v.Original()
	}
	return vers
}

func (gh *ghch) getRemote() string {
	if gh.remote != "" {
		return gh.remote
	}
	return "origin"
}

var repoURLReg = regexp.MustCompile(`([^/:]+)/([^/]+?)(?:\.git)?$`)

func (gh *ghch) ownerAndRepo() (owner, repo string) {
	out, _ := gh.cmd("remote", "-v")
	remotes := strings.Split(out, "\n")
	for _, r := range remotes {
		fields := strings.Fields(r)
		if len(fields) > 1 && fields[0] == gh.getRemote() {
			if matches := repoURLReg.FindStringSubmatch(fields[1]); len(matches) > 2 {
				return matches[1], matches[2]
			}
		}
	}
	return
}

func (gh *ghch) mergedPRs(from, to string) (prs []*octokit.PullRequest) {
	owner, repo := gh.ownerAndRepo()
	nums := gh.mergedPRNums(from, to)

	var wg sync.WaitGroup
	prCh := make(chan *octokit.PullRequest)

	go func() {
		for pr := range prCh {
			prs = append(prs, pr)
		}
	}()

	for _, num := range nums {
		wg.Add(1)
		go func(num int) {
			defer wg.Done()
			url, _ := octokit.PullRequestsURL.Expand(octokit.M{"owner": owner, "repo": repo, "number": num})
			pr, r := gh.client.PullRequests(url).One()
			if r.HasError() {
				log.Print(r.Err)
				return
			}
			if !gh.verbose {
				pr = reducePR(pr)
			}
			prCh <- pr
		}(num)
	}
	wg.Wait()
	close(prCh)

	return
}

var prMergeReg = regexp.MustCompile(`^[a-f0-9]{7} Merge pull request #([0-9]+) from`)

func (gh *ghch) mergedPRNums(from, to string) (nums []int) {
	if from == "" {
		vers := gh.versions()
		if len(vers) < 1 {
			return
		}
		from = vers[0]
	}
	revisionRange := fmt.Sprintf("%s..%s", from, to)
	out, err := gh.cmd("log", revisionRange, "--merges", "--oneline")
	if err != nil {
		return
	}
	return parseMergedPRNums(out)
}

func parseMergedPRNums(out string) (nums []int) {
	lines := strings.Split(out, "\n")
	for _, line := range lines {
		if matches := prMergeReg.FindStringSubmatch(line); len(matches) > 1 {
			i, _ := strconv.Atoi(matches[1])
			nums = append(nums, i)
		}
	}
	return
}
<|endoftext|>


<commit_before>package godb

import (
	"database/sql"
	"errors"
	"log"
	"time"

	"github.com/samonzeweb/godb/adapters"
)

// DB stores a connection to the database, the current transaction, logger, ...
// Everything starts with a DB.
// DB is not thread safe (see Clone).
type DB struct {
	adapter      adapters.Adapter
	sqlDB        *sql.DB
	sqlTx        *sql.Tx
	logger       *log.Logger
	consumedTime time.Duration

	// Prepared Statement cache for DB and Tx
	stmtCacheDB *StmtCache
	stmtCacheTx *StmtCache
}

// Placeholder is the placeholder string, use it to build queries.
// Adapters could change it before queries are executed.
const Placeholder string = "?"

// ErrOpLock is an error returned when Optimistic Locking failure occurs
var ErrOpLock = errors.New("Optimistic Locking Failure")

// Open creates a new DB struct and initialise a sql.DB connection.
func Open(adapter adapters.Adapter, dataSourceName string) (*DB, error) {
	db := DB{
		adapter:     adapter,
		stmtCacheDB: newStmtCache(),
		stmtCacheTx: newStmtCache(),
	}

	// Prepared statements cache is disabled by default except for Tx
	db.stmtCacheDB.Disable()

	var err error
	db.sqlDB, err = sql.Open(adapter.DriverName(), dataSourceName)
	if err != nil {
		return nil, err
	}
	return &db, nil
}

// Clone creates a copy of an existing DB, without the current transaction.
// The clone has no consumed time, and new prepared statements caches with
// the same characteristics.
// Use it to create new DB object before starting a goroutine.
func (db *DB) Clone() *DB {
	clone := &DB{
		adapter:      db.adapter,
		sqlDB:        db.sqlDB,
		sqlTx:        nil,
		logger:       db.logger,
		consumedTime: 0,
		stmtCacheDB:  newStmtCache(),
		stmtCacheTx:  newStmtCache(),
	}

	clone.stmtCacheDB.SetSize(db.stmtCacheDB.GetSize())
	if !db.stmtCacheDB.IsEnabled() {
		clone.stmtCacheDB.Disable()
	}

	clone.stmtCacheTx.SetSize(db.stmtCacheTx.GetSize())
	if !db.stmtCacheTx.IsEnabled() {
		clone.stmtCacheTx.Disable()
	}

	return clone
}

// Close closes an existing DB created by Open.
// Dont't close a cloned DB still used by others goroutines as the sql.DB
// is shared !
// Don't use a DB anymore after a call to Close.
func (db *DB) Close() error {
	db.logPrintln("CLOSE DB")
	if db.sqlTx != nil {
		db.logPrintln("Warning, there is a current transaction")
	}
	return db.sqlDB.Close()
}

// Adapter returns the current adapter.
func (db *DB) Adapter() adapters.Adapter {
	return db.adapter
}

// CurrentDB returns the current *sql.DB.
// Use it wisely.
func (db *DB) CurrentDB() *sql.DB {
	return db.sqlDB
}

// ConsumedTime returns the time consumed by SQL queries executions
// The duration is reseted when the DB is cloned.
func (db *DB) ConsumedTime() time.Duration {
	return db.consumedTime
}

// ResetConsumedTime resets the time consumed by SQL queries executions
func (db *DB) ResetConsumedTime() {
	db.consumedTime = 0
}

// addConsumedTime adds duration to the consumed time
func (db *DB) addConsumedTime(duration time.Duration) {
	db.consumedTime += duration
}

// timeElapsedSince returns the time elapsed (duration) since a given
// start time.
func timeElapsedSince(startTime time.Time) time.Duration {
	return time.Now().Sub(startTime)
}

// quoteAll returns all strings given quoted by the adapter.
func (db *DB) quoteAll(identifiers []string) []string {
	quotedIdentifiers := make([]string, 0, len(identifiers))
	for _, identifier := range identifiers {
		quotedIdentifiers = append(quotedIdentifiers, db.adapter.Quote(identifier))
	}
	return quotedIdentifiers
}

// replacePlaceholders uses the adapter to change placehodlers according to
// the database used.
func (db *DB) replacePlaceholders(sql string) string {
	placeholderReplacer, ok := (db.adapter).(adapters.PlaceholdersReplacer)
	if !ok {
		return sql
	}

	return placeholderReplacer.ReplacePlaceholders(Placeholder, sql)
}
<commit_msg>added Wrap(sql.DB) for preconfigured sql.DB instance (#5)<commit_after>package godb

import (
	"database/sql"
	"errors"
	"log"
	"time"

	"github.com/samonzeweb/godb/adapters"
)

// DB stores a connection to the database, the current transaction, logger, ...
// Everything starts with a DB.
// DB is not thread safe (see Clone).
type DB struct {
	adapter      adapters.Adapter
	sqlDB        *sql.DB
	sqlTx        *sql.Tx
	logger       *log.Logger
	consumedTime time.Duration

	// Prepared Statement cache for DB and Tx
	stmtCacheDB *StmtCache
	stmtCacheTx *StmtCache
}

// Placeholder is the placeholder string, use it to build queries.
// Adapters could change it before queries are executed.
const Placeholder string = "?"

// ErrOpLock is an error returned when Optimistic Locking failure occurs
var ErrOpLock = errors.New("optimistic locking failure")

// Open creates a new DB struct and initialise a sql.DB connection.
func Open(adapter adapters.Adapter, dataSourceName string) (*DB, error) {
	db := DB{
		adapter:     adapter,
		stmtCacheDB: newStmtCache(),
		stmtCacheTx: newStmtCache(),
	}

	// Prepared statements cache is disabled by default except for Tx
	db.stmtCacheDB.Disable()

	var err error
	db.sqlDB, err = sql.Open(adapter.DriverName(), dataSourceName)
	if err != nil {
		return nil, err
	}
	return &db, nil
}

// Wrap creates a godb.DB by using provided and initialized sql.DB Helpful for
// using custom configured sql.DB instance for godb. Can be used before
// starting a goroutine.
func Wrap(adapter adapters.Adapter, dbInst *sql.DB) *DB {
	db := DB{
		adapter:     adapter,
		stmtCacheDB: newStmtCache(),
		stmtCacheTx: newStmtCache(),
	}

	// Prepared statements cache is disabled by default except for Tx
	db.stmtCacheDB.Disable()
	db.sqlDB = dbInst
	return &db
}

// Clone creates a copy of an existing DB, without the current transaction.
// The clone has consumedTime set to zero, and new prepared statements caches with
// the same characteristics.
// Use it to create new DB object before starting a goroutine.
func (db *DB) Clone() *DB {
	clone := &DB{
		adapter:      db.adapter,
		sqlDB:        db.sqlDB,
		sqlTx:        nil,
		logger:       db.logger,
		consumedTime: 0,
		stmtCacheDB:  newStmtCache(),
		stmtCacheTx:  newStmtCache(),
	}

	clone.stmtCacheDB.SetSize(db.stmtCacheDB.GetSize())
	if !db.stmtCacheDB.IsEnabled() {
		clone.stmtCacheDB.Disable()
	}

	clone.stmtCacheTx.SetSize(db.stmtCacheTx.GetSize())
	if !db.stmtCacheTx.IsEnabled() {
		clone.stmtCacheTx.Disable()
	}

	return clone
}

// Close closes an existing DB created by Open.
// Don't close a cloned DB still used by others goroutines as the sql.DB
// is shared !
// Don't use a DB anymore after a call to Close.
func (db *DB) Close() error {
	db.logPrintln("CLOSE DB")
	if db.sqlTx != nil {
		db.logPrintln("Warning, there is a current transaction")
	}
	return db.sqlDB.Close()
}

// Adapter returns the current adapter.
func (db *DB) Adapter() adapters.Adapter {
	return db.adapter
}

// CurrentDB returns the current *sql.DB.
// Use it wisely.
func (db *DB) CurrentDB() *sql.DB {
	return db.sqlDB
}

// ConsumedTime returns the time consumed by SQL queries executions
// The duration is reseted when the DB is cloned.
func (db *DB) ConsumedTime() time.Duration {
	return db.consumedTime
}

// ResetConsumedTime resets the time consumed by SQL queries executions
func (db *DB) ResetConsumedTime() {
	db.consumedTime = 0
}

// addConsumedTime adds duration to the consumed time
func (db *DB) addConsumedTime(duration time.Duration) {
	db.consumedTime += duration
}

// timeElapsedSince returns the time elapsed (duration) since a given
// start time.
func timeElapsedSince(startTime time.Time) time.Duration {
	return time.Now().Sub(startTime)
}

// quoteAll returns all strings given quoted by the adapter.
func (db *DB) quoteAll(identifiers []string) []string {
	quotedIdentifiers := make([]string, 0, len(identifiers))
	for _, identifier := range identifiers {
		quotedIdentifiers = append(quotedIdentifiers, db.adapter.Quote(identifier))
	}
	return quotedIdentifiers
}

// replacePlaceholders uses the adapter to change placeholders according to
// the database used.
func (db *DB) replacePlaceholders(sql string) string {
	placeholderReplacer, ok := (db.adapter).(adapters.PlaceholdersReplacer)
	if !ok {
		return sql
	}

	return placeholderReplacer.ReplacePlaceholders(Placeholder, sql)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/toqueteos/webbrowser"
	"gopkg.in/alecthomas/kingpin.v2"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"
	"io"
)

var (
	port = kingpin.Flag("port", "Listening port used by webserver").Short('p').Default("1110").Int()
	file = kingpin.Arg("file", "Markdown file").String()
)

type EditorView struct {
	File    string
	Content string
}

func main() {
	kingpin.Version("0.0.1")
	kingpin.Parse()

	e := echo.New()

	t := &Template{
		templates: template.Must(template.ParseGlob("template/*.html")),
	}
	e.SetRenderer(t)

	e.Use(middleware.Logger())
	e.Use(middleware.Recover())

	e.Static("/static/", "static")

	edit := e.Group("/edit")
	edit.Get("/*", EditHandler)
	edit.Post("/*", EditHandler)

	go WaitForServer(port)
	e.Run(fmt.Sprintf("127.0.0.1:%d", *port))
}

type Template struct {
	templates *template.Template
}

func (t *Template) Render(w io.Writer, name string, data interface{}) error {
	return t.templates.ExecuteTemplate(w, name, data)
}

func EditHandler(c *echo.Context) error {
	filepath := c.P(0)
	if c.Request().Method == "POST" {
		c.Request().ParseForm()
		ioutil.WriteFile(filepath, []byte(c.Request().PostForm.Get("content")), 0644)
	}
	content, err := ioutil.ReadFile(filepath)
	if err != nil {
		return echo.NewHTTPError(http.StatusInternalServerError, "Unable to read requested file")
		log.Fatalf("Unable to open file " + filepath)
	}
	ev := EditorView{File: filepath, Content: string(content)}
	return c.Render(http.StatusOK, "base", ev)
}

func WaitForServer(port *int) {
	log.Printf("Waiting for listener on port %d", *port)
	url := fmt.Sprintf("http://localhost:%d/edit/%s", *port, url.QueryEscape(*file))
	for {
		time.Sleep(time.Millisecond * 50)
		resp, err := http.Get(url)
		if err != nil {
			continue
		}
		resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			continue
		}
		break
	}
	log.Println("Opening " + url)
	if err := webbrowser.Open(url); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Organize input arguments in a struct<commit_after>package main

import (
	"fmt"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/toqueteos/webbrowser"
	"gopkg.in/alecthomas/kingpin.v2"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"
)

type InputArgs struct {
	Port *int
	File *string
}

var args = InputArgs{
	Port: kingpin.Flag("port", "Listening port used by webserver").Short('p').Default("1110").Int(),
	File: kingpin.Arg("file", "Markdown file").String(),
}

type EditorView struct {
	File    string
	Content string
}

func main() {
	kingpin.Version("0.0.1")
	kingpin.Parse()

	e := echo.New()

	t := &Template{
		templates: template.Must(template.ParseGlob("template/*.html")),
	}
	e.SetRenderer(t)

	e.Use(middleware.Logger())
	e.Use(middleware.Recover())

	e.Static("/static/", "static")

	edit := e.Group("/edit")
	edit.Get("/*", EditHandler)
	edit.Post("/*", EditHandler)

	go WaitForServer()
	e.Run(fmt.Sprintf("127.0.0.1:%d", *args.Port))
}

type Template struct {
	templates *template.Template
}

func (t *Template) Render(w io.Writer, name string, data interface{}) error {
	return t.templates.ExecuteTemplate(w, name, data)
}

func EditHandler(c *echo.Context) error {
	filepath := c.P(0)
	if c.Request().Method == "POST" {
		c.Request().ParseForm()
		ioutil.WriteFile(filepath, []byte(c.Request().PostForm.Get("content")), 0644)
	}
	content, err := ioutil.ReadFile(filepath)
	if err != nil {
		return echo.NewHTTPError(http.StatusInternalServerError, "Unable to read requested file")
		log.Fatalf("Unable to open file " + filepath)
	}
	ev := EditorView{File: filepath, Content: string(content)}
	return c.Render(http.StatusOK, "base", ev)
}

func WaitForServer() {
	log.Printf("Waiting for listener on port %d", *args.Port)
	url := fmt.Sprintf("http://localhost:%d/edit/%s", *args.Port, url.QueryEscape(*args.File))
	for {
		time.Sleep(time.Millisecond * 50)
		resp, err := http.Get(url)
		if err != nil {
			continue
		}
		resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			continue
		}
		break
	}
	log.Println("Opening " + url)
	if err := webbrowser.Open(url); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Program gops is a tool to list currently running Go processes.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/google/gops/internal/objfile"

	ps "github.com/keybase/go-ps"
)

const helpText = `Usage: gops is a tool to list and diagnose Go processes.

    gops                  Lists all Go processes currently running.
    gops [cmd] -p=<pid>   See the section below.

Commands:
    stack     Prints the stack trace.
    gc        Runs the garbage collector and blocks until successful.
    memstats  Prints the garbage collection stats.
    version   Prints the Go version used to build the program.

All commands require the agent running on the Go process.
`

// TODO(jbd): add link that explains the use of agent.

func main() {
	if len(os.Args) < 2 {
		processes()
		return
	}

	cmd := os.Args[1]
	fn, ok := cmds[cmd]
	if !ok {
		usage("unknown subcommand")
	}

	var pid int
	flag.IntVar(&pid, "p", -1, "")
	flag.CommandLine.Parse(os.Args[2:])
	if pid == -1 {
		usage("missing -p=<pid> flag")
	}

	if err := fn(pid); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}

func processes() {
	pss, err := ps.Processes()
	if err != nil {
		log.Fatal(err)
	}
	var undetermined int
	for _, pr := range pss {
		name, err := pr.Path()
		if err != nil {
			undetermined++
			continue
		}
		ok, err := isGo(name)
		if err != nil {
			// TODO(jbd): worth to report the number?
			continue
		}
		if ok {
			// TODO(jbd): List if the program is running the agent.
			fmt.Printf("%d\t%v\t(%v)\n", pr.Pid(), pr.Executable(), name)
		}
	}
	if undetermined > 0 {
		fmt.Printf("\n%d processes left undetermined\n", undetermined)
	}
}

func isGo(executable string) (ok bool, err error) {
	obj, err := objfile.Open(executable)
	if err != nil {
		return false, err
	}
	defer obj.Close()

	symbols, err := obj.Symbols()
	if err != nil {
		return false, err
	}

	// TODO(jbd): find a faster way to determine Go programs.
	for _, s := range symbols {
		if s.Name == "runtime.buildVersion" {
			return true, nil
		}
		if strings.HasPrefix(s.Name, "github.com/google/gops") {
			return true, nil
		}
	}
	return false, nil
}

func usage(msg string) {
	if msg != "" {
		fmt.Printf("gops: %v\n", msg)
	}
	fmt.Fprintf(os.Stderr, "%v\n", helpText)
	os.Exit(1)
}
<commit_msg>remove unnecessary agent sym check<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Program gops is a tool to list currently running Go processes.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/google/gops/internal/objfile"

	ps "github.com/keybase/go-ps"
)

const helpText = `Usage: gops is a tool to list and diagnose Go processes.

    gops                  Lists all Go processes currently running.
    gops [cmd] -p=<pid>   See the section below.

Commands:
    stack     Prints the stack trace.
    gc        Runs the garbage collector and blocks until successful.
    memstats  Prints the garbage collection stats.
    version   Prints the Go version used to build the program.

All commands require the agent running on the Go process.
`

// TODO(jbd): add link that explains the use of agent.

func main() {
	if len(os.Args) < 2 {
		processes()
		return
	}

	cmd := os.Args[1]
	fn, ok := cmds[cmd]
	if !ok {
		usage("unknown subcommand")
	}

	var pid int
	flag.IntVar(&pid, "p", -1, "")
	flag.CommandLine.Parse(os.Args[2:])
	if pid == -1 {
		usage("missing -p=<pid> flag")
	}

	if err := fn(pid); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}

func processes() {
	pss, err := ps.Processes()
	if err != nil {
		log.Fatal(err)
	}
	var undetermined int
	for _, pr := range pss {
		name, err := pr.Path()
		if err != nil {
			undetermined++
			continue
		}
		ok, err := isGo(name)
		if err != nil {
			// TODO(jbd): worth to report the number?
			continue
		}
		if ok {
			// TODO(jbd): List if the program is running the agent.
			fmt.Printf("%d\t%v\t(%v)\n", pr.Pid(), pr.Executable(), name)
		}
	}
	if undetermined > 0 {
		fmt.Printf("\n%d processes left undetermined\n", undetermined)
	}
}

func isGo(executable string) (ok bool, err error) {
	obj, err := objfile.Open(executable)
	if err != nil {
		return false, err
	}
	defer obj.Close()

	symbols, err := obj.Symbols()
	if err != nil {
		return false, err
	}

	// TODO(jbd): find a faster way to determine Go programs.
	for _, s := range symbols {
		if s.Name == "runtime.buildVersion" {
			return true, nil
		}
	}
	return false, nil
}

func usage(msg string) {
	if msg != "" {
		fmt.Printf("gops: %v\n", msg)
	}
	fmt.Fprintf(os.Stderr, "%v\n", helpText)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package main

import (
    "bufio"
    "flag"
    "fmt"
    "log"
    "os"
    "regexp"
    "strings"
    "time"
    "path"
)

type SubData struct {
    Num         string
    TimeStart   time.Time
    TimeEnd     time.Time
    Text        string
}

const TIME_FORMAT    = "15:04:05.000"

var timeReference   time.Time
var subLines        []SubData
var replaceComma    bool
var subFileName     string

func main() {
    subFile := flag.String("f", "", "subtitle file")
    shiftTime := flag.String("t", "", "time shift with format hh:mm:ss.000")

    replaceComma = false

    flag.Parse()

    log.SetFlags(0)

    timeReference = setClock(time.Now(), 0, 0, 0, 0)

    readSubFile(*subFile)
    shiftSub(*shiftTime)
    writeSub()
}

func readSubFile(fileName string) {
    subFileName = fileName

    log.Println("Processing " + subFileName)

    file, err := os.Open(subFileName)

    if err != nil {
        log.Fatalf("Error opening subtitle file: %v", err)
        return
    }

    defer file.Close()

    scanner := bufio.NewScanner(file)

    subLines = make([]SubData, 0)

    var sub SubData

    for scanner.Scan() {
        text := scanner.Text()

        reNum := regexp.MustCompile(`^\d+$`)
        reTime := regexp.MustCompile(`^(\d{2}:\d{2}:[\d(,|\.)]+).+(\d{2}:\d{2}:[\d(,|\.)]+)$`)

        switch {
        case reNum.MatchString(text):
            sub = SubData{}
            sub.Num = text

        case reTime.MatchString(text):
            subtime := reTime.FindStringSubmatch(text)

            if strings.LastIndex(subtime[1], ",") != -1 {
                subtime[1] = strings.Replace(subtime[1], ",", ".", 1)
                subtime[2] = strings.Replace(subtime[2], ",", ".", 1)

                replaceComma = true
            }

            timeStart, _ := time.Parse(TIME_FORMAT, subtime[1])
            timeEnd, _ := time.Parse(TIME_FORMAT, subtime[2])

            sub.TimeStart = setClock(timeReference,
                timeStart.Hour(), timeStart.Minute(),
                timeStart.Second(), timeStart.Nanosecond())
            sub.TimeEnd = setClock(timeReference,
                timeEnd.Hour(), timeEnd.Minute(),
                timeEnd.Second(), timeEnd.Nanosecond())

        case text != "":
            sub.Text += text + "\n"

        default:
            sub.Text = strings.TrimSpace(sub.Text)
            subLines = append(subLines, sub)

        }
    }

    if err := scanner.Err(); err != nil {
        log.Fatal(err)
    }
}

func shiftSub(shift string) {
    timeFormat := []string{
        "5",
        "4:5",
        "15:4:5",
        "5.000",
        "4:5.000",
        "15:4:5.000",
    }

    var sTime time.Time
    var err error
    sign := 1;
    trimmedShift := shift

    if strings.HasPrefix(shift, "-") {
        trimmedShift = strings.Trim(shift, "- ")
        sign = -1;
    }

    for i := 0; i < len(timeFormat); i++ {
        sTime, err = time.Parse(timeFormat[i], trimmedShift)

        if err != nil {
            if i < 2 {
                continue
            } else {
                log.Fatalf("shift time (%s) not recognized", shift)
            }
        }

        break;
    }

    tfmt := sTime.Format(fmt.Sprintf("%d:04:05.000", sign * 15))
    log.Println("Shifting by: " + tfmt)

    shiftDur := time.Duration(sTime.Hour()) * time.Hour +
                time.Duration(sTime.Minute()) * time.Minute +
                time.Duration(sTime.Second()) * time.Second +
                time.Duration(sTime.Nanosecond()) * time.Nanosecond

    if sign == -1 {
        shiftDur = -shiftDur
    }

    for i, sub := range subLines {
        var timeStart, timeEnd time.Time

        timeStart = sub.TimeStart.Add(shiftDur)
        timeEnd = sub.TimeEnd.Add(shiftDur)

        if timeStart.Before(timeReference) {
            timeStart = setClock(timeReference, 0, 0, 0, 0)
        }

        if timeEnd.Before(timeReference) {
            timeEnd = setClock(timeReference, 0, 0, 0, 0)
        }

        sub.TimeStart = timeStart
        sub.TimeEnd = timeEnd

        subLines[i] = sub
    }
}

func writeSub() {
    ext := path.Ext(subFileName)
    fname := strings.TrimSuffix(subFileName, ext)
    resName := fname + "-resync" + ext

    file, err := os.Create(resName)

    if err != nil {
        log.Fatalf("Error creating new subtitle file: %v", err)
        return
    }

    defer file.Close()

    for _, sub := range subLines {
        file.WriteString(sub.Num + "\n")

        start := sub.TimeStart.Format("15:04:05.000")
        end := sub.TimeEnd.Format("15:04:05.000")

        file.WriteString(start + " --> " + end + "\n")
        file.WriteString(sub.Text + "\n\n")
    }

    log.Printf("Result has been saved as %s", resName)
}

func setClock(originalTime time.Time, hour, minute, second, nanosecond int) time.Time {
    refYear, refMonth, refDay := originalTime.Date()

    return time.Date(refYear, refMonth, refDay,
        hour, minute, second, nanosecond,
        originalTime.Location())
}
<commit_msg>remove any UTF BOM<commit_after>package main

import (
    "bufio"
    "flag"
    "fmt"
    "log"
    "os"
    "regexp"
    "strings"
    "time"
    "path"
)

type SubData struct {
    Num         string
    TimeStart   time.Time
    TimeEnd     time.Time
    Text        string
}

const TIME_FORMAT    = "15:04:05.000"

var timeReference   time.Time
var subLines        []SubData
var replaceComma    bool
var subFileName     string

func main() {
    subFile := flag.String("f", "", "subtitle file")
    shiftTime := flag.String("t", "", "time shift with format hh:mm:ss.000")

    replaceComma = false

    flag.Parse()

    log.SetFlags(0)

    timeReference = setClock(time.Now(), 0, 0, 0, 0)

    readSubFile(*subFile)
    shiftSub(*shiftTime)
    writeSub()
}

func readSubFile(fileName string) {
    subFileName = fileName

    log.Println("Processing " + subFileName)

    file, err := os.Open(subFileName)

    if err != nil {
        log.Fatalf("Error opening subtitle file: %v", err)
        return
    }

    defer file.Close()

    scanner := bufio.NewScanner(file)

    subLines = make([]SubData, 0)

    var sub SubData

    for scanner.Scan() {
        text := strings.TrimSpace(scanner.Text())

        reBom := regexp.MustCompile(`\x{feff}`)
        reNum := regexp.MustCompile(`^\d+$`)
        reTime := regexp.MustCompile(`^(\d{2}:\d{2}:[\d(,|\.)]+).+(\d{2}:\d{2}:[\d(,|\.)]+)$`)

        // remove any UTF BOM
        text = reBom.ReplaceAllString(text, "")

        switch {
        case reNum.MatchString(text):
            sub = SubData{}
            sub.Num = text

        case reTime.MatchString(text):
            subtime := reTime.FindStringSubmatch(text)

            if strings.LastIndex(subtime[1], ",") != -1 {
                subtime[1] = strings.Replace(subtime[1], ",", ".", 1)
                subtime[2] = strings.Replace(subtime[2], ",", ".", 1)

                replaceComma = true
            }

            timeStart, _ := time.Parse(TIME_FORMAT, subtime[1])
            timeEnd, _ := time.Parse(TIME_FORMAT, subtime[2])

            sub.TimeStart = setClock(timeReference,
                timeStart.Hour(), timeStart.Minute(),
                timeStart.Second(), timeStart.Nanosecond())
            sub.TimeEnd = setClock(timeReference,
                timeEnd.Hour(), timeEnd.Minute(),
                timeEnd.Second(), timeEnd.Nanosecond())

        case text != "":
            sub.Text += text + "\n"

        default:
            sub.Text = strings.TrimSpace(sub.Text)
            subLines = append(subLines, sub)

        }
    }

    if err := scanner.Err(); err != nil {
        log.Fatal(err)
    }
}

func shiftSub(shift string) {
    timeFormat := []string{
        "5",
        "4:5",
        "15:4:5",
        "5.000",
        "4:5.000",
        "15:4:5.000",
    }

    var sTime time.Time
    var err error
    sign := 1;
    trimmedShift := shift

    if strings.HasPrefix(shift, "-") {
        trimmedShift = strings.Trim(shift, "- ")
        sign = -1;
    }

    for i := 0; i < len(timeFormat); i++ {
        sTime, err = time.Parse(timeFormat[i], trimmedShift)

        if err != nil {
            if i < 2 {
                continue
            } else {
                log.Fatalf("shift time (%s) not recognized", shift)
            }
        }

        break;
    }

    tfmt := sTime.Format(fmt.Sprintf("%d:04:05.000", sign * 15))
    log.Println("Shifting by: " + tfmt)

    shiftDur := time.Duration(sTime.Hour()) * time.Hour +
                time.Duration(sTime.Minute()) * time.Minute +
                time.Duration(sTime.Second()) * time.Second +
                time.Duration(sTime.Nanosecond()) * time.Nanosecond

    if sign == -1 {
        shiftDur = -shiftDur
    }

    for i, sub := range subLines {
        var timeStart, timeEnd time.Time

        timeStart = sub.TimeStart.Add(shiftDur)
        timeEnd = sub.TimeEnd.Add(shiftDur)

        if timeStart.Before(timeReference) {
            timeStart = setClock(timeReference, 0, 0, 0, 0)
        }

        if timeEnd.Before(timeReference) {
            timeEnd = setClock(timeReference, 0, 0, 0, 0)
        }

        sub.TimeStart = timeStart
        sub.TimeEnd = timeEnd

        subLines[i] = sub
    }
}

func writeSub() {
    ext := path.Ext(subFileName)
    fname := strings.TrimSuffix(subFileName, ext)
    resName := fname + "-resync" + ext

    file, err := os.Create(resName)

    if err != nil {
        log.Fatalf("Error creating new subtitle file: %v", err)
        return
    }

    defer file.Close()

    for _, sub := range subLines {
        start := sub.TimeStart.Format("15:04:05.000")
        end := sub.TimeEnd.Format("15:04:05.000")

        if replaceComma {
            start = strings.Replace(start, ".", ",", 1)
            end = strings.Replace(end, ".", ",", 1)
        }

        file.WriteString(sub.Num + "\n")
        file.WriteString(start + " --> " + end + "\n")
        file.WriteString(sub.Text + "\n\n")
    }

    log.Printf("Result has been saved as %s", resName)
}

func setClock(originalTime time.Time, hour, minute, second, nanosecond int) time.Time {
    refYear, refMonth, refDay := originalTime.Date()

    return time.Date(refYear, refMonth, refDay,
        hour, minute, second, nanosecond,
        originalTime.Location())
}
<|endoftext|>


<commit_before>package grim

// Copyright 2015 MediaMath <http://www.mediamath.com>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

import "fmt"

// Instance models the state of a configured Grim instance.
type Instance struct {
	configRoot *string
	queue      *sqsQueue
}

// SetConfigRoot sets the base path of the configuration directory and clears any previously read config values from memory.
func (i *Instance) SetConfigRoot(path string) {
	i.configRoot = &path
	i.queue = nil
}

// PrepareGrimQueue creates or reuses the Amazon SQS queue named in the config.
func (i *Instance) PrepareGrimQueue() error {
	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	queue, err := prepareSQSQueue(config.awsKey, config.awsSecret, config.awsRegion, config.grimQueueName)
	if err != nil {
		return fatalGrimErrorf("error preparing queue: %v", err)
	}

	i.queue = queue

	return nil
}

// PrepareRepos discovers all repos that are configured then sets up SNS and GitHub.
// It is an error to call this without calling PrepareGrimQueue first.
func (i *Instance) PrepareRepos() error {
	if err := i.checkGrimQueue(); err != nil {
		return err
	}

	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	repos := getAllConfiguredRepos(configRoot)

	var topicARNs []string
	for _, repo := range repos {
		localConfig, err := getEffectiveConfig(configRoot, repo.owner, repo.name)
		if err != nil {
			return fatalGrimErrorf("error while reading local config: %v", err)
		}

		snsTopicName := fmt.Sprintf("grim-%v-%v-repo-topic", repo.owner, repo.name)

		snsTopicARN, err := prepareSNSTopic(config.awsKey, config.awsSecret, config.awsRegion, snsTopicName)
		if err != nil {
			return fatalGrimErrorf("error creating SNS topic: %v", err)
		}

		err = prepareSubscription(config.awsKey, config.awsSecret, config.awsRegion, snsTopicARN, i.queue.ARN)
		if err != nil {
			return fatalGrimErrorf("error subscribing Grim queue %q to SNS topic %q: %v", i.queue.ARN, snsTopicARN, err)
		}
		err = prepareAmazonSNSService(localConfig.gitHubToken, repo.owner, repo.name, snsTopicARN, config.awsKey, config.awsSecret, config.awsRegion)
		if err != nil {
			return fatalGrimErrorf("error creating configuring GitHub AmazonSNS service: %v", err)
		}
		topicARNs = append(topicARNs, snsTopicARN)
	}

	err = setPolicy(config.awsKey, config.awsSecret, config.awsRegion, i.queue.ARN, i.queue.URL, topicARNs)
	if err != nil {
		return fatalGrimErrorf("error setting policy for Grim queue %q with topics %v: %v", i.queue.ARN, topicARNs, err)
	}

	return nil
}

// BuildNextInGrimQueue creates or reuses an SQS queue as a source of work.
func (i *Instance) BuildNextInGrimQueue() error {
	if err := i.checkGrimQueue(); err != nil {
		return err
	}

	configRoot := getEffectiveConfigRoot(i.configRoot)

	globalConfig, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return grimErrorf("error while reading config: %v", err)
	}

	message, err := getNextMessage(globalConfig.awsKey, globalConfig.awsSecret, globalConfig.awsRegion, i.queue.URL)
	if err != nil {
		return grimErrorf("error retrieving message from Grim queue %q: %v", i.queue.URL, err)
	}

	if message != "" {
		hook, err := extractHookEvent(message)
		if err != nil {
			return grimErrorf("error extracting hook from message: %v", err)
		}

		if !(hook.eventName == "push" || hook.eventName == "pull_request" && (hook.action == "opened" || hook.action == "reopened" || hook.action == "synchronize")) {
			return nil
		}

		if hook.eventName == "pull_request" {
			sha, err := pollForMergeCommitSha(globalConfig.gitHubToken, hook.owner, hook.repo, hook.prNumber)
			if err != nil {
				return grimErrorf("error getting merge commit sha: %v", err)
			} else if sha == "" {
				return grimErrorf("error getting merge commit sha: field empty")
			}
			hook.ref = sha
		}

		localConfig, err := getEffectiveConfig(configRoot, hook.owner, hook.repo)
		if err != nil {
			return grimErrorf("error while reading config: %v", err)
		}

		return buildForHook(configRoot, localConfig, *hook)
	}

	return nil
}

// BuildRef builds a git ref immediately.
func (i *Instance) BuildRef(owner, repo, ref string) error {
	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveConfig(configRoot, owner, repo)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	return buildForHook(configRoot, config, hookEvent{
		owner: owner,
		repo:  repo,
		ref:   ref,
	})
}

func buildForHook(configRoot string, config *effectiveConfig, hook hookEvent) error {
	extraEnv := hook.env()

	// TODO: do something with the err
	notifyPending(config, hook)

	result, err := build(configRoot, config.workspaceRoot, config.pathToCloneIn, hook.owner, hook.repo, extraEnv)
	if err != nil {
		notifyError(config, hook)
		return fatalGrimErrorf("error during %v: %v", describeHook(hook), err)
	}

	var notifyError error
	if result.ExitCode == 0 {
		notifyError = notifySuccess(config, hook)
	} else {
		notifyError = notifyFailure(config, hook)
	}

	err = appendResult(config.resultRoot, hook.owner, hook.repo, *result)
	if err != nil {
		return fatalGrimErrorf("error while storing result: %v", err)
	}

	return notifyError
}

func describeHook(hook hookEvent) string {
	return fmt.Sprintf("build of %v/%v initiated by a %q to %q by %q", hook.owner, hook.repo, hook.eventName, hook.target, hook.userName)
}

func notifyPending(config *effectiveConfig, hook hookEvent) error {
	return notify(config, hook, RSPending, fmt.Sprintf("Starting %v", describeHook(hook)), ColorYellow)
}

func notifyError(config *effectiveConfig, hook hookEvent) error {
	return notify(config, hook, RSError, fmt.Sprintf("Error during %v", describeHook(hook)), ColorGray)
}

func notifyFailure(config *effectiveConfig, hook hookEvent) error {
	return notify(config, hook, RSFailure, fmt.Sprintf("Failure during %v", describeHook(hook)), ColorRed)
}

func notifySuccess(config *effectiveConfig, hook hookEvent) error {
	return notify(config, hook, RSSuccess, fmt.Sprintf("Success after %v", describeHook(hook)), ColorGreen)
}

func notify(config *effectiveConfig, hook hookEvent, state refStatus, message string, color messageColor) error {
	if hook.eventName != "push" && hook.eventName != "pull_request" {
		return nil
	}

	//add grimServerID/grimQueueName to hipchat message for test
	message += "ServerID:" + config.grimServerID
	fmt.Printf("The updated message is:", message)

	ghErr := setRefStatus(config.gitHubToken, hook.owner, hook.repo, hook.statusRef, state, "", message)

	if config.hipChatToken != "" && config.hipChatRoom != "" {
		err := sendMessageToRoom(config.hipChatToken, config.hipChatRoom, "Grim", message, color)
		if err != nil {
			return err
		}
	}

	return ghErr
}

func (i *Instance) checkGrimQueue() error {
	if i.queue == nil {
		return fatalGrimErrorf("the Grim queue must be prepared first")
	}

	return nil
}
<commit_msg>grim local test<commit_after>package grim

// Copyright 2015 MediaMath <http://www.mediamath.com>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

import "fmt"

// Instance models the state of a configured Grim instance.
type Instance struct {
	configRoot *string
	queue      *sqsQueue
}

// SetConfigRoot sets the base path of the configuration directory and clears any previously read config values from memory.
func (i *Instance) SetConfigRoot(path string) {
	i.configRoot = &path
	i.queue = nil
}

// PrepareGrimQueue creates or reuses the Amazon SQS queue named in the config.
func (i *Instance) PrepareGrimQueue() error {
	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	queue, err := prepareSQSQueue(config.awsKey, config.awsSecret, config.awsRegion, config.grimQueueName)
	if err != nil {
		return fatalGrimErrorf("error preparing queue: %v", err)
	}

	i.queue = queue

	return nil
}

// PrepareRepos discovers all repos that are configured then sets up SNS and GitHub.
// It is an error to call this without calling PrepareGrimQueue first.
func (i *Instance) PrepareRepos() error {
	if err := i.checkGrimQueue(); err != nil {
		return err
	}

	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	repos := getAllConfiguredRepos(configRoot)

	var topicARNs []string
	for _, repo := range repos {
		localConfig, err := getEffectiveConfig(configRoot, repo.owner, repo.name)
		if err != nil {
			return fatalGrimErrorf("error while reading local config: %v", err)
		}

		snsTopicName := fmt.Sprintf("grim-%v-%v-repo-topic", repo.owner, repo.name)

		snsTopicARN, err := prepareSNSTopic(config.awsKey, config.awsSecret, config.awsRegion, snsTopicName)
		if err != nil {
			return fatalGrimErrorf("error creating SNS topic: %v", err)
		}

		err = prepareSubscription(config.awsKey, config.awsSecret, config.awsRegion, snsTopicARN, i.queue.ARN)
		if err != nil {
			return fatalGrimErrorf("error subscribing Grim queue %q to SNS topic %q: %v", i.queue.ARN, snsTopicARN, err)
		}
		err = prepareAmazonSNSService(localConfig.gitHubToken, repo.owner, repo.name, snsTopicARN, config.awsKey, config.awsSecret, config.awsRegion)
		if err != nil {
			return fatalGrimErrorf("error creating configuring GitHub AmazonSNS service: %v", err)
		}
		topicARNs = append(topicARNs, snsTopicARN)
	}

	err = setPolicy(config.awsKey, config.awsSecret, config.awsRegion, i.queue.ARN, i.queue.URL, topicARNs)
	if err != nil {
		return fatalGrimErrorf("error setting policy for Grim queue %q with topics %v: %v", i.queue.ARN, topicARNs, err)
	}

	return nil
}

// BuildNextInGrimQueue creates or reuses an SQS queue as a source of work.
func (i *Instance) BuildNextInGrimQueue() error {
	if err := i.checkGrimQueue(); err != nil {
		return err
	}

	configRoot := getEffectiveConfigRoot(i.configRoot)

	globalConfig, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return grimErrorf("error while reading config: %v", err)
	}

	message, err := getNextMessage(globalConfig.awsKey, globalConfig.awsSecret, globalConfig.awsRegion, i.queue.URL)
	if err != nil {
		return grimErrorf("error retrieving message from Grim queue %q: %v", i.queue.URL, err)
	}

	if message != "" {
		hook, err := extractHookEvent(message)
		if err != nil {
			return grimErrorf("error extracting hook from message: %v", err)
		}

		if !(hook.eventName == "push" || hook.eventName == "pull_request" && (hook.action == "opened" || hook.action == "reopened" || hook.action == "synchronize")) {
			return nil
		}

		if hook.eventName == "pull_request" {
			sha, err := pollForMergeCommitSha(globalConfig.gitHubToken, hook.owner, hook.repo, hook.prNumber)
			if err != nil {
				return grimErrorf("error getting merge commit sha: %v", err)
			} else if sha == "" {
				return grimErrorf("error getting merge commit sha: field empty")
			}
			hook.ref = sha
		}

		localConfig, err := getEffectiveConfig(configRoot, hook.owner, hook.repo)
		if err != nil {
			return grimErrorf("error while reading config: %v", err)
		}

		return buildForHook(configRoot, localConfig, *hook)
	}

	return nil
}

// BuildRef builds a git ref immediately.
func (i *Instance) BuildRef(owner, repo, ref string) error {
	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveConfig(configRoot, owner, repo)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	return buildForHook(configRoot, config, hookEvent{
		owner: owner,
		repo:  repo,
		ref:   ref,
	})
}

func buildForHook(configRoot string, config *effectiveConfig, hook hookEvent) error {
	extraEnv := hook.env()

	// TODO: do something with the err
	notifyPending(config, hook)

	result, err := build(configRoot, config.workspaceRoot, config.pathToCloneIn, hook.owner, hook.repo, extraEnv)
	if err != nil {
		notifyError(config, hook)
		return fatalGrimErrorf("error during %v: %v", describeHook(hook), err)
	}

	var notifyError error
	if result.ExitCode == 0 {
		notifyError = notifySuccess(config, hook)
	} else {
		notifyError = notifyFailure(config, hook)
	}

	err = appendResult(config.resultRoot, hook.owner, hook.repo, *result)
	if err != nil {
		return fatalGrimErrorf("error while storing result: %v", err)
	}

	return notifyError
}

func describeHook(hook hookEvent) string {
	return fmt.Sprintf("build of %v/%v initiated by a %q to %q by %q", hook.owner, hook.repo, hook.eventName, hook.target, hook.userName)
}

func notifyPending(config *effectiveConfig, hook hookEvent) error {
	return notify(config, hook, RSPending, fmt.Sprintf("Starting %v", describeHook(hook)), ColorYellow)
}

func notifyError(config *effectiveConfig, hook hookEvent) error {
	return notify(config, hook, RSError, fmt.Sprintf("Error during %v", describeHook(hook)), ColorGray)
}

func notifyFailure(config *effectiveConfig, hook hookEvent) error {
	return notify(config, hook, RSFailure, fmt.Sprintf("Failure during %v", describeHook(hook)), ColorRed)
}

func notifySuccess(config *effectiveConfig, hook hookEvent) error {
	return notify(config, hook, RSSuccess, fmt.Sprintf("Success after %v", describeHook(hook)), ColorGreen)
}

func notify(config *effectiveConfig, hook hookEvent, state refStatus, message string, color messageColor) error {
	if hook.eventName != "push" && hook.eventName != "pull_request" {
		return nil
	}

	//add grimServerID/grimQueueName to hipchat message
	message += ":" + "ServerID/QueueName - " + config.grimServerID

	ghErr := setRefStatus(config.gitHubToken, hook.owner, hook.repo, hook.statusRef, state, "", message)

	if config.hipChatToken != "" && config.hipChatRoom != "" {
		err := sendMessageToRoom(config.hipChatToken, config.hipChatRoom, "Grim", message, color)
		if err != nil {
			return err
		}
	}

	return ghErr
}

func (i *Instance) checkGrimQueue() error {
	if i.queue == nil {
		return fatalGrimErrorf("the Grim queue must be prepared first")
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	pb "github.com/itshouldntdothis/swagger-http-grpc"

	"go.uber.org/ratelimit"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/reflection"
	"gopkg.in/resty.v0"
)

// GrcpServerOptions implements processing functions
type GrcpServerOptions struct {
	Addr        string
	Server      *grpc.Server
	RateLimited bool
	RateLimiter ratelimit.Limiter
	RestyClient *resty.Client
	UserAgent   string
}

// NewGrcpOptions returns a initialized object
func NewGrcpOptions() *GrcpServerOptions {
	Transport := &http.Transport{
		MaxIdleConns: 200,
		DialContext: (&net.Dialer{
			Timeout:   60 * time.Second,
			KeepAlive: 60 * time.Second,
		}).DialContext,
		IdleConnTimeout:       60 * time.Second,
		TLSHandshakeTimeout:   20 * time.Second,
		ResponseHeaderTimeout: 20 * time.Second,
		ExpectContinueTimeout: 0,
		MaxIdleConnsPerHost:   40,
	}

	port := "50051"
	envGrcpPort := os.Getenv("SW_GRPC_PORT")
	if len(envGrcpPort) != 0 {
		port = envGrcpPort
	}
	Addr := ":" + port
	log.Printf("SET GRPC server port to %v", Addr)

	UserAgent := os.Getenv("SW_USER_AGENT")
	if len(UserAgent) == 0 {
		log.Fatal("Default useragent env `SW_USER_AGENT` is required")
	}
	log.Printf("SET GRPC HTTP WORKER User-Agent to `%v`", UserAgent)

	RateLimit := 700
	var RateLimiter ratelimit.Limiter
	var RateLimited bool
	envRateLimit := os.Getenv("SW_REQUEST_LIMIT")
	if len(envRateLimit) != 0 {
		parsedRequestLimit, err := strconv.Atoi(envRateLimit)
		if err != nil {
			log.Fatalf("Unable to covert ENV 'SW_REQUEST_LIMIT' of %v to int : %v", envRateLimit, err)
		}
		RateLimit = parsedRequestLimit
	}

	if RateLimit == 0 {
		RateLimited = false
		log.Println("Rate Limit of 0 is not recommended")
	} else {
		RateLimited = true
		log.Printf("Rate Limit set to %v", RateLimit)
		RateLimiter = ratelimit.New(RateLimit)
	}

	return &GrcpServerOptions{
		Server:      grpc.NewServer(),
		Addr:        Addr,
		RestyClient: resty.New().SetHeader("Via", via).SetTransport(Transport),
		RateLimited: RateLimited,
		RateLimiter: RateLimiter,
		UserAgent:   UserAgent,
	}
}

func (o *GrcpServerOptions) start() {
	lis, err := net.Listen("tcp", o.Addr)
	if err != nil {
		log.Fatalf("Failed to listen: %v", err)
	}
	pb.RegisterWorkersServer(o.Server, o)
	// Register reflection service on gRPC server.
	reflection.Register(o.Server)

	log.Println("Starting grpc server")
	if err := o.Server.Serve(lis); err != nil {
		log.Fatalf("Failed to serve grpc server: %v", err)
	}
}

func (o *GrcpServerOptions) close() {
	log.Println("Closing down grpc server")
	o.Server.GracefulStop()
}

// DoRequest processes a single http request from grpc
func (o *GrcpServerOptions) DoRequest(ctx context.Context, in *pb.Request) (*pb.Response, error) {
	method := in.GetMethod()
	if method != "GET" && method != "POST" && method != "PUT" && method != "DELETE" {
		return nil, errors.New("We don't support " + method + " method at this time")
	}

	restyReq := o.RestyClient.R()
	if len(in.GetHeaders()) > 0 {
		restyReq = restyReq.SetHeaders(in.GetHeaders())
	}

	if restyReq.Header.Get("User-Agent") == "" {
		restyReq = restyReq.SetHeader("User-Agent", o.UserAgent)
	}

	if in.GetBody() != "" {
		restyReq = restyReq.SetBody(in.GetBody())
	}

	if o.RateLimited {
		o.RateLimiter.Take()
	}

	resp, err := restyReq.Execute(method, in.GetUrl())
	if err != nil {
		log.Printf("Fetch Error %v", err)
		return nil, err
	}

	log.Printf("%v successful for %v took %v", method, resp.Request.URL, resp.Time())

	headers := make(map[string]*pb.Header)
	for k, v := range resp.Header() {
		headers[strings.ToLower(k)] = &pb.Header{Value: v}
	}

	return &pb.Response{
		Ok:         isOk(resp.StatusCode()),
		Url:        resp.Request.URL,
		Status:     int32(resp.StatusCode()),
		StatusText: resp.Status(),
		Body:       resp.String(),
		Headers:    headers,
	}, nil
}

func isOk(status int) bool {
	if status >= 200 && status <= 299 {
		return true
	}
	return false
}
<commit_msg>Log time delayed<commit_after>package main

import (
	"errors"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	pb "github.com/itshouldntdothis/swagger-http-grpc"

	"go.uber.org/ratelimit"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/reflection"
	"gopkg.in/resty.v0"
)

// GrcpServerOptions implements processing functions
type GrcpServerOptions struct {
	Addr        string
	Server      *grpc.Server
	RateLimited bool
	RateLimiter ratelimit.Limiter
	RestyClient *resty.Client
	UserAgent   string
}

// NewGrcpOptions returns a initialized object
func NewGrcpOptions() *GrcpServerOptions {
	Transport := &http.Transport{
		MaxIdleConns: 200,
		DialContext: (&net.Dialer{
			Timeout:   60 * time.Second,
			KeepAlive: 60 * time.Second,
		}).DialContext,
		IdleConnTimeout:       60 * time.Second,
		TLSHandshakeTimeout:   20 * time.Second,
		ResponseHeaderTimeout: 20 * time.Second,
		ExpectContinueTimeout: 0,
		MaxIdleConnsPerHost:   40,
	}

	port := "50051"
	envGrcpPort := os.Getenv("SW_GRPC_PORT")
	if len(envGrcpPort) != 0 {
		port = envGrcpPort
	}
	Addr := ":" + port
	log.Printf("SET GRPC server port to %v", Addr)

	UserAgent := os.Getenv("SW_USER_AGENT")
	if len(UserAgent) == 0 {
		log.Fatal("Default useragent env `SW_USER_AGENT` is required")
	}
	log.Printf("SET GRPC HTTP WORKER User-Agent to `%v`", UserAgent)

	RateLimit := 700
	var RateLimiter ratelimit.Limiter
	var RateLimited bool
	envRateLimit := os.Getenv("SW_REQUEST_LIMIT")
	if len(envRateLimit) != 0 {
		parsedRequestLimit, err := strconv.Atoi(envRateLimit)
		if err != nil {
			log.Fatalf("Unable to covert ENV 'SW_REQUEST_LIMIT' of %v to int : %v", envRateLimit, err)
		}
		RateLimit = parsedRequestLimit
	}

	if RateLimit == 0 {
		RateLimited = false
		log.Println("Rate Limit of 0 is not recommended")
	} else {
		RateLimited = true
		log.Printf("Rate Limit set to %v", RateLimit)
		RateLimiter = ratelimit.New(RateLimit)
	}

	return &GrcpServerOptions{
		Server:      grpc.NewServer(),
		Addr:        Addr,
		RestyClient: resty.New().SetHeader("Via", via).SetTransport(Transport),
		RateLimited: RateLimited,
		RateLimiter: RateLimiter,
		UserAgent:   UserAgent,
	}
}

func (o *GrcpServerOptions) start() {
	lis, err := net.Listen("tcp", o.Addr)
	if err != nil {
		log.Fatalf("Failed to listen: %v", err)
	}
	pb.RegisterWorkersServer(o.Server, o)
	// Register reflection service on gRPC server.
	reflection.Register(o.Server)

	log.Println("Starting grpc server")
	if err := o.Server.Serve(lis); err != nil {
		log.Fatalf("Failed to serve grpc server: %v", err)
	}
}

func (o *GrcpServerOptions) close() {
	log.Println("Closing down grpc server")
	o.Server.GracefulStop()
}

// DoRequest processes a single http request from grpc
func (o *GrcpServerOptions) DoRequest(ctx context.Context, in *pb.Request) (*pb.Response, error) {
	method := in.GetMethod()
	if method != "GET" && method != "POST" && method != "PUT" && method != "DELETE" {
		return nil, errors.New("We don't support " + method + " method at this time")
	}

	restyReq := o.RestyClient.R()
	if len(in.GetHeaders()) > 0 {
		restyReq = restyReq.SetHeaders(in.GetHeaders())
	}

	if restyReq.Header.Get("User-Agent") == "" {
		restyReq = restyReq.SetHeader("User-Agent", o.UserAgent)
	}

	if in.GetBody() != "" {
		restyReq = restyReq.SetBody(in.GetBody())
	}

	var delay time.Duration
	if o.RateLimited {
		start := time.Now()
		now := o.RateLimiter.Take()
		delay = now.Sub(start)
	}

	resp, err := restyReq.Execute(method, in.GetUrl())
	if err != nil {
		log.Printf("Fetch Error %v", err)
		return nil, err
	}

	headers := make(map[string]*pb.Header)
	for k, v := range resp.Header() {
		headers[strings.ToLower(k)] = &pb.Header{Value: v}
	}

	log.Printf("%v successful for %v took %v delayed by %v", method, resp.Request.URL, resp.Time(), delay)
	return &pb.Response{
		Ok:         isOk(resp.StatusCode()),
		Url:        resp.Request.URL,
		Status:     int32(resp.StatusCode()),
		StatusText: resp.Status(),
		Body:       resp.String(),
		Headers:    headers,
	}, nil
}

func isOk(status int) bool {
	if status >= 200 && status <= 299 {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013-2017, Jeremy Bingham (<jeremy@goiardi.gl>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"github.com/ctdk/goiardi/actor"
	"github.com/ctdk/goiardi/gerror"
	"github.com/ctdk/goiardi/util"
	"github.com/tideland/golib/logger"
	"net/http"
)

// Functions and types for HEAD responses for various endpoints

// Implements Chef RFCnnn (unassigned so far)

// Full table of what everything means will be added when the RFC is finalized

type headChecker interface {
	actor.Actor
	DoesExist(string) (bool, util.Gerror)
}

type exists func(resource string) (bool, util.Gerror)

type permChecker func(r *http.Request, resource string, obj actor.Actor) util.Gerror

// for when no perm check is actually necessary
func nilPermCheck(r *http.Request, resource string, obj actor.Actor) util.Gerror {
	return nil
}

func headResponse(w http.ResponseWriter, r *http.Request, status int) {
	logger.Debugf("HEAD response status %d for %s", status, r.URL.Path)
	w.WriteHeader(status)
	return
}

func headDefaultResponse(w http.ResponseWriter, r *http.Request) {
	logger.Debugf("HEAD default response issued for %s", r.URL.Path)
	w.WriteHeader(http.StatusOK)
}

func headChecking(w http.ResponseWriter, r *http.Request, obj actor.Actor, resource string, doesExist exists, permCheck permChecker) {
	found, err := doesExist(resource)
	if err != nil {
		headResponse(w, r, err.Status())
	}
	err = permCheck(r, resource, obj)
	if err != nil {
		headResponse(w, r, err.Status())
	}
	if !found {
		headResponse(w, r, http.StatusNotFound)
	}
	headResponse(w, r, http.StatusOK)
}

func headForbidden() util.Gerror {
	err := gerror.StatusError("forbidden", http.StatusForbidden)
	return err
}
<commit_msg>Chef RFC090 has been accepted and numbered<commit_after>/*
 * Copyright (c) 2013-2017, Jeremy Bingham (<jeremy@goiardi.gl>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"github.com/ctdk/goiardi/actor"
	"github.com/ctdk/goiardi/gerror"
	"github.com/ctdk/goiardi/util"
	"github.com/tideland/golib/logger"
	"net/http"
)

// Functions and types for HEAD responses for various endpoints
//
// Implements Chef RFC090 -
// (https://github.com/chef/chef-rfc/blob/master/rfc090-server-http-head.md)
//
// From the RFC:
//
// Code 	Reason
// ---- 	------
// 200 		object exists
// 401 		request signature is invalid
// 403 		requestor does not have READ on the object
// 404 		object does not exist
//
// Obviously, the 403 bit will be a little bit different for goiardi 0.x.y vs.
// the 1.0.0 branch because of the different permissions scheme.
//
// HEAD should be present everywhere GET is, even if the HEAD request isn't
// particularly meaningful.

type headChecker interface {
	actor.Actor
	DoesExist(string) (bool, util.Gerror)
}

type exists func(resource string) (bool, util.Gerror)

type permChecker func(r *http.Request, resource string, obj actor.Actor) util.Gerror

// for when no perm check is actually necessary
func nilPermCheck(r *http.Request, resource string, obj actor.Actor) util.Gerror {
	return nil
}

func headResponse(w http.ResponseWriter, r *http.Request, status int) {
	logger.Debugf("HEAD response status %d for %s", status, r.URL.Path)
	w.WriteHeader(status)
	return
}

func headDefaultResponse(w http.ResponseWriter, r *http.Request) {
	logger.Debugf("HEAD default response issued for %s", r.URL.Path)
	w.WriteHeader(http.StatusOK)
}

func headChecking(w http.ResponseWriter, r *http.Request, obj actor.Actor, resource string, doesExist exists, permCheck permChecker) {
	found, err := doesExist(resource)
	if err != nil {
		headResponse(w, r, err.Status())
	}
	err = permCheck(r, resource, obj)
	if err != nil {
		headResponse(w, r, err.Status())
	}
	if !found {
		headResponse(w, r, http.StatusNotFound)
	}
	headResponse(w, r, http.StatusOK)
}

func headForbidden() util.Gerror {
	err := gerror.StatusError("forbidden", http.StatusForbidden)
	return err
}
<|endoftext|>


<commit_before>package goose

import (
	resty "github.com/go-resty/resty/v2"
	"github.com/pkg/errors"
)

type HtmlRequester interface {
	fetchHTML(string) (string, error)
}

// Crawler can fetch the target HTML page
type htmlrequester struct {
	config Configuration
}

// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
func NewHtmlRequester(config Configuration) HtmlRequester {
	return htmlrequester{
		config: config,
	}
}

func (hr htmlrequester) fetchHTML(url string) (string, error) {
	client := resty.New()
	client.SetTimeout(hr.config.timeout)
	resp, err := client.R().
		SetHeader("Content-Type", "application/json").
		SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30").
		Get(url)

	if err != nil {
		return "", errors.Wrap(err, "could not perform request on "+url)
	}
	if resp.IsError() {
		return "", &badRequest{Message: "could not perform request with " + url + " status code " + string(resp.StatusCode())}
	}
	return resp.String(), nil
}

type badRequest struct {
	Message string `json:"message,omitempty"`
}

func (BadRequest *badRequest) Error() string {
	return "Required request fields are not filled"
}
<commit_msg>Why use application/json to fetch HTML?<commit_after>package goose

import (
	resty "github.com/go-resty/resty/v2"
	"github.com/pkg/errors"
)

type HtmlRequester interface {
	fetchHTML(string) (string, error)
}

// Crawler can fetch the target HTML page
type htmlrequester struct {
	config Configuration
}

// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
func NewHtmlRequester(config Configuration) HtmlRequester {
	return htmlrequester{
		config: config,
	}
}

func (hr htmlrequester) fetchHTML(url string) (string, error) {
	client := resty.New()
	client.SetTimeout(hr.config.timeout)
	resp, err := client.R().
		SetHeader("Content-Type", "text/html").
		SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30").
		Get(url)

	if err != nil {
		return "", errors.Wrap(err, "could not perform request on "+url)
	}
	if resp.IsError() {
		return "", &badRequest{Message: "could not perform request with " + url + " status code " + string(resp.StatusCode())}
	}
	return resp.String(), nil
}

type badRequest struct {
	Message string `json:"message,omitempty"`
}

func (BadRequest *badRequest) Error() string {
	return "Required request fields are not filled"
}
<|endoftext|>