<commit_before>package configo

import (
	"github.com/shafreeck/configo/rule"
	"github.com/shafreeck/toml"
	"github.com/shafreeck/toml/ast"

	"fmt"
	goast "go/ast"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unicode"
)

const (
	fieldTagName = "cfg"
)

func init() {
	toml.SetValue = fieldValidate
}

func fieldValidate(field string, rv reflect.Value, av ast.Value, tag *toml.CfgTag) error {
	if tag == nil {
		return nil
	}
	val, ok := av.(*ast.String)
	if tag.Check != "" && ok {
		return validate(field, val.Value, tag.Check)
	}
	return nil
}
func isEmptyValue(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.String, reflect.Array:
		return v.Len() == 0
	case reflect.Map, reflect.Slice:
		return v.Len() == 0 || v.IsNil()
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}
	return reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface())
}

func extractTag(tag string) *toml.CfgTag {
	tags := strings.SplitN(tag, ";", 4)
	cfg := &toml.CfgTag{}
	switch c := len(tags); c {
	case 1:
		cfg.Name = strings.TrimSpace(tags[0])
	case 2:
		cfg.Name = strings.TrimSpace(tags[0])
		cfg.Value = strings.TrimSpace(tags[1])
	case 3:
		cfg.Name = strings.TrimSpace(tags[0])
		cfg.Value = strings.TrimSpace(tags[1])
		cfg.Check = strings.TrimSpace(tags[2])
	case 4:
		cfg.Name = strings.TrimSpace(tags[0])
		cfg.Value = strings.TrimSpace(tags[1])
		cfg.Check = strings.TrimSpace(tags[2])
		cfg.Description = strings.TrimSpace(tags[3])
	default:
		return nil
	}
	return cfg
}

func validate(key, value string, check string) error {
	r := rule.Rule(check)
	vlds, err := r.Parse()
	if err != nil {
		return err
	}
	for _, vld := range vlds {
		if err := vld.Validate(value); err != nil {
			return fmt.Errorf("validate %s failed, %s does not match rule %q, reason: %v", key, value, check, err)
		}
	}
	return nil
}

//parse a toml array
func unmarshalArray(key, value string, v interface{}) error {
	//construct a valid toml array
	data := key + " = " + value
	if err := toml.Unmarshal([]byte(data), v); err != nil {
		return err
	}
	return nil
}

func applyDefaultValue(fv reflect.Value, ft reflect.StructField, rv reflect.Value, ignoreRequired bool) (err error) {
	tag := extractTag(ft.Tag.Get(fieldTagName))

	//Default value is not supported
	if tag.Value == "required" {
		if ignoreRequired {
			return nil
		}
		return fmt.Errorf("value of %q is required in %v", ft.Name, rv.Type())
	}

	//No default value supplied
	if tag.Value == "" {
		return nil
	}

	//Validate the default value
	//reflect.Slice will be validated by unmarshalArray
	if tag.Check != "" && fv.Kind() != reflect.Slice {
		if err := validate(ft.Name, tag.Value, tag.Check); err != nil {
			return err
		}
	}

	//Set the default value
	switch fv.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16,
		reflect.Int32, reflect.Int64:
		var v int64
		if v, err = strconv.ParseInt(tag.Value, 10, 64); err != nil {
			if fv.Kind() == reflect.Int64 {
				//try to parse a time.Duration
				if d, err := time.ParseDuration(tag.Value); err == nil {
					fv.SetInt(int64(d))
					return nil
				}
			}
			return err
		}
		fv.SetInt(v)
	case reflect.Uint, reflect.Uint8, reflect.Uint16,
		reflect.Uint32, reflect.Uint64:
		var v uint64
		if v, err = strconv.ParseUint(tag.Value, 10, 64); err != nil {
			return err
		}
		fv.SetUint(v)
	case reflect.Float32, reflect.Float64:
		var v float64
		if v, err = strconv.ParseFloat(tag.Value, 64); err != nil {
			return err
		}
		fv.SetFloat(v)
	case reflect.Bool:
		var v bool
		if v, err = strconv.ParseBool(tag.Value); err != nil {
			return err
		}
		fv.SetBool(v)
	case reflect.String:
		fv.SetString(tag.Value)
	case reflect.Slice:
		v := rv.Addr().Interface()
		if err := unmarshalArray(ft.Name, tag.Value, v); err != nil {
			return err
		}
	default:
		return fmt.Errorf("set default value of type %s is not supported yet", ft.Type)
	}
	return nil
}

//Notice toCamelCase is copied from github.com/naoina/toml
// toCamelCase returns a copy of the string s with all Unicode letters mapped to their camel case.
// It will convert to upper case previous letter of '_' and first letter, and remove letter of '_'.
func toUnderscore(s string) string {
	if s == "" {
		return ""
	}
	result := make([]rune, 0, len(s))

	result = append(result, unicode.ToLower(rune(s[0])))
	for _, r := range s[1:] {
		if unicode.ToUpper(r) == r {
			result = append(result, '_', unicode.ToLower(r))
			continue
		}
		result = append(result, r)
	}
	return string(result)
}

func findField(t *ast.Table, field reflect.StructField) (interface{}, bool) {
	if t == nil {
		return nil, false
	}
	tag := extractTag(field.Tag.Get(fieldTagName))
	if tag != nil && tag.Name != "" {
		if f, found := t.Fields[tag.Name]; found {
			return f, found
		}
		return nil, false
	}

	name := field.Name
	for _, n := range []string{name, strings.ToLower(name), toUnderscore(name)} {
		if f, found := t.Fields[n]; found {
			return f, found
		}
	}
	return nil, false
}

func applyDefault(t *ast.Table, rv reflect.Value, ignoreRequired bool) error {
	for rv.Kind() == reflect.Ptr {
		rv = rv.Elem()
	}

	rt := rv.Type()

	if kind := rt.Kind(); kind == reflect.Struct {
		for i := 0; i < rt.NumField(); i++ {
			ft := rt.Field(i)
			fv := rv.Field(i)
			if !goast.IsExported(ft.Name) {
				continue
			}
			for fv.Kind() == reflect.Ptr {
				fv = fv.Elem()
			}
			if fv.Kind() == reflect.Struct {
				var subt *ast.Table
				var ok bool
				if f, found := findField(t, ft); found {
					subt, ok = f.(*ast.Table)
					//Assgin t back to subt
					//This is becuase the reflect.Struct is emmbed
					// type D struct {
					//    time.Duration
					// }
					// D is a struct , but there is no sub table in conf
					if !ok {
						subt = t
					}
				}

				if err := applyDefault(subt, fv, ignoreRequired); err != nil {
					return err
				}
				continue
			}
			if fv.IsValid() && isEmptyValue(fv) {
				if _, found := findField(t, ft); !found {
					if err := applyDefaultValue(fv, ft, rv, ignoreRequired); err != nil {
						return err
					}
				}
			}
		}
	}

	return nil
}

//Unmarshal data into struct v, v shoud be a pointer to struct
func Unmarshal(data []byte, v interface{}) error {
	table, err := toml.Parse(data)
	if err != nil {
		return err
	}

	if err := toml.UnmarshalTable(table, v); err != nil {
		return err
	}

	if err := applyDefault(table, reflect.ValueOf(v), false); err != nil {
		return err
	}
	return nil
}

//Marshal v to configuration in toml format
func Marshal(v interface{}) ([]byte, error) {
	rv := reflect.ValueOf(v)
	for rv.Kind() == reflect.Ptr {
		rv = rv.Elem()
	}
	pv := reflect.New(rv.Type())
	pv.Elem().Set(rv)

	if err := applyDefault(nil, pv, true); err != nil {
		return nil, err
	}
	return toml.Marshal(pv.Interface())
}

//Patch the base using the value from v, the new bytes returned
//combines the base's value and v's default value
func Patch(base []byte, v interface{}) ([]byte, error) {
	//Clone struct v, v shoud not be modified
	rv := reflect.ValueOf(v)
	for rv.Kind() == reflect.Ptr {
		rv = rv.Elem()
	}
	pv := reflect.New(rv.Type())
	pv.Elem().Set(rv)

	nv := pv.Interface()

	//unmarshal base
	table, err := toml.Parse(base)
	if err != nil {
		return nil, err
	}

	if err := toml.UnmarshalTable(table, nv); err != nil {
		return nil, err
	}

	return Marshal(nv)
}
<commit_msg>Fix #3<commit_after>package configo

import (
	"github.com/shafreeck/configo/rule"
	"github.com/shafreeck/toml"
	"github.com/shafreeck/toml/ast"

	"fmt"
	goast "go/ast"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unicode"
)

const (
	fieldTagName = "cfg"
)

func init() {
	toml.SetValue = fieldValidate
}

func fieldValidate(field string, rv reflect.Value, av ast.Value, tag *toml.CfgTag) error {
	if tag == nil {
		return nil
	}
	val, ok := av.(*ast.String)
	if tag.Check != "" && ok {
		return validate(field, val.Value, tag.Check)
	}
	return nil
}
func isEmptyValue(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.String, reflect.Array:
		return v.Len() == 0
	case reflect.Map, reflect.Slice:
		return v.Len() == 0 || v.IsNil()
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}
	return reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface())
}

func extractTag(tag string) *toml.CfgTag {
	tags := strings.SplitN(tag, ";", 4)
	cfg := &toml.CfgTag{}
	switch c := len(tags); c {
	case 1:
		cfg.Name = strings.TrimSpace(tags[0])
	case 2:
		cfg.Name = strings.TrimSpace(tags[0])
		cfg.Value = strings.TrimSpace(tags[1])
	case 3:
		cfg.Name = strings.TrimSpace(tags[0])
		cfg.Value = strings.TrimSpace(tags[1])
		cfg.Check = strings.TrimSpace(tags[2])
	case 4:
		cfg.Name = strings.TrimSpace(tags[0])
		cfg.Value = strings.TrimSpace(tags[1])
		cfg.Check = strings.TrimSpace(tags[2])
		cfg.Description = strings.TrimSpace(tags[3])
	default:
		return nil
	}
	return cfg
}

func validate(key, value string, check string) error {
	r := rule.Rule(check)
	vlds, err := r.Parse()
	if err != nil {
		return err
	}
	for _, vld := range vlds {
		if err := vld.Validate(value); err != nil {
			return fmt.Errorf("validate %s failed, %s does not match rule %q, reason: %v", key, value, check, err)
		}
	}
	return nil
}

//parse a toml array
func unmarshalArray(key, value string, v interface{}) error {
	//construct a valid toml array
	data := key + " = " + value
	if err := toml.Unmarshal([]byte(data), v); err != nil {
		return err
	}
	return nil
}

func applyDefaultValue(fv reflect.Value, ft reflect.StructField, rv reflect.Value, ignoreRequired bool) (err error) {
	tag := extractTag(ft.Tag.Get(fieldTagName))

	//Default value is not supported
	if tag.Value == "required" {
		if ignoreRequired {
			return nil
		}
		return fmt.Errorf("value of %q is required in %v", ft.Name, rv.Type())
	}

	//No default value supplied
	if tag.Value == "" {
		return nil
	}

	//Validate the default value
	//reflect.Slice will be validated by unmarshalArray
	if tag.Check != "" && fv.Kind() != reflect.Slice {
		if err := validate(ft.Name, tag.Value, tag.Check); err != nil {
			return err
		}
	}

	//Set the default value
	switch fv.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16,
		reflect.Int32, reflect.Int64:
		var v int64
		if v, err = strconv.ParseInt(tag.Value, 10, 64); err != nil {
			if fv.Kind() == reflect.Int64 {
				//try to parse a time.Duration
				if d, err := time.ParseDuration(tag.Value); err == nil {
					fv.SetInt(int64(d))
					return nil
				}
			}
			return err
		}
		fv.SetInt(v)
	case reflect.Uint, reflect.Uint8, reflect.Uint16,
		reflect.Uint32, reflect.Uint64:
		var v uint64
		if v, err = strconv.ParseUint(tag.Value, 10, 64); err != nil {
			return err
		}
		fv.SetUint(v)
	case reflect.Float32, reflect.Float64:
		var v float64
		if v, err = strconv.ParseFloat(tag.Value, 64); err != nil {
			return err
		}
		fv.SetFloat(v)
	case reflect.Bool:
		var v bool
		if v, err = strconv.ParseBool(tag.Value); err != nil {
			return err
		}
		fv.SetBool(v)
	case reflect.String:
		fv.SetString(tag.Value)
	case reflect.Slice:
		v := rv.Addr().Interface()
		if err := unmarshalArray(ft.Name, tag.Value, v); err != nil {
			return err
		}
	default:
		return fmt.Errorf("set default value of type %s is not supported yet", ft.Type)
	}
	return nil
}

//Notice toCamelCase is copied from github.com/naoina/toml
// toCamelCase returns a copy of the string s with all Unicode letters mapped to their camel case.
// It will convert to upper case previous letter of '_' and first letter, and remove letter of '_'.
func toUnderscore(s string) string {
	if s == "" {
		return ""
	}
	result := make([]rune, 0, len(s))

	result = append(result, unicode.ToLower(rune(s[0])))
	for _, r := range s[1:] {
		if unicode.ToUpper(r) == r {
			result = append(result, '_', unicode.ToLower(r))
			continue
		}
		result = append(result, r)
	}
	return string(result)
}

func findField(t *ast.Table, field reflect.StructField) (interface{}, bool) {
	if t == nil {
		return nil, false
	}
	tag := extractTag(field.Tag.Get(fieldTagName))
	if tag != nil && tag.Name != "" {
		if f, found := t.Fields[tag.Name]; found {
			return f, found
		}
		return nil, false
	}

	name := field.Name
	for _, n := range []string{name, strings.ToLower(name), toUnderscore(name)} {
		if f, found := t.Fields[n]; found {
			return f, found
		}
	}
	return nil, false
}

func applyDefault(t *ast.Table, rv reflect.Value, ignoreRequired bool) error {
	for rv.Kind() == reflect.Ptr {
		rv = rv.Elem()
	}

	rt := rv.Type()

	if kind := rt.Kind(); kind == reflect.Struct {
		for i := 0; i < rt.NumField(); i++ {
			ft := rt.Field(i)
			fv := rv.Field(i)
			if !goast.IsExported(ft.Name) {
				continue
			}
			for fv.Kind() == reflect.Ptr {
				fv = fv.Elem()
			}
			if fv.Kind() == reflect.Struct {
				var subt *ast.Table
				var ok bool
				if f, found := findField(t, ft); found {
					subt, ok = f.(*ast.Table)
					//Assgin t back to subt
					//This is becuase the reflect.Struct is emmbed
					// type D struct {
					//    time.Duration
					// }
					// D is a struct , but there is no sub table in conf
					if !ok {
						subt = t
					}
				}

				if err := applyDefault(subt, fv, ignoreRequired); err != nil {
					return err
				}
				continue
			}

			//Maybe array of table
			if fv.IsValid() && !isEmptyValue(fv) && fv.Kind() == reflect.Slice {
				if arrtable, found := findField(t, ft); found {
					arrtable, ok := arrtable.([]*ast.Table)
					if ok {
						for i = 0; i < fv.Len(); i++ {
							ev := fv.Index(i)
							st := arrtable[i]
							if err := applyDefault(st, ev, ignoreRequired); err != nil {
								return err
							}
						}
					}
				}
			}

			if fv.IsValid() && isEmptyValue(fv) {
				if _, found := findField(t, ft); !found {
					if err := applyDefaultValue(fv, ft, rv, ignoreRequired); err != nil {
						return err
					}
				}
			}
		}
	}

	return nil
}

//Unmarshal data into struct v, v shoud be a pointer to struct
func Unmarshal(data []byte, v interface{}) error {
	table, err := toml.Parse(data)
	if err != nil {
		return err
	}

	if err := toml.UnmarshalTable(table, v); err != nil {
		return err
	}

	if err := applyDefault(table, reflect.ValueOf(v), false); err != nil {
		return err
	}
	return nil
}

//Marshal v to configuration in toml format
func Marshal(v interface{}) ([]byte, error) {
	rv := reflect.ValueOf(v)
	for rv.Kind() == reflect.Ptr {
		rv = rv.Elem()
	}
	pv := reflect.New(rv.Type())
	pv.Elem().Set(rv)

	if err := applyDefault(nil, pv, true); err != nil {
		return nil, err
	}
	return toml.Marshal(pv.Interface())
}

//Patch the base using the value from v, the new bytes returned
//combines the base's value and v's default value
func Patch(base []byte, v interface{}) ([]byte, error) {
	//Clone struct v, v shoud not be modified
	rv := reflect.ValueOf(v)
	for rv.Kind() == reflect.Ptr {
		rv = rv.Elem()
	}
	pv := reflect.New(rv.Type())
	pv.Elem().Set(rv)

	nv := pv.Interface()

	//unmarshal base
	table, err := toml.Parse(base)
	if err != nil {
		return nil, err
	}

	if err := toml.UnmarshalTable(table, nv); err != nil {
		return nil, err
	}

	return Marshal(nv)
}
<|endoftext|>


<commit_before>// Package hamming contains functions for calculating the hamming edit distance between
// different bytes and byte arrays
package hamming

// Distance calculates the hamming edit distance between the 2 arrays (a & b)
// both a and b must be of the same length or a panic will ensue
func Distance(a []byte, b []byte) int {
	if len(a) != len(b) {
		// TODO this should reall just use a multiple return and include an Error object
		// even if its going to panic, it should really be pancing with an Error object
		// not a string
		panic("cannot compute a hamming distance for arrays of differing length")
	}

	distance := 0
	for i := range a {
		distance += byteDistance(a[i], b[i])
	}
	return distance
}

// calculate the hamming distance of two bytes by xoring them and then counting
// how many bits are set to 1 in the resultant value
func byteDistance(a byte, b byte) int {
	x := a ^ b
	distance := 0
	// for each bit in a byte, shift that bit into the right most position
	// and then 0 out all other positions, yielding a 1 if that bit is 1, and a 0 otherwise
	for i := 0; i < 8; i++ {
		distance += int(x >> uint(i) & 1)
	}
	return distance
}
<commit_msg>typo fixes<commit_after>// Package hamming contains functions for calculating the hamming edit distance between
// different bytes and byte arrays
package hamming

// Distance calculates the hamming edit distance between the 2 arrays (a & b)
// both a and b must be of the same length or a panic will ensue
func Distance(a []byte, b []byte) int {
	if len(a) != len(b) {
		// TODO this should really just use a multiple return and include an Error object
		// even if its going to panic, it should really be panicing with an Error object
		// not a string
		panic("cannot compute a hamming distance for arrays of differing length")
	}

	distance := 0
	for i := range a {
		distance += byteDistance(a[i], b[i])
	}
	return distance
}

// calculate the hamming distance of two bytes by xoring them and then counting
// how many bits are set to 1 in the resultant value
func byteDistance(a byte, b byte) int {
	x := a ^ b
	distance := 0
	// for each bit in a byte, shift that bit into the right most position
	// and then 0 out all other positions, yielding a 1 if that bit is 1, and a 0 otherwise
	for i := 0; i < 8; i++ {
		distance += int(x >> uint(i) & 1)
	}
	return distance
}
<|endoftext|>


<commit_before>package fire

import (
	"net/http"

	"github.com/256dpi/fire/coal"

	"github.com/256dpi/jsonapi"
	"gopkg.in/mgo.v2/bson"
)

// An Operation indicates the purpose of a yield to a callback in the processing
// flow of an API request by a controller. These operations may occur multiple
// times during a single request.
type Operation int

// All the available operations.
const (
	_ Operation = iota

	// The list operation will used to authorize the loading of multiple
	// resources from a collection.
	//
	// Note: This operation is also used to load related resources.
	List

	// The find operation will be used to authorize the loading of a specific
	// resource from a collection.
	//
	// Note: This operation is also used to load a specific related resource.
	Find

	// The create operation will be used to authorize and validate the creation
	// of a new resource in a collection.
	Create

	// The update operation will be used to authorize the loading and validate
	// the updating of a specific resource in a collection.
	//
	// Note: Updates can include attributes, relationships or both.
	Update

	// The delete operation will be used to authorize the loading and validate
	// the deletion of a specific resource in a collection.
	Delete

	// The collection action operation will be used to authorize the execution
	// of a callback for a collection action.
	CollectionAction

	// The resource action operation will be used to authorize the execution
	// of a callback for a resource action.
	ResourceAction
)

// Read will return true when this operations does only read data.
func (o Operation) Read() bool {
	return o == List || o == Find
}

// Write will return true when this operation does write data.
func (o Operation) Write() bool {
	return o == Create || o == Update || o == Delete
}

// Action will return true when this operation is a collection or resource action.
func (o Operation) Action() bool {
	return o == CollectionAction || o == ResourceAction
}

// String returns the name of the operation.
func (o Operation) String() string {
	switch o {
	case List:
		return "List"
	case Find:
		return "Find"
	case Create:
		return "Create"
	case Update:
		return "Update"
	case Delete:
		return "Delete"
	case CollectionAction:
		return "CollectionAction"
	case ResourceAction:
		return "ResourceAction"
	}

	return ""
}

// A Context provides useful contextual information.
type Context struct {
	// The current operation in process (read only).
	Operation Operation

	// The query that will be used during an List, Find, Update, Delete or
	// ResourceAction operation to select a list of models or a specific model.
	//
	// On Find, Update and Delete operations, the "_id" key is preset to the
	// resource id, while on forwarded List operations the relationship filter
	// is preset.
	Selector bson.M

	// TODO: Split up Selector up in ID, Reference and ReferenceID?

	// The query that will be used during an List, Find, Update, Delete or
	// ResourceAction operation to further filter the selection of a list of
	// models or a specific model.
	//
	// On List operations, attribute and relationship filters are preset.
	Filters []bson.M

	// TODO: Split up preset and custom filters?

	// The Model that will be saved during Create, updated during Update or
	// deleted during Delete.
	Model coal.Model

	// The sorting that will be used during List.
	Sorting []string

	// TODO: Simplify Sorting e.g. Field: Asc / Desc.

	// The filtered fields if at least one is available.
	Fields []string

	// The document that will be written to the client during List, Find, Create
	// and Update.
	//
	// Note: The document will be set before notifiers are run.
	Response *jsonapi.Document

	// The store that is used to retrieve and persist the model (read only).
	Store *coal.SubStore

	// The underlying JSON-API request (read only).
	JSONAPIRequest *jsonapi.Request

	// The underlying HTTP request (read only).
	//
	// Note: The path is not updated when a controller forwards a request to
	// a related controller.
	HTTPRequest *http.Request

	// The underlying HTTP response writer. The response writer should only be
	// used during collection or resource actions to write a custom response.
	ResponseWriter http.ResponseWriter

	// The Controller that is managing the request (read only).
	Controller *Controller

	// The Group that received the request (read only).
	Group *Group

	// The Tracer used to tracer code execution (read only).
	Tracer *Tracer

	original coal.Model
}

// Query returns the composite query of Selector and Filter.
func (c *Context) Query() bson.M {
	return bson.M{"$and": append([]bson.M{c.Selector}, c.Filters...)}
}

// Original will return the stored version of the model. This method is intended
// to be used to calculate the changed fields during an Update operation. Any
// returned error is already marked as fatal. This function will cache and reuse
// loaded models between multiple callbacks.
//
// Note: The method will panic if being used during any other operation than Update.
func (c *Context) Original() (coal.Model, error) {
	// begin trace
	c.Tracer.Push("fire/Context.Original")

	// check operation
	if c.Operation != Update {
		panic("fire: the original can only be loaded during an update operation")
	}

	// return cached model
	if c.original != nil {
		c.Tracer.Pop()
		return c.original, nil
	}

	// create a new model
	m := c.Model.Meta().Make()

	// read original document
	c.Tracer.Push("mgo/Query.One")
	c.Tracer.Tag("id", c.Model.ID())
	err := c.Store.C(c.Model).FindId(c.Model.ID()).One(m)
	if err != nil {
		return nil, err
	}
	c.Tracer.Pop()

	// cache model
	c.original = coal.Init(m)

	// finish trace
	c.Tracer.Pop()

	return c.original, nil
}
<commit_msg>updated<commit_after>package fire

import (
	"net/http"

	"github.com/256dpi/fire/coal"

	"github.com/256dpi/jsonapi"
	"gopkg.in/mgo.v2/bson"
)

// An Operation indicates the purpose of a yield to a callback in the processing
// flow of an API request by a controller. These operations may occur multiple
// times during a single request.
type Operation int

// All the available operations.
const (
	_ Operation = iota

	// The list operation will used to authorize the loading of multiple
	// resources from a collection.
	//
	// Note: This operation is also used to load related resources.
	List

	// The find operation will be used to authorize the loading of a specific
	// resource from a collection.
	//
	// Note: This operation is also used to load a specific related resource.
	Find

	// The create operation will be used to authorize and validate the creation
	// of a new resource in a collection.
	Create

	// The update operation will be used to authorize the loading and validate
	// the updating of a specific resource in a collection.
	//
	// Note: Updates can include attributes, relationships or both.
	Update

	// The delete operation will be used to authorize the loading and validate
	// the deletion of a specific resource in a collection.
	Delete

	// The collection action operation will be used to authorize the execution
	// of a callback for a collection action.
	CollectionAction

	// The resource action operation will be used to authorize the execution
	// of a callback for a resource action.
	ResourceAction
)

// Read will return true when this operations does only read data.
func (o Operation) Read() bool {
	return o == List || o == Find
}

// Write will return true when this operation does write data.
func (o Operation) Write() bool {
	return o == Create || o == Update || o == Delete
}

// Action will return true when this operation is a collection or resource action.
func (o Operation) Action() bool {
	return o == CollectionAction || o == ResourceAction
}

// String returns the name of the operation.
func (o Operation) String() string {
	switch o {
	case List:
		return "List"
	case Find:
		return "Find"
	case Create:
		return "Create"
	case Update:
		return "Update"
	case Delete:
		return "Delete"
	case CollectionAction:
		return "CollectionAction"
	case ResourceAction:
		return "ResourceAction"
	}

	return ""
}

// A Context provides useful contextual information.
type Context struct {
	// The current operation in process (read only).
	Operation Operation

	// The query that will be used during an List, Find, Update, Delete or
	// ResourceAction operation to select a list of models or a specific model.
	//
	// On Find, Update and Delete operations, the "_id" key is preset to the
	// resource id, while on forwarded List operations the relationship filter
	// is preset.
	Selector bson.M

	// TODO: Split up Selector up in ID, Reference and ReferenceID?

	// The filters that will be used during an List, Find, Update, Delete or
	// ResourceAction operation to further filter the selection of a list of
	// models or a specific model.
	//
	// On List operations, attribute and relationship filters are preset.
	Filters []bson.M

	// The Model that will be saved during Create, updated during Update or
	// deleted during Delete.
	Model coal.Model

	// The sorting that will be used during List.
	Sorting []string

	// The filtered fields if at least one is available.
	Fields []string

	// The document that will be written to the client during List, Find, Create
	// and Update.
	//
	// Note: The document will be set before notifiers are run.
	Response *jsonapi.Document

	// The store that is used to retrieve and persist the model (read only).
	Store *coal.SubStore

	// The underlying JSON-API request (read only).
	JSONAPIRequest *jsonapi.Request

	// The underlying HTTP request (read only).
	//
	// Note: The path is not updated when a controller forwards a request to
	// a related controller.
	HTTPRequest *http.Request

	// The underlying HTTP response writer. The response writer should only be
	// used during collection or resource actions to write a custom response.
	ResponseWriter http.ResponseWriter

	// The Controller that is managing the request (read only).
	Controller *Controller

	// The Group that received the request (read only).
	Group *Group

	// The Tracer used to tracer code execution (read only).
	Tracer *Tracer

	original coal.Model
}

// Query returns the composite query of Selector and Filter.
func (c *Context) Query() bson.M {
	return bson.M{"$and": append([]bson.M{c.Selector}, c.Filters...)}
}

// Original will return the stored version of the model. This method is intended
// to be used to calculate the changed fields during an Update operation. Any
// returned error is already marked as fatal. This function will cache and reuse
// loaded models between multiple callbacks.
//
// Note: The method will panic if being used during any other operation than Update.
func (c *Context) Original() (coal.Model, error) {
	// begin trace
	c.Tracer.Push("fire/Context.Original")

	// check operation
	if c.Operation != Update {
		panic("fire: the original can only be loaded during an update operation")
	}

	// return cached model
	if c.original != nil {
		c.Tracer.Pop()
		return c.original, nil
	}

	// create a new model
	m := c.Model.Meta().Make()

	// read original document
	c.Tracer.Push("mgo/Query.One")
	c.Tracer.Tag("id", c.Model.ID())
	err := c.Store.C(c.Model).FindId(c.Model.ID()).One(m)
	if err != nil {
		return nil, err
	}
	c.Tracer.Pop()

	// cache model
	c.original = coal.Init(m)

	// finish trace
	c.Tracer.Pop()

	return c.original, nil
}
<|endoftext|>


<commit_before>package hyper

import (
	"context"

	"github.com/vaniila/hyper/router"
)

// Context reads router context from context.Context
func Context(c context.Context) router.Context {
	return c.Value(router.RequestContext).(router.Context)
}
<commit_msg>Parse context utility<commit_after>package hyper

import (
	"context"

	"github.com/vaniila/hyper/router"
)

// Context reads router context from context.Context
func Context(c context.Context) router.Context {
	return c.Value(router.RequestContext).(router.Context)
}

// Parse reads router context from context.Context
func Parse(c context.Context) (router.Context, bool) {
	if o, ok := c.Value(router.RequestContext).(router.Context); ok {
		return o, true
	}
	return nil, false
}
<|endoftext|>


<commit_before>package registry

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/go-kit/kit/log"
	dockerregistry "github.com/heroku/docker-registry-client/registry"
	"github.com/pkg/errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/registry/cache"
)

// A client represents an entity that returns manifest and tags
// information.  It might be a cache, it might be a real registry.
type Client interface {
	Tags(id flux.ImageID) ([]string, error)
	Manifest(id flux.ImageID) (flux.Image, error)
	Cancel()
}

// ---

// An implementation of Client that represents a Remote registry.
// E.g. docker hub.
type Remote struct {
	Registry   *herokuManifestAdaptor
	CancelFunc context.CancelFunc
}

// Return the tags for this repository.
func (a *Remote) Tags(id flux.ImageID) ([]string, error) {
	return a.Registry.Tags(id.Image)
}

// We need to do some adapting here to convert from the return values
// from dockerregistry to our domain types.
func (a *Remote) Manifest(id flux.ImageID) (flux.Image, error) {
	manifestV2, err := a.Registry.ManifestV2(id.Image, id.Tag)
	if err != nil {
		if err, ok := err.(*url.Error); ok {
			if err, ok := (err.Err).(*dockerregistry.HttpStatusError); ok {
				if err.Response.StatusCode == http.StatusNotFound {
					return a.ManifestFromV1(id)
				}
			}
		}
		fmt.Printf("%#v\n", err)
		return flux.Image{}, err
	}
	// The above request will happily return a bogus, empty manifest
	// if handed something other than a schema2 manifest.
	if manifestV2.Config.Digest == "" {
		return a.ManifestFromV1(id)
	}

	// schema2 manifests have a reference to a blog that contains the
	// image config. We have to fetch that in order to get the created
	// datetime.
	conf := manifestV2.Config
	reader, err := a.Registry.DownloadLayer(id.Image, conf.Digest)
	if err != nil {
		return flux.Image{}, err
	}
	if reader == nil {
		return flux.Image{}, fmt.Errorf("nil reader from DownloadLayer")
	}

	type config struct {
		Created time.Time `json:created`
	}
	var imageConf config

	err = json.NewDecoder(reader).Decode(&imageConf)
	if err != nil {
		return flux.Image{}, err
	}
	return flux.Image{
		ID:        id,
		CreatedAt: imageConf.Created,
	}, nil
}

func (a *Remote) ManifestFromV1(id flux.ImageID) (flux.Image, error) {
	history, err := a.Registry.Manifest(id.Image, id.Tag)
	if err != nil || history == nil {
		return flux.Image{}, errors.Wrap(err, "getting remote manifest")
	}

	// the manifest includes some v1-backwards-compatibility data,
	// oddly called "History", which are layer metadata as JSON
	// strings; these appear most-recent (i.e., topmost layer) first,
	// so happily we can just decode the first entry to get a created
	// time.
	type v1image struct {
		Created time.Time `json:"created"`
	}
	var topmost v1image
	var img flux.Image
	img.ID = id
	if len(history) > 0 {
		if err = json.Unmarshal([]byte(history[0].V1Compatibility), &topmost); err == nil {
			if !topmost.Created.IsZero() {
				img.CreatedAt = topmost.Created
			}
		}
	}

	return img, nil
}

// Cancel the remote request
func (a *Remote) Cancel() {
	a.CancelFunc()
}

// ---

// An implementation of Client backed by Memcache
type Cache struct {
	creds  Credentials
	expiry time.Duration
	cr     cache.Reader
	logger log.Logger
}

func (*Cache) Cancel() {
	return
}

func NewCache(creds Credentials, cr cache.Reader, expiry time.Duration, logger log.Logger) Client {
	return &Cache{
		creds:  creds,
		expiry: expiry,
		cr:     cr,
		logger: logger,
	}
}

func (c *Cache) Manifest(id flux.ImageID) (flux.Image, error) {
	creds := c.creds.credsFor(id.Host)
	key, err := cache.NewManifestKey(creds.username, id)
	if err != nil {
		return flux.Image{}, err
	}
	val, err := c.cr.GetKey(key)
	if err != nil {
		return flux.Image{}, err
	}
	var img flux.Image
	err = json.Unmarshal(val, &img)
	if err != nil {
		c.logger.Log("err", err.Error)
		return flux.Image{}, err
	}
	return img, nil
}

func (c *Cache) Tags(id flux.ImageID) ([]string, error) {
	creds := c.creds.credsFor(id.Host)
	key, err := cache.NewTagKey(creds.username, id)
	if err != nil {
		return []string{}, err
	}
	val, err := c.cr.GetKey(key)
	if err != nil {
		return []string{}, err
	}
	var tags []string
	err = json.Unmarshal(val, &tags)
	if err != nil {
		c.logger.Log("err", err.Error)
		return []string{}, err
	}
	return tags, nil
}
<commit_msg>Remove debug printf<commit_after>package registry

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/go-kit/kit/log"
	dockerregistry "github.com/heroku/docker-registry-client/registry"
	"github.com/pkg/errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/registry/cache"
)

// A client represents an entity that returns manifest and tags
// information.  It might be a cache, it might be a real registry.
type Client interface {
	Tags(id flux.ImageID) ([]string, error)
	Manifest(id flux.ImageID) (flux.Image, error)
	Cancel()
}

// ---

// An implementation of Client that represents a Remote registry.
// E.g. docker hub.
type Remote struct {
	Registry   *herokuManifestAdaptor
	CancelFunc context.CancelFunc
}

// Return the tags for this repository.
func (a *Remote) Tags(id flux.ImageID) ([]string, error) {
	return a.Registry.Tags(id.Image)
}

// We need to do some adapting here to convert from the return values
// from dockerregistry to our domain types.
func (a *Remote) Manifest(id flux.ImageID) (flux.Image, error) {
	manifestV2, err := a.Registry.ManifestV2(id.Image, id.Tag)
	if err != nil {
		if err, ok := err.(*url.Error); ok {
			if err, ok := (err.Err).(*dockerregistry.HttpStatusError); ok {
				if err.Response.StatusCode == http.StatusNotFound {
					return a.ManifestFromV1(id)
				}
			}
		}
		return flux.Image{}, err
	}
	// The above request will happily return a bogus, empty manifest
	// if handed something other than a schema2 manifest.
	if manifestV2.Config.Digest == "" {
		return a.ManifestFromV1(id)
	}

	// schema2 manifests have a reference to a blog that contains the
	// image config. We have to fetch that in order to get the created
	// datetime.
	conf := manifestV2.Config
	reader, err := a.Registry.DownloadLayer(id.Image, conf.Digest)
	if err != nil {
		return flux.Image{}, err
	}
	if reader == nil {
		return flux.Image{}, fmt.Errorf("nil reader from DownloadLayer")
	}

	type config struct {
		Created time.Time `json:created`
	}
	var imageConf config

	err = json.NewDecoder(reader).Decode(&imageConf)
	if err != nil {
		return flux.Image{}, err
	}
	return flux.Image{
		ID:        id,
		CreatedAt: imageConf.Created,
	}, nil
}

func (a *Remote) ManifestFromV1(id flux.ImageID) (flux.Image, error) {
	history, err := a.Registry.Manifest(id.Image, id.Tag)
	if err != nil || history == nil {
		return flux.Image{}, errors.Wrap(err, "getting remote manifest")
	}

	// the manifest includes some v1-backwards-compatibility data,
	// oddly called "History", which are layer metadata as JSON
	// strings; these appear most-recent (i.e., topmost layer) first,
	// so happily we can just decode the first entry to get a created
	// time.
	type v1image struct {
		Created time.Time `json:"created"`
	}
	var topmost v1image
	var img flux.Image
	img.ID = id
	if len(history) > 0 {
		if err = json.Unmarshal([]byte(history[0].V1Compatibility), &topmost); err == nil {
			if !topmost.Created.IsZero() {
				img.CreatedAt = topmost.Created
			}
		}
	}

	return img, nil
}

// Cancel the remote request
func (a *Remote) Cancel() {
	a.CancelFunc()
}

// ---

// An implementation of Client backed by Memcache
type Cache struct {
	creds  Credentials
	expiry time.Duration
	cr     cache.Reader
	logger log.Logger
}

func (*Cache) Cancel() {
	return
}

func NewCache(creds Credentials, cr cache.Reader, expiry time.Duration, logger log.Logger) Client {
	return &Cache{
		creds:  creds,
		expiry: expiry,
		cr:     cr,
		logger: logger,
	}
}

func (c *Cache) Manifest(id flux.ImageID) (flux.Image, error) {
	creds := c.creds.credsFor(id.Host)
	key, err := cache.NewManifestKey(creds.username, id)
	if err != nil {
		return flux.Image{}, err
	}
	val, err := c.cr.GetKey(key)
	if err != nil {
		return flux.Image{}, err
	}
	var img flux.Image
	err = json.Unmarshal(val, &img)
	if err != nil {
		c.logger.Log("err", err.Error)
		return flux.Image{}, err
	}
	return img, nil
}

func (c *Cache) Tags(id flux.ImageID) ([]string, error) {
	creds := c.creds.credsFor(id.Host)
	key, err := cache.NewTagKey(creds.username, id)
	if err != nil {
		return []string{}, err
	}
	val, err := c.cr.GetKey(key)
	if err != nil {
		return []string{}, err
	}
	var tags []string
	err = json.Unmarshal(val, &tags)
	if err != nil {
		c.logger.Log("err", err.Error)
		return []string{}, err
	}
	return tags, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"time"
)

var prefixes = "0123456789abcdef"

type CacheEntry struct {
	HTTPCode int
	Expires  time.Time
}

type diskCacheEntry struct {
	Data []byte
	CacheEntry
}

type DiskCache struct {
	cacheRoot  string
	cacheFiles map[string]CacheEntry
	sync.RWMutex
}

func (d *DiskCache) init() {
	d.Lock()
	defer d.Unlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to load uninitialized cache.")
	}

	os.Mkdir(d.cacheRoot, 0770)

	for _, dir := range prefixes {
		dirName := d.cacheRoot + "/" + string(dir)
		err := os.Mkdir(dirName, 0770)
		if err != nil {
			//Either cannot create directory, or directory already exists, let's try opening it to find out.
			dirf, derr := os.Open(dirName)
			if derr != nil {
				// Couldn't open directory, panic.
				log.Fatalf("Couldn't create or open %s: %s/%s", dirName, err, derr)
			}

			files, err := dirf.Readdirnames(0)
			if err != nil {
				log.Fatalf("Couldn't read %s: %s", dirName, err)
			}

			var de diskCacheEntry
			for _, filename := range files {
				fullname := dirName + "/" + filename

				jsondata, err := ioutil.ReadFile(fullname)
				if err != nil {
					log.Fatalf("Failed to read %s: %s", fullname, err)
				}

				err = json.Unmarshal(jsondata, &de)
				if err != nil {
					log.Printf("Recovering from cache consistency error for %s: %s ", fullname, err)
				}

				if err != nil || time.Now().After(de.Expires) {
					err := os.Remove(fullname)
					if err != nil {
						log.Fatalf("Failed to remove expired cache entry %s: %s", fullname, err)
					}
					continue
				}

				d.cacheFiles[filename] = de.CacheEntry
			}
		}
	}
}

var cleanOnce = &sync.Once{}

func (d *DiskCache) clean() {
	log.Printf("Cleaning Up.")
	now := time.Now()

	cleancount := 0
	for tag, ce := range d.cacheFiles {
		if now.After(ce.Expires) {
			os.Remove(d.filename(tag))
			delete(d.cacheFiles, tag)

			cleancount++
		}
	}
	log.Printf("Cleaned up %d entries.", cleancount)
	cleanOnce = &sync.Once{}
}

var storeCount int64

func (d *DiskCache) filename(tag string) string {
	return d.cacheRoot + "/" + string(tag[0]) + "/" + tag
}

func (d *DiskCache) Store(cacheTag string, HTTPCode int, data []byte, Expires time.Time) error {
	d.Lock()
	defer d.Unlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to store to uninitialized cache.")
	}

	storeCount++
	if storeCount%500 == 0 {
		go cleanOnce.Do(func() { d.clean() })
	}

	ce := CacheEntry{HTTPCode, Expires}

	de := diskCacheEntry{data, ce}
	jsondata, err := json.Marshal(&de)
	if err != nil {
		log.Printf("Unknown JSON Marshal Error: %s", err)
		return err
	}

	err = ioutil.WriteFile(d.filename(cacheTag), jsondata, 0660)
	if err != nil {
		log.Printf("Unknown File Error: %s", err)
		return err
	}

	d.cacheFiles[cacheTag] = ce
	return nil
}

func (d *DiskCache) Get(cacheTag string) (int, []byte, time.Time, error) {
	d.RLock()
	defer d.RUnlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to get from uninitialized cache.")
	}

	ce, exists := d.cacheFiles[cacheTag]
	if !exists || time.Now().After(ce.Expires) {
		return 0, nil, ce.Expires, fmt.Errorf("Not cached.")
	}

	jsondata, err := ioutil.ReadFile(d.filename(cacheTag))
	if err != nil {
		delete(d.cacheFiles, cacheTag)
		return 0, nil, ce.Expires, fmt.Errorf("Cache error - File not found.")
	}

	var de diskCacheEntry
	err = json.Unmarshal(jsondata, &de)
	if err != nil || de.Expires != ce.Expires {
		log.Printf("Cache consistency error: %s (Got: %s Expected: %s)", err, de.Expires, ce.Expires)

		delete(d.cacheFiles, cacheTag)
		return 0, nil, ce.Expires, fmt.Errorf("Cache error - Cache invalid.")
	}

	return ce.HTTPCode, de.Data, ce.Expires, nil
}

func (d *DiskCache) LogStats() {
	d.RLock()
	defer d.RUnlock()

	entries := 0
	expired := 0

	now := time.Now()
	for _, ce := range d.cacheFiles {
		entries++
		if now.After(ce.Expires) {
			expired++
		}
	}

	log.Printf("Cache Entries: %d  Expired Entries: %d", entries, expired)
}

func NewDiskCache(rootDir string) *DiskCache {
	var dc DiskCache

	dc.cacheRoot = rootDir
	dc.cacheFiles = make(map[string]CacheEntry)

	dc.init()

	return &dc
}
<commit_msg>fixed race condition in removing stale items<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"time"
)

var prefixes = "0123456789abcdef"

type CacheEntry struct {
	HTTPCode int
	Expires  time.Time
}

type diskCacheEntry struct {
	Data []byte
	CacheEntry
}

type DiskCache struct {
	cacheRoot  string
	cacheFiles map[string]CacheEntry
	sync.RWMutex
}

func (d *DiskCache) init() {
	d.Lock()
	defer d.Unlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to load uninitialized cache.")
	}

	os.Mkdir(d.cacheRoot, 0770)

	for _, dir := range prefixes {
		dirName := d.cacheRoot + "/" + string(dir)
		err := os.Mkdir(dirName, 0770)
		if err != nil {
			//Either cannot create directory, or directory already exists, let's try opening it to find out.
			dirf, derr := os.Open(dirName)
			if derr != nil {
				// Couldn't open directory, panic.
				log.Fatalf("Couldn't create or open %s: %s/%s", dirName, err, derr)
			}

			files, err := dirf.Readdirnames(0)
			if err != nil {
				log.Fatalf("Couldn't read %s: %s", dirName, err)
			}

			var de diskCacheEntry
			for _, filename := range files {
				fullname := dirName + "/" + filename

				jsondata, err := ioutil.ReadFile(fullname)
				if err != nil {
					log.Fatalf("Failed to read %s: %s", fullname, err)
				}

				err = json.Unmarshal(jsondata, &de)
				if err != nil {
					log.Printf("Recovering from cache consistency error for %s: %s ", fullname, err)
				}

				if err != nil || time.Now().After(de.Expires) {
					err := os.Remove(fullname)
					if err != nil {
						log.Fatalf("Failed to remove expired cache entry %s: %s", fullname, err)
					}
					continue
				}

				d.cacheFiles[filename] = de.CacheEntry
			}
		}
	}
}

var cleanOnce = &sync.Once{}

func (d *DiskCache) clean() {
	log.Printf("Cleaning Up.")
	now := time.Now()

	d.Lock()
	cleancount := 0
	for tag, ce := range d.cacheFiles {
		if now.After(ce.Expires) {
			os.Remove(d.filename(tag))
			delete(d.cacheFiles, tag)

			cleancount++
		}
	}
	d.Unlock()
	log.Printf("Cleaned up %d entries.", cleancount)
	cleanOnce = &sync.Once{}
}

var storeCount int64

func (d *DiskCache) filename(tag string) string {
	return d.cacheRoot + "/" + string(tag[0]) + "/" + tag
}

func (d *DiskCache) Store(cacheTag string, HTTPCode int, data []byte, Expires time.Time) error {
	d.Lock()
	defer d.Unlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to store to uninitialized cache.")
	}

	storeCount++
	if storeCount%50 == 0 {
		go cleanOnce.Do(func() { d.clean() })
	}

	ce := CacheEntry{HTTPCode, Expires}

	de := diskCacheEntry{data, ce}
	jsondata, err := json.Marshal(&de)
	if err != nil {
		log.Printf("Unknown JSON Marshal Error: %s", err)
		return err
	}

	err = ioutil.WriteFile(d.filename(cacheTag), jsondata, 0660)
	if err != nil {
		log.Printf("Unknown File Error: %s", err)
		return err
	}

	d.cacheFiles[cacheTag] = ce
	return nil
}

func (d *DiskCache) Get(cacheTag string) (int, []byte, time.Time, error) {
	d.RLock()
	defer d.RUnlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to get from uninitialized cache.")
	}

	ce, exists := d.cacheFiles[cacheTag]
	if !exists || time.Now().After(ce.Expires) {
		return 0, nil, ce.Expires, fmt.Errorf("Not cached.")
	}

	jsondata, err := ioutil.ReadFile(d.filename(cacheTag))
	if err != nil {
		d.RUnlock()
		d.Lock()
		delete(d.cacheFiles, cacheTag)
		d.Unlock()

		return 0, nil, ce.Expires, fmt.Errorf("Cache error - File not found.")
	}

	var de diskCacheEntry
	err = json.Unmarshal(jsondata, &de)
	if err != nil || de.Expires != ce.Expires {
		log.Printf("Cache consistency error: %s (Got: %s Expected: %s)", err, de.Expires, ce.Expires)

		d.RUnlock()
		d.Lock()
		delete(d.cacheFiles, cacheTag)
		d.Unlock()

		return 0, nil, ce.Expires, fmt.Errorf("Cache error - Cache invalid.")
	}

	return ce.HTTPCode, de.Data, ce.Expires, nil
}

func (d *DiskCache) LogStats() {
	d.RLock()
	defer d.RUnlock()

	entries := 0
	expired := 0

	now := time.Now()
	for _, ce := range d.cacheFiles {
		entries++
		if now.After(ce.Expires) {
			expired++
		}
	}

	log.Printf("Cache Entries: %d  Expired Entries: %d", entries, expired)
}

func NewDiskCache(rootDir string) *DiskCache {
	var dc DiskCache

	dc.cacheRoot = rootDir
	dc.cacheFiles = make(map[string]CacheEntry)

	dc.init()

	return &dc
}
<|endoftext|>


<commit_before>package scraper

import (
	"hnews/Godeps/_workspace/src/github.com/yhat/scrape"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html/atom"
	"hnews/services"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type Scraper struct {
}

func NewScraper() *Scraper {
	scraper := new(Scraper)
	go startScraping()
	return scraper
}

func startScraping() {
	newsCh := make(chan []services.News)
	commentsCh := make(chan []services.Comment)
	go scrapeFrontPage(newsCh)
	go scrapeComments(commentsCh)

	for {
		select {
		case newNews := <-newsCh:
			go services.SaveNews(newNews)
		case newComments := <-commentsCh:
			go services.SaveComments(newComments)
		}
	}
}

/********************** News **********************/
// Parses the front page and sends a []News of the content
func scrapeFrontPage(newsCh chan []services.News) {
	for {
		for i := 1; i <= 16; i++ {
			var news []services.News

			url := "https://news.ycombinator.com/news?p=" + strconv.Itoa(i)
			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}

			points := parsePoints(root)
			ranks := parseRanks(root)
			titles, links := parseArticles(root)
			authors := parseAuthors(root)
			times := parseTimes(root)
			comments := parseNumComments(root)
			ids := parseIDs(root)

			for i := 0; i < len(ranks); i++ {
				rank := int32(ranks[i])
				title := titles[i]

				var time time.Time
				if i < len(times) {
					time = times[i]
				}

				var link string
				if i < len(links) {
					link = links[i]
				}

				var author string
				if i < len(authors) {
					author = authors[i]
				}

				var numPoints int32
				if i < len(points) {
					numPoints = int32(points[i])
				}

				var numComments int32
				if i < len(comments) {
					numComments = int32(comments[i])
				}

				var id int32
				if i < len(ids) {
					id = int32(ids[i])
				}

				news = append(news, services.News{id, rank, title, link, author, numPoints, time, numComments})
			}
			newsCh <- news
		}
		time.Sleep(1 * time.Second)
	}
}

// Parses out the rank of the articles.
func parseRanks(root *html.Node) []int {
	var ranks []int

	rankMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil && n.Parent.Parent != nil {
			grandparent := scrape.Attr(n.Parent.Parent, "class") == "athing"
			self := scrape.Attr(n, "class") == "rank"
			return grandparent && self
		}
		return false
	}

	rankNodes := scrape.FindAll(root, rankMatcher)
	for _, rankNode := range rankNodes {
		text := strings.Replace(scrape.Text(rankNode), ".", "", -1)
		rank, err := strconv.Atoi(text)
		if err != nil {
			rank = 0
		}
		ranks = append(ranks, rank)
	}
	return ranks
}

// Parses the authors of each Story on the frontpage
func parseAuthors(root *html.Node) []string {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "user" {
				return parent && true
			}
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	return authors
}

// Parses the number of comments on each Story on the frontpage
func parseNumComments(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "item" {
				return parent && true
			}
		}
		return false
	}

	var numComms []int // Number of comments for each Story
	commentNodes := scrape.FindAll(root, matcher)
	for _, commentNode := range commentNodes {
		text := scrape.Text(commentNode)
		words := strings.Fields(text)
		numComm, err := strconv.Atoi(words[0])
		if err != nil {
			numComm = 0
		}
		numComms = append(numComms, numComm)
	}
	return numComms
}

// Parses the timestamps from the HTML of the frontpage
func parseTimes(root *html.Node) []time.Time {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		text := scrape.Text(timeNode)
		date, err := parseTimeString(text)
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	return dates
}

// Parses the id of each Story on the page
func parseIDs(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		idTemp := scrape.Attr(idNode, "id")
		if len(idTemp) <= 6 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(idTemp[6:len(idTemp)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	return ids
}

// Quantity is of "hours"/"days"/"minutes"
// Text is of "4 hours ago", "41 days ago", etc
func parseTimeString(text string) (time.Time, error) {
	now := time.Now()
	words := strings.Fields(text)

	timeAgo, err := strconv.Atoi(words[0])
	if err != nil {
		return now, err
	}

	var result time.Time
	switch words[1] {
	case "minutes":
		result = now.Add(time.Duration(-timeAgo) * time.Minute)
	case "hours":
		result = now.Add(time.Duration(-timeAgo) * time.Hour)
	case "days":
		result = now.Add(time.Duration(-timeAgo*24) * time.Hour)
	case "day":
		result = now.Add(time.Duration(-timeAgo*24) * time.Hour)
	}
	return result, err
}

// Parses the title and the link of each Story on the page
func parseArticles(root *html.Node) ([]string, []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil && n.Parent.Parent != nil {
			return scrape.Attr(n.Parent.Parent, "class") == "athing"
		}
		return false
	}

	var titles []string
	var links []string

	articles := scrape.FindAll(root, matcher)
	for _, article := range articles {
		title := scrape.Text(article)
		link := scrape.Attr(article, "href")
		titles = append(titles, title)
		links = append(links, link)
	}
	return titles, links
}

func parsePoints(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		// must check for nil values
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var points []int

	pointsNodes := scrape.FindAll(root, matcher)
	for _, pointsNode := range pointsNodes {
		pointS := strings.Replace(scrape.Text(pointsNode), " points", "", -1)
		point, err := strconv.Atoi(pointS)
		if err != nil {
			point = 0
		}
		points = append(points, point)
	}
	return points
}

/********************** News **********************/

/******************** Comments ********************/
type Pair struct {
	author string
	time   time.Time
}

// Scrapes the Comments for every News item currently in the database.
func scrapeComments(commentsCh chan []services.Comment) {
	for {
		ids := services.ReadNewsIds()
		for _, id := range ids {
			url := "https://news.ycombinator.com/item?id=" + strconv.Itoa(int(id))

			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}

			comments := parseComments(root, id)
			commentsCh <- comments
		}
		time.Sleep(1 * time.Second)
	}
}

// Parses all the Comments for a particular News item.
func parseComments(root *html.Node, newsid int32) []services.Comment {
	offsets := parseOffsets(root)
	ids := parseCommentIDs(root)
	authors := parseCommentAuthors(root)
	times := parseCommentTimes(root)
	texts := parseCommentText(root)

	var rootComments []services.Comment
	for i, _ := range authors {
		comment := services.Comment{newsid, int32(ids[i]), int32(offsets[i]),
			times[i], authors[i], texts[i]}
		rootComments = append(rootComments, comment)
	}
	return rootComments
}

// Parses the level for each Comment in the comment tree. Interval: 0-inf.
func parseOffsets(root *html.Node) []int {
	offsetMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Img && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "ind"
			return parent
		}
		return false
	}

	offsets := scrape.FindAll(root, offsetMatcher)

	var norm []int
	for _, offset := range offsets {
		lvl, _ := strconv.Atoi(scrape.Attr(offset, "width"))
		norm = append(norm, int(lvl/40))
	}
	return norm
}

// Parses all the comments authors
func parseCommentAuthors(root *html.Node) []string {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "comhead"
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	return authors
}

// Parses all the comments itemids
func parseCommentIDs(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		href := scrape.Attr(idNode, "href")
		if len(href) <= 8 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(href[8:len(href)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	return ids
}

// Parses all the comments timestamps
func parseCommentTimes(root *html.Node) []time.Time {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		date, err := parseTimeString(scrape.Text(timeNode))
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	return dates
}

// Parses the text of all Comments for a News
func parseCommentText(root *html.Node) []string {
	textMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "comment"
			return parent
		}
		return false
	}

	textNodes := scrape.FindAll(root, textMatcher)

	var texts []string
	for _, text := range textNodes {
		content := scrape.Text(text)
		// BUG: Remove trailing trash from the 'Replay' HTML node ...
		texts = append(texts, content) //[0:len(content)-5])
	}
	return texts
}

/******************** Comments ********************/
<commit_msg>Removed unused struct in Scraper.<commit_after>package scraper

import (
	"hnews/Godeps/_workspace/src/github.com/yhat/scrape"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html/atom"
	"hnews/services"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type Scraper struct {
}

func NewScraper() *Scraper {
	scraper := new(Scraper)
	go startScraping()
	return scraper
}

func startScraping() {
	newsCh := make(chan []services.News)
	commentsCh := make(chan []services.Comment)
	go scrapeFrontPage(newsCh)
	go scrapeComments(commentsCh)

	for {
		select {
		case newNews := <-newsCh:
			go services.SaveNews(newNews)
		case newComments := <-commentsCh:
			go services.SaveComments(newComments)
		}
	}
}

/********************** News **********************/
// Parses the front page and sends a []News of the content
func scrapeFrontPage(newsCh chan []services.News) {
	for {
		for i := 1; i <= 16; i++ {
			var news []services.News

			url := "https://news.ycombinator.com/news?p=" + strconv.Itoa(i)
			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}

			points := parsePoints(root)
			ranks := parseRanks(root)
			titles, links := parseArticles(root)
			authors := parseAuthors(root)
			times := parseTimes(root)
			comments := parseNumComments(root)
			ids := parseIDs(root)

			for i := 0; i < len(ranks); i++ {
				rank := int32(ranks[i])
				title := titles[i]

				var time time.Time
				if i < len(times) {
					time = times[i]
				}

				var link string
				if i < len(links) {
					link = links[i]
				}

				var author string
				if i < len(authors) {
					author = authors[i]
				}

				var numPoints int32
				if i < len(points) {
					numPoints = int32(points[i])
				}

				var numComments int32
				if i < len(comments) {
					numComments = int32(comments[i])
				}

				var id int32
				if i < len(ids) {
					id = int32(ids[i])
				}

				news = append(news, services.News{id, rank, title, link, author, numPoints, time, numComments})
			}
			newsCh <- news
		}
		time.Sleep(1 * time.Second)
	}
}

// Parses out the rank of the articles.
func parseRanks(root *html.Node) []int {
	var ranks []int

	rankMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil && n.Parent.Parent != nil {
			grandparent := scrape.Attr(n.Parent.Parent, "class") == "athing"
			self := scrape.Attr(n, "class") == "rank"
			return grandparent && self
		}
		return false
	}

	rankNodes := scrape.FindAll(root, rankMatcher)
	for _, rankNode := range rankNodes {
		text := strings.Replace(scrape.Text(rankNode), ".", "", -1)
		rank, err := strconv.Atoi(text)
		if err != nil {
			rank = 0
		}
		ranks = append(ranks, rank)
	}
	return ranks
}

// Parses the authors of each Story on the frontpage
func parseAuthors(root *html.Node) []string {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "user" {
				return parent && true
			}
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	return authors
}

// Parses the number of comments on each Story on the frontpage
func parseNumComments(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "item" {
				return parent && true
			}
		}
		return false
	}

	var numComms []int // Number of comments for each Story
	commentNodes := scrape.FindAll(root, matcher)
	for _, commentNode := range commentNodes {
		text := scrape.Text(commentNode)
		words := strings.Fields(text)
		numComm, err := strconv.Atoi(words[0])
		if err != nil {
			numComm = 0
		}
		numComms = append(numComms, numComm)
	}
	return numComms
}

// Parses the timestamps from the HTML of the frontpage
func parseTimes(root *html.Node) []time.Time {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		text := scrape.Text(timeNode)
		date, err := parseTimeString(text)
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	return dates
}

// Parses the id of each Story on the page
func parseIDs(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		idTemp := scrape.Attr(idNode, "id")
		if len(idTemp) <= 6 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(idTemp[6:len(idTemp)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	return ids
}

// Quantity is of "hours"/"days"/"minutes"
// Text is of "4 hours ago", "41 days ago", etc
func parseTimeString(text string) (time.Time, error) {
	now := time.Now()
	words := strings.Fields(text)

	timeAgo, err := strconv.Atoi(words[0])
	if err != nil {
		return now, err
	}

	var result time.Time
	switch words[1] {
	case "minutes":
		result = now.Add(time.Duration(-timeAgo) * time.Minute)
	case "hours":
		result = now.Add(time.Duration(-timeAgo) * time.Hour)
	case "days":
		result = now.Add(time.Duration(-timeAgo*24) * time.Hour)
	case "day":
		result = now.Add(time.Duration(-timeAgo*24) * time.Hour)
	}
	return result, err
}

// Parses the title and the link of each Story on the page
func parseArticles(root *html.Node) ([]string, []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil && n.Parent.Parent != nil {
			return scrape.Attr(n.Parent.Parent, "class") == "athing"
		}
		return false
	}

	var titles []string
	var links []string

	articles := scrape.FindAll(root, matcher)
	for _, article := range articles {
		title := scrape.Text(article)
		link := scrape.Attr(article, "href")
		titles = append(titles, title)
		links = append(links, link)
	}
	return titles, links
}

func parsePoints(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		// must check for nil values
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var points []int

	pointsNodes := scrape.FindAll(root, matcher)
	for _, pointsNode := range pointsNodes {
		pointS := strings.Replace(scrape.Text(pointsNode), " points", "", -1)
		point, err := strconv.Atoi(pointS)
		if err != nil {
			point = 0
		}
		points = append(points, point)
	}
	return points
}

/********************** News **********************/

/******************** Comments ********************/
// Scrapes the Comments for every News item currently in the database.
func scrapeComments(commentsCh chan []services.Comment) {
	for {
		ids := services.ReadNewsIds()
		for _, id := range ids {
			url := "https://news.ycombinator.com/item?id=" + strconv.Itoa(int(id))

			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}

			comments := parseComments(root, id)
			commentsCh <- comments
		}
		time.Sleep(1 * time.Second)
	}
}

// Parses all the Comments for a particular News item.
func parseComments(root *html.Node, newsid int32) []services.Comment {
	offsets := parseOffsets(root)
	ids := parseCommentIDs(root)
	authors := parseCommentAuthors(root)
	times := parseCommentTimes(root)
	texts := parseCommentText(root)

	var rootComments []services.Comment
	for i, _ := range authors {
		comment := services.Comment{newsid, int32(ids[i]), int32(offsets[i]),
			times[i], authors[i], texts[i]}
		rootComments = append(rootComments, comment)
	}
	return rootComments
}

// Parses the level for each Comment in the comment tree. Interval: 0-inf.
func parseOffsets(root *html.Node) []int {
	offsetMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Img && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "ind"
			return parent
		}
		return false
	}

	offsets := scrape.FindAll(root, offsetMatcher)

	var norm []int
	for _, offset := range offsets {
		lvl, _ := strconv.Atoi(scrape.Attr(offset, "width"))
		norm = append(norm, int(lvl/40))
	}
	return norm
}

// Parses all the comments authors
func parseCommentAuthors(root *html.Node) []string {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "comhead"
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	return authors
}

// Parses all the comments itemids
func parseCommentIDs(root *html.Node) []int {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		href := scrape.Attr(idNode, "href")
		if len(href) <= 8 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(href[8:len(href)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	return ids
}

// Parses all the comments timestamps
func parseCommentTimes(root *html.Node) []time.Time {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		date, err := parseTimeString(scrape.Text(timeNode))
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	return dates
}

// Parses the text of all Comments for a News
func parseCommentText(root *html.Node) []string {
	textMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "comment"
			return parent
		}
		return false
	}

	textNodes := scrape.FindAll(root, textMatcher)

	var texts []string
	for _, text := range textNodes {
		content := scrape.Text(text)
		// BUG: Remove trailing trash from the 'Replay' HTML node ...
		texts = append(texts, content) //[0:len(content)-5])
	}
	return texts
}

/******************** Comments ********************/
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ghutil_test

import (
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/google/go-github/github"

	"github.com/google/code-review-bot/ghutil"
)

type MockGitHubClient struct {
	Organizations *ghutil.MockOrganizationsService
	PullRequests  *ghutil.MockPullRequestsService
	Issues        *ghutil.MockIssuesService
	Repositories  *ghutil.MockRepositoriesService
}

func NewMockGitHubClient(ghc *ghutil.GitHubClient, ctrl *gomock.Controller) *MockGitHubClient {
	mockGhc := &MockGitHubClient{
		Organizations: ghutil.NewMockOrganizationsService(ctrl),
		PullRequests:  ghutil.NewMockPullRequestsService(ctrl),
		Issues:        ghutil.NewMockIssuesService(ctrl),
		Repositories:  ghutil.NewMockRepositoriesService(ctrl),
	}

	// Patch the original GitHubClient with our mock services.
	ghc.Organizations = mockGhc.Organizations
	ghc.PullRequests = mockGhc.PullRequests
	ghc.Issues = mockGhc.Issues
	ghc.Repositories = mockGhc.Repositories

	return mockGhc
}

// Common parameters used across most, if not all, tests.
var (
	ctrl    *gomock.Controller
	ghc     *ghutil.GitHubClient
	mockGhc *MockGitHubClient

	noLabel *github.Label = nil
)

const (
	orgName   = "org"
	repoName  = "repo"
	emptyRepo = ""
)

func setUp(t *testing.T) {
	ctrl = gomock.NewController(t)
	ghc = &ghutil.GitHubClient{}
	mockGhc = NewMockGitHubClient(ghc, ctrl)
}

func tearDown(t *testing.T) {
	ctrl.Finish()
}

func TestGetAllRepos_OrgAndRepo(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	repo := github.Repository{}

	mockGhc.Repositories.EXPECT().Get(orgName, repoName).Return(&repo, nil, nil)

	repos := ghc.GetAllRepos(orgName, repoName)
	if len(repos) != 1 {
		t.Logf("repos is not of length 1: %v", repos)
		t.Fail()
	}
}

func TestGetAllRepos_OrgOnly(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	expectedRepos := []*github.Repository{
		&github.Repository{},
		&github.Repository{},
	}

	mockGhc.Repositories.EXPECT().List(orgName, nil).Return(expectedRepos, nil, nil)

	actualRepos := ghc.GetAllRepos(orgName, "")
	if len(expectedRepos) != len(actualRepos) {
		t.Logf("Expected repos: %v, actual repos: %v", expectedRepos, actualRepos)
		t.Fail()
	}
}

func TestVerifyRepoHasClaLabels_NoLabels(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaYes).Return(noLabel, nil, nil)
	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaNo).Return(noLabel, nil, nil)

	if ghc.VerifyRepoHasClaLabels(orgName, repoName) {
		t.Log("Should have returned false")
		t.Fail()
	}
}

func TestVerifyRepoHasClaLabels_HasYesOnly(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	label := github.Label{}

	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaYes).Return(&label, nil, nil)
	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaNo).Return(noLabel, nil, nil)

	if ghc.VerifyRepoHasClaLabels(orgName, repoName) {
		t.Log("Should have returned false")
		t.Fail()
	}
}

func TestVerifyRepoHasClaLabels_HasNoOnly(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	label := github.Label{}

	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaYes).Return(noLabel, nil, nil)
	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaNo).Return(&label, nil, nil)

	if ghc.VerifyRepoHasClaLabels(orgName, repoName) {
		t.Log("Should have returned false")
		t.Fail()
	}
}

func TestVerifyRepoHasClaLabels_YesAndNoLabels(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	labelYes := github.Label{}
	labelNo := github.Label{}

	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaYes).Return(&labelYes, nil, nil)
	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaNo).Return(&labelNo, nil, nil)

	if !ghc.VerifyRepoHasClaLabels(orgName, repoName) {
		t.Log("Should have returned true")
		t.Fail()
	}
}
<commit_msg>Executes gofmt -s in all files<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ghutil_test

import (
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/google/go-github/github"

	"github.com/google/code-review-bot/ghutil"
)

type MockGitHubClient struct {
	Organizations *ghutil.MockOrganizationsService
	PullRequests  *ghutil.MockPullRequestsService
	Issues        *ghutil.MockIssuesService
	Repositories  *ghutil.MockRepositoriesService
}

func NewMockGitHubClient(ghc *ghutil.GitHubClient, ctrl *gomock.Controller) *MockGitHubClient {
	mockGhc := &MockGitHubClient{
		Organizations: ghutil.NewMockOrganizationsService(ctrl),
		PullRequests:  ghutil.NewMockPullRequestsService(ctrl),
		Issues:        ghutil.NewMockIssuesService(ctrl),
		Repositories:  ghutil.NewMockRepositoriesService(ctrl),
	}

	// Patch the original GitHubClient with our mock services.
	ghc.Organizations = mockGhc.Organizations
	ghc.PullRequests = mockGhc.PullRequests
	ghc.Issues = mockGhc.Issues
	ghc.Repositories = mockGhc.Repositories

	return mockGhc
}

// Common parameters used across most, if not all, tests.
var (
	ctrl    *gomock.Controller
	ghc     *ghutil.GitHubClient
	mockGhc *MockGitHubClient

	noLabel *github.Label = nil
)

const (
	orgName   = "org"
	repoName  = "repo"
	emptyRepo = ""
)

func setUp(t *testing.T) {
	ctrl = gomock.NewController(t)
	ghc = &ghutil.GitHubClient{}
	mockGhc = NewMockGitHubClient(ghc, ctrl)
}

func tearDown(t *testing.T) {
	ctrl.Finish()
}

func TestGetAllRepos_OrgAndRepo(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	repo := github.Repository{}

	mockGhc.Repositories.EXPECT().Get(orgName, repoName).Return(&repo, nil, nil)

	repos := ghc.GetAllRepos(orgName, repoName)
	if len(repos) != 1 {
		t.Logf("repos is not of length 1: %v", repos)
		t.Fail()
	}
}

func TestGetAllRepos_OrgOnly(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	expectedRepos := []*github.Repository{
		{},
		{},
	}

	mockGhc.Repositories.EXPECT().List(orgName, nil).Return(expectedRepos, nil, nil)

	actualRepos := ghc.GetAllRepos(orgName, "")
	if len(expectedRepos) != len(actualRepos) {
		t.Logf("Expected repos: %v, actual repos: %v", expectedRepos, actualRepos)
		t.Fail()
	}
}

func TestVerifyRepoHasClaLabels_NoLabels(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaYes).Return(noLabel, nil, nil)
	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaNo).Return(noLabel, nil, nil)

	if ghc.VerifyRepoHasClaLabels(orgName, repoName) {
		t.Log("Should have returned false")
		t.Fail()
	}
}

func TestVerifyRepoHasClaLabels_HasYesOnly(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	label := github.Label{}

	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaYes).Return(&label, nil, nil)
	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaNo).Return(noLabel, nil, nil)

	if ghc.VerifyRepoHasClaLabels(orgName, repoName) {
		t.Log("Should have returned false")
		t.Fail()
	}
}

func TestVerifyRepoHasClaLabels_HasNoOnly(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	label := github.Label{}

	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaYes).Return(noLabel, nil, nil)
	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaNo).Return(&label, nil, nil)

	if ghc.VerifyRepoHasClaLabels(orgName, repoName) {
		t.Log("Should have returned false")
		t.Fail()
	}
}

func TestVerifyRepoHasClaLabels_YesAndNoLabels(t *testing.T) {
	setUp(t)
	defer tearDown(t)

	labelYes := github.Label{}
	labelNo := github.Label{}

	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaYes).Return(&labelYes, nil, nil)
	mockGhc.Issues.EXPECT().GetLabel(orgName, repoName, ghutil.LabelClaNo).Return(&labelNo, nil, nil)

	if !ghc.VerifyRepoHasClaLabels(orgName, repoName) {
		t.Log("Should have returned true")
		t.Fail()
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !appengine

package main

import (
	"bufio"
	"bytes"
	"compress/bzip2"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	client "net/http"
	"net/url"
	"os"
	"path/filepath"
	"runtime"

	"github.com/barakmich/glog"

	"github.com/google/cayley/config"
	"github.com/google/cayley/db"
	"github.com/google/cayley/graph"
	"github.com/google/cayley/http"
	"github.com/google/cayley/quad"
	"github.com/google/cayley/quad/cquads"
	"github.com/google/cayley/quad/nquads"

	// Load all supported backends.
	_ "github.com/google/cayley/graph/leveldb"
	_ "github.com/google/cayley/graph/memstore"
	_ "github.com/google/cayley/graph/mongo"

	// Load writer registry
	_ "github.com/google/cayley/writer"
)

var (
	tripleFile    = flag.String("triples", "", "Triple File to load before going to REPL.")
	tripleType    = flag.String("format", "cquad", `Triple format to use for loading ("cquad" or "nquad").`)
	cpuprofile    = flag.String("prof", "", "Output profiling file.")
	queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.")
	configFile    = flag.String("config", "", "Path to an explicit configuration file.")
)

// Filled in by `go build ldflags="-X main.VERSION `ver`"`.
var (
	BUILD_DATE string
	VERSION    string
)

func usage() {
	fmt.Fprintln(os.Stderr, `Cayley is a graph store and graph query layer.

Usage:
  cayley COMMAND [flags]

Commands:
  init      Create an empty database.
  load      Bulk-load a triple file into the database.
  http      Serve an HTTP endpoint on the given host and port.
  repl      Drop into a REPL of the given query language.
  version   Version information.

Flags:`)
	flag.PrintDefaults()
}

func init() {
	flag.Usage = usage
}

func main() {
	// No command? It's time for usage.
	if len(os.Args) == 1 {
		usage()
		os.Exit(1)
	}

	cmd := os.Args[1]
	os.Args = append(os.Args[:1], os.Args[2:]...)
	flag.Parse()

	var buildString string
	if VERSION != "" {
		buildString = fmt.Sprint("Cayley ", VERSION, " built ", BUILD_DATE)
		glog.Infoln(buildString)
	}

	cfg := config.ParseConfigFromFlagsAndFile(*configFile)

	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
		glog.Infoln("Setting GOMAXPROCS to", runtime.NumCPU())
	} else {
		glog.Infoln("GOMAXPROCS currently", os.Getenv("GOMAXPROCS"), " -- not adjusting")
	}

	var (
		handle *graph.Handle
		err    error
	)
	switch cmd {
	case "version":
		if VERSION != "" {
			fmt.Println(buildString)
		} else {
			fmt.Println("Cayley snapshot")
		}
		os.Exit(0)

	case "init":
		err = db.Init(cfg)
		if err != nil {
			break
		}
		if *tripleFile != "" {
			handle, err = db.Open(cfg)
			if err != nil {
				break
			}
			err = load(handle.QuadWriter, cfg, *tripleFile, *tripleType)
			if err != nil {
				break
			}
			handle.Close()
		}

	case "load":
		handle, err = db.Open(cfg)
		if err != nil {
			break
		}
		err = load(handle.QuadWriter, cfg, *tripleFile, *tripleType)
		if err != nil {
			break
		}

		handle.Close()

	case "repl":
		handle, err = db.Open(cfg)
		if err != nil {
			break
		}
		if !graph.IsPersistent(cfg.DatabaseType) {
			err = load(handle.QuadWriter, cfg, "", *tripleType)
			if err != nil {
				break
			}
		}

		err = db.Repl(handle, *queryLanguage, cfg)

		handle.Close()

	case "http":
		handle, err = db.Open(cfg)
		if err != nil {
			break
		}
		if !graph.IsPersistent(cfg.DatabaseType) {
			err = load(handle.QuadWriter, cfg, "", *tripleType)
			if err != nil {
				break
			}
		}

		http.Serve(handle, cfg)

		handle.Close()

	default:
		fmt.Println("No command", cmd)
		usage()
	}
	if err != nil {
		glog.Errorln(err)
	}
}

func load(qw graph.QuadWriter, cfg *config.Config, path, typ string) error {
	return decompressAndLoad(qw, cfg, path, typ, db.Load)
}

func removeAll(qw graph.QuadWriter, cfg *config.Config, path, typ string) error {
	return decompressAndLoad(qw, cfg, path, typ, remove)
}

func remove(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		qw.RemoveQuad(t)
	}
	return nil
}

func decompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Unmarshaler) error) error {
	var r io.Reader

	if path == "" {
		path = cfg.DatabasePath
	}
	if path == "" {
		return nil
	}
	u, err := url.Parse(path)
	if err != nil || u.Scheme == "file" || u.Scheme == "" {
		// Don't alter relative URL path or non-URL path parameter.
		if u.Scheme != "" && err == nil {
			// Recovery heuristic for mistyping "file://path/to/file".
			path = filepath.Join(u.Host, u.Path)
		}
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf("could not open file %q: %v", path, err)
		}
		defer f.Close()
		r = f
	} else {
		res, err := client.Get(path)
		if err != nil {
			return fmt.Errorf("could not get resource <%s>: %v", u, err)
		}
		defer res.Body.Close()
		r = res.Body
	}

	r, err = decompressor(r)
	if err != nil {
		return err
	}

	var dec quad.Unmarshaler
	switch typ {
	case "cquad":
		dec = cquads.NewDecoder(r)
	case "nquad":
		dec = nquads.NewDecoder(r)
	default:
		return fmt.Errorf("unknown quad format %q", typ)
	}

	return db.Load(qw, cfg, dec)
}

const (
	gzipMagic  = "\x1f\x8b"
	b2zipMagic = "BZh"
)

func decompressor(r io.Reader) (io.Reader, error) {
	br := bufio.NewReader(r)
	buf, err := br.Peek(3)
	if err != nil {
		return nil, err
	}
	switch {
	case bytes.Compare(buf[:2], []byte(gzipMagic)) == 0:
		return gzip.NewReader(br)
	case bytes.Compare(buf[:3], []byte(b2zipMagic)) == 0:
		return bzip2.NewReader(br), nil
	default:
		return br, nil
	}
}
<commit_msg>Make usage cayley intro banner contextual<commit_after>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !appengine

package main

import (
	"bufio"
	"bytes"
	"compress/bzip2"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	client "net/http"
	"net/url"
	"os"
	"path/filepath"
	"runtime"

	"github.com/barakmich/glog"

	"github.com/google/cayley/config"
	"github.com/google/cayley/db"
	"github.com/google/cayley/graph"
	"github.com/google/cayley/http"
	"github.com/google/cayley/quad"
	"github.com/google/cayley/quad/cquads"
	"github.com/google/cayley/quad/nquads"

	// Load all supported backends.
	_ "github.com/google/cayley/graph/leveldb"
	_ "github.com/google/cayley/graph/memstore"
	_ "github.com/google/cayley/graph/mongo"

	// Load writer registry
	_ "github.com/google/cayley/writer"
)

var (
	tripleFile    = flag.String("triples", "", "Triple File to load before going to REPL.")
	tripleType    = flag.String("format", "cquad", `Triple format to use for loading ("cquad" or "nquad").`)
	cpuprofile    = flag.String("prof", "", "Output profiling file.")
	queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.")
	configFile    = flag.String("config", "", "Path to an explicit configuration file.")
)

// Filled in by `go build ldflags="-X main.VERSION `ver`"`.
var (
	BUILD_DATE string
	VERSION    string
)

func usage() {
	fmt.Fprintln(os.Stderr, `
Usage:
  cayley COMMAND [flags]

Commands:
  init      Create an empty database.
  load      Bulk-load a triple file into the database.
  http      Serve an HTTP endpoint on the given host and port.
  repl      Drop into a REPL of the given query language.
  version   Version information.

Flags:`)
	flag.PrintDefaults()
}

func init() {
	flag.Usage = usage
}

func main() {
	// No command? It's time for usage.
	if len(os.Args) == 1 {
		fmt.Fprintln(os.Stderr, "Cayley is a graph store and graph query layer.")
		usage()
		os.Exit(1)
	}

	cmd := os.Args[1]
	os.Args = append(os.Args[:1], os.Args[2:]...)
	flag.Parse()

	var buildString string
	if VERSION != "" {
		buildString = fmt.Sprint("Cayley ", VERSION, " built ", BUILD_DATE)
		glog.Infoln(buildString)
	}

	cfg := config.ParseConfigFromFlagsAndFile(*configFile)

	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
		glog.Infoln("Setting GOMAXPROCS to", runtime.NumCPU())
	} else {
		glog.Infoln("GOMAXPROCS currently", os.Getenv("GOMAXPROCS"), " -- not adjusting")
	}

	var (
		handle *graph.Handle
		err    error
	)
	switch cmd {
	case "version":
		if VERSION != "" {
			fmt.Println(buildString)
		} else {
			fmt.Println("Cayley snapshot")
		}
		os.Exit(0)

	case "init":
		err = db.Init(cfg)
		if err != nil {
			break
		}
		if *tripleFile != "" {
			handle, err = db.Open(cfg)
			if err != nil {
				break
			}
			err = load(handle.QuadWriter, cfg, *tripleFile, *tripleType)
			if err != nil {
				break
			}
			handle.Close()
		}

	case "load":
		handle, err = db.Open(cfg)
		if err != nil {
			break
		}
		err = load(handle.QuadWriter, cfg, *tripleFile, *tripleType)
		if err != nil {
			break
		}

		handle.Close()

	case "repl":
		handle, err = db.Open(cfg)
		if err != nil {
			break
		}
		if !graph.IsPersistent(cfg.DatabaseType) {
			err = load(handle.QuadWriter, cfg, "", *tripleType)
			if err != nil {
				break
			}
		}

		err = db.Repl(handle, *queryLanguage, cfg)

		handle.Close()

	case "http":
		handle, err = db.Open(cfg)
		if err != nil {
			break
		}
		if !graph.IsPersistent(cfg.DatabaseType) {
			err = load(handle.QuadWriter, cfg, "", *tripleType)
			if err != nil {
				break
			}
		}

		http.Serve(handle, cfg)

		handle.Close()

	default:
		fmt.Println("No command", cmd)
		usage()
	}
	if err != nil {
		glog.Errorln(err)
	}
}

func load(qw graph.QuadWriter, cfg *config.Config, path, typ string) error {
	return decompressAndLoad(qw, cfg, path, typ, db.Load)
}

func removeAll(qw graph.QuadWriter, cfg *config.Config, path, typ string) error {
	return decompressAndLoad(qw, cfg, path, typ, remove)
}

func remove(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		qw.RemoveQuad(t)
	}
	return nil
}

func decompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Unmarshaler) error) error {
	var r io.Reader

	if path == "" {
		path = cfg.DatabasePath
	}
	if path == "" {
		return nil
	}
	u, err := url.Parse(path)
	if err != nil || u.Scheme == "file" || u.Scheme == "" {
		// Don't alter relative URL path or non-URL path parameter.
		if u.Scheme != "" && err == nil {
			// Recovery heuristic for mistyping "file://path/to/file".
			path = filepath.Join(u.Host, u.Path)
		}
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf("could not open file %q: %v", path, err)
		}
		defer f.Close()
		r = f
	} else {
		res, err := client.Get(path)
		if err != nil {
			return fmt.Errorf("could not get resource <%s>: %v", u, err)
		}
		defer res.Body.Close()
		r = res.Body
	}

	r, err = decompressor(r)
	if err != nil {
		return err
	}

	var dec quad.Unmarshaler
	switch typ {
	case "cquad":
		dec = cquads.NewDecoder(r)
	case "nquad":
		dec = nquads.NewDecoder(r)
	default:
		return fmt.Errorf("unknown quad format %q", typ)
	}

	return db.Load(qw, cfg, dec)
}

const (
	gzipMagic  = "\x1f\x8b"
	b2zipMagic = "BZh"
)

func decompressor(r io.Reader) (io.Reader, error) {
	br := bufio.NewReader(r)
	buf, err := br.Peek(3)
	if err != nil {
		return nil, err
	}
	switch {
	case bytes.Compare(buf[:2], []byte(gzipMagic)) == 0:
		return gzip.NewReader(br)
	case bytes.Compare(buf[:3], []byte(b2zipMagic)) == 0:
		return bzip2.NewReader(br), nil
	default:
		return br, nil
	}
}
<|endoftext|>


<commit_before>package main

// CLI for determining how ahead/behind a repo is from master
import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/client9/gosupplychain"
	//	"github.com/google/go-github/github"
)

func main() {
	flag.Parse()

	token := os.Getenv("GITHUB_OAUTH_TOKEN")
	if len(token) == 0 {
		log.Fatalf("Set GITHUB_OAUTH_TOKEN env")
	}

	args := flag.Args()
	if len(args) == 0 {
		log.Fatalf("Need path to godeps file")
	}
	gd, err := gosupplychain.LoadGodepsFile(args[0])
	if err != nil {
		log.Fatalf("Error loading godeps file %q: %s", args[0], err)
	}
	gh := gosupplychain.NewGitHub(token)

	for _, dep := range gd.Deps {
		parts := strings.Split(dep.ImportPath, "/")
		if len(parts) < 2 {
			log.Printf("Skipping %s", dep.ImportPath)
			continue
		}
		if parts[0] == "golang.org" && parts[1] == "x" {
			parts[0] = "github.com"
			parts[1] = "golang"
		}

		if parts[0] != "github.com" {
			log.Printf("Skipping %s", dep.ImportPath)
			continue
		}

		compare, _, err := gh.Client.Repositories.CompareCommits(parts[1], parts[2], dep.Rev, "HEAD")
		if err != nil {
			log.Printf("got error reading repo %s: %s", dep.ImportPath, err)
			continue
		}

		fmt.Printf("%s: %s\n", dep.ImportPath, *compare.Status)
		for pos, commit := range compare.Commits {
			msg := ""
			if commit.Commit.Message != nil {
				msg = *commit.Commit.Message
				msg = strings.Replace(msg, "\t", " ", -1)
				msg = strings.Replace(msg, "\r", " ", -1)
				msg = strings.Replace(msg, "\n", " ", -1)
				msg = strings.Replace(msg, "  ", " ", -1)
				if len(msg) > 80 {
					msg = msg[:80] + "..."
				}
			}
			sha := *commit.SHA
			fmt.Printf("    %d %s %s\n", pos, sha[0:7], msg)
		}
	}
}
<commit_msg>dont duplicate subpackages<commit_after>package main

// CLI for determining how ahead/behind a repo is from master
import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/client9/gosupplychain"
	"golang.org/x/tools/go/vcs"
)

func main() {
	flag.Parse()

	token := os.Getenv("GITHUB_OAUTH_TOKEN")
	if len(token) == 0 {
		log.Fatalf("Set GITHUB_OAUTH_TOKEN env")
	}

	args := flag.Args()
	if len(args) == 0 {
		log.Fatalf("Need path to godeps file")
	}
	gd, err := gosupplychain.LoadGodepsFile(args[0])
	if err != nil {
		log.Fatalf("Error loading godeps file %q: %s", args[0], err)
	}
	gh := gosupplychain.NewGitHub(token)

	roots := make(map[string]bool, len(gd.Deps))
	for _, dep := range gd.Deps {
		rr, err := vcs.RepoRootForImportPath(dep.ImportPath, true)
		if err != nil {
			log.Printf("Unable to process %s: %s", dep.ImportPath, err)
			continue
		}
		if roots[rr.Root] {
			continue
		}
		roots[rr.Root] = true
		parts := strings.Split(dep.ImportPath, "/")
		if len(parts) < 2 {
			log.Printf("Skipping %s", dep.ImportPath)
			continue
		}
		if parts[0] == "golang.org" && parts[1] == "x" {
			parts[0] = "github.com"
			parts[1] = "golang"
		}

		if parts[0] != "github.com" {
			log.Printf("Skipping %s", dep.ImportPath)
			continue
		}

		compare, _, err := gh.Client.Repositories.CompareCommits(parts[1], parts[2], dep.Rev, "HEAD")
		if err != nil {
			log.Printf("got error reading repo %s: %s", dep.ImportPath, err)
			continue
		}

		fmt.Printf("%s: %s\n", dep.ImportPath, *compare.Status)
		for pos, commit := range compare.Commits {
			msg := ""
			if commit.Commit.Message != nil {
				msg = *commit.Commit.Message
				msg = strings.Replace(msg, "\t", " ", -1)
				msg = strings.Replace(msg, "\r", " ", -1)
				msg = strings.Replace(msg, "\n", " ", -1)
				msg = strings.Replace(msg, "  ", " ", -1)
				if len(msg) > 80 {
					msg = msg[:80] + "..."
				}
			}
			sha := *commit.SHA
			fmt.Printf("    %d %s %s\n", pos, sha[0:7], msg)
		}
	}
}
<|endoftext|>