<commit_before>package jsoniter

import (
	"fmt"
	"reflect"
	"sort"
	"strings"
	"unicode"
	"unsafe"
)

var typeDecoders = map[string]ValDecoder{}
var fieldDecoders = map[string]ValDecoder{}
var typeEncoders = map[string]ValEncoder{}
var fieldEncoders = map[string]ValEncoder{}
var extensions = []Extension{}

type StructDescriptor struct {
	onePtrEmbedded     bool
	onePtrOptimization bool
	Type               reflect.Type
	Fields             []*Binding
}

func (structDescriptor *StructDescriptor) GetField(fieldName string) *Binding {
	for _, binding := range structDescriptor.Fields {
		if binding.Field.Name == fieldName {
			return binding
		}
	}
	return nil
}

type Binding struct {
	levels    []int
	Field     *reflect.StructField
	FromNames []string
	ToNames   []string
	Encoder   ValEncoder
	Decoder   ValDecoder
}

type Extension interface {
	UpdateStructDescriptor(structDescriptor *StructDescriptor)
	CreateDecoder(typ reflect.Type) ValDecoder
	CreateEncoder(typ reflect.Type) ValEncoder
	DecorateDecoder(typ reflect.Type, decoder ValDecoder) ValDecoder
	DecorateEncoder(typ reflect.Type, encoder ValEncoder) ValEncoder
}

type DummyExtension struct {
}

func (extension *DummyExtension) UpdateStructDescriptor(structDescriptor *StructDescriptor) {
}

func (extension *DummyExtension) CreateDecoder(typ reflect.Type) ValDecoder {
	return nil
}

func (extension *DummyExtension) CreateEncoder(typ reflect.Type) ValEncoder {
	return nil
}

func (extension *DummyExtension) DecorateDecoder(typ reflect.Type, decoder ValDecoder) ValDecoder {
	return decoder
}

func (extension *DummyExtension) DecorateEncoder(typ reflect.Type, encoder ValEncoder) ValEncoder {
	return encoder
}

type funcDecoder struct {
	fun DecoderFunc
}

func (decoder *funcDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) {
	decoder.fun(ptr, iter)
}

type funcEncoder struct {
	fun         EncoderFunc
	isEmptyFunc func(ptr unsafe.Pointer) bool
}

func (encoder *funcEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
	encoder.fun(ptr, stream)
}

func (encoder *funcEncoder) EncodeInterface(val interface{}, stream *Stream) {
	WriteToStream(val, stream, encoder)
}

func (encoder *funcEncoder) IsEmpty(ptr unsafe.Pointer) bool {
	if encoder.isEmptyFunc == nil {
		return false
	}
	return encoder.isEmptyFunc(ptr)
}

func RegisterTypeDecoderFunc(typ string, fun DecoderFunc) {
	typeDecoders[typ] = &funcDecoder{fun}
}

func RegisterTypeDecoder(typ string, decoder ValDecoder) {
	typeDecoders[typ] = decoder
}

func RegisterFieldDecoderFunc(typ string, field string, fun DecoderFunc) {
	RegisterFieldDecoder(typ, field, &funcDecoder{fun})
}

func RegisterFieldDecoder(typ string, field string, decoder ValDecoder) {
	fieldDecoders[fmt.Sprintf("%s/%s", typ, field)] = decoder
}

func RegisterTypeEncoderFunc(typ string, fun EncoderFunc, isEmptyFunc func(unsafe.Pointer) bool) {
	typeEncoders[typ] = &funcEncoder{fun, isEmptyFunc}
}

func RegisterTypeEncoder(typ string, encoder ValEncoder) {
	typeEncoders[typ] = encoder
}

func RegisterFieldEncoderFunc(typ string, field string, fun EncoderFunc, isEmptyFunc func(unsafe.Pointer) bool) {
	RegisterFieldEncoder(typ, field, &funcEncoder{fun, isEmptyFunc})
}

func RegisterFieldEncoder(typ string, field string, encoder ValEncoder) {
	fieldEncoders[fmt.Sprintf("%s/%s", typ, field)] = encoder
}

func RegisterExtension(extension Extension) {
	extensions = append(extensions, extension)
}

func getTypeDecoderFromExtension(typ reflect.Type) ValDecoder {
	decoder := _getTypeDecoderFromExtension(typ)
	if decoder != nil {
		for _, extension := range extensions {
			decoder = extension.DecorateDecoder(typ, decoder)
		}
	}
	return decoder
}
func _getTypeDecoderFromExtension(typ reflect.Type) ValDecoder {
	for _, extension := range extensions {
		decoder := extension.CreateDecoder(typ)
		if decoder != nil {
			return decoder
		}
	}
	typeName := typ.String()
	decoder := typeDecoders[typeName]
	if decoder != nil {
		return decoder
	}
	if typ.Kind() == reflect.Ptr {
		decoder := typeDecoders[typ.Elem().String()]
		if decoder != nil {
			return &optionalDecoder{typ.Elem(), decoder}
		}
	}
	return nil
}

func getTypeEncoderFromExtension(typ reflect.Type) ValEncoder {
	encoder := _getTypeEncoderFromExtension(typ)
	if encoder != nil {
		for _, extension := range extensions {
			encoder = extension.DecorateEncoder(typ, encoder)
		}
	}
	return encoder
}

func _getTypeEncoderFromExtension(typ reflect.Type) ValEncoder {
	for _, extension := range extensions {
		encoder := extension.CreateEncoder(typ)
		if encoder != nil {
			return encoder
		}
	}
	typeName := typ.String()
	encoder := typeEncoders[typeName]
	if encoder != nil {
		return encoder
	}
	if typ.Kind() == reflect.Ptr {
		encoder := typeEncoders[typ.Elem().String()]
		if encoder != nil {
			return &optionalEncoder{encoder}
		}
	}
	return nil
}

func describeStruct(cfg *frozenConfig, typ reflect.Type) (*StructDescriptor, error) {
	embeddedBindings := []*Binding{}
	bindings := []*Binding{}
	for i := 0; i < typ.NumField(); i++ {
		field := typ.Field(i)
		if field.Anonymous && (field.Tag.Get("json") == "" || strings.Split(field.Tag.Get("json"), ",")[0] == "") {
			if field.Type.Kind() == reflect.Struct {
				structDescriptor, err := describeStruct(cfg, field.Type)
				if err != nil {
					return nil, err
				}
				for _, binding := range structDescriptor.Fields {
					binding.levels = append([]int{i}, binding.levels...)
					omitempty := binding.Encoder.(*structFieldEncoder).omitempty
					binding.Encoder = &structFieldEncoder{&field, binding.Encoder, omitempty}
					binding.Decoder = &structFieldDecoder{&field, binding.Decoder}
					embeddedBindings = append(embeddedBindings, binding)
				}
				continue
			} else if field.Type.Kind() == reflect.Ptr && field.Type.Elem().Kind() == reflect.Struct {
				structDescriptor, err := describeStruct(cfg, field.Type.Elem())
				if err != nil {
					return nil, err
				}
				for _, binding := range structDescriptor.Fields {
					binding.levels = append([]int{i}, binding.levels...)
					omitempty := binding.Encoder.(*structFieldEncoder).omitempty
					binding.Encoder = &optionalEncoder{binding.Encoder}
					binding.Encoder = &structFieldEncoder{&field, binding.Encoder, omitempty}
					binding.Decoder = &deferenceDecoder{field.Type.Elem(), binding.Decoder}
					binding.Decoder = &structFieldDecoder{&field, binding.Decoder}
					embeddedBindings = append(embeddedBindings, binding)
				}
				continue
			}
		}
		tagParts := strings.Split(field.Tag.Get("json"), ",")
		fieldNames := calcFieldNames(field.Name, tagParts[0], string(field.Tag.Get("json")))
		fieldCacheKey := fmt.Sprintf("%s/%s", typ.String(), field.Name)
		decoder := fieldDecoders[fieldCacheKey]
		if decoder == nil {
			var err error
			decoder, err = decoderOfType(cfg, field.Type)
			if err != nil {
				return nil, err
			}
		}
		encoder := fieldEncoders[fieldCacheKey]
		if encoder == nil {
			var err error
			encoder, err = encoderOfType(cfg, field.Type)
			if err != nil {
				return nil, err
			}
			// map is stored as pointer in the struct
			if field.Type.Kind() == reflect.Map {
				encoder = &optionalEncoder{encoder}
			}
		}
		binding := &Binding{
			Field:     &field,
			FromNames: fieldNames,
			ToNames:   fieldNames,
			Decoder:   decoder,
			Encoder:   encoder,
		}
		binding.levels = []int{i}
		bindings = append(bindings, binding)
	}
	return createStructDescriptor(cfg, typ, bindings, embeddedBindings), nil
}
func createStructDescriptor(cfg *frozenConfig, typ reflect.Type, bindings []*Binding, embeddedBindings []*Binding) *StructDescriptor {
	onePtrEmbedded := false
	onePtrOptimization := false
	if typ.NumField() == 1 {
		firstField := typ.Field(0)
		switch firstField.Type.Kind() {
		case reflect.Ptr:
			if firstField.Anonymous && firstField.Type.Elem().Kind() == reflect.Struct {
				onePtrEmbedded = true
			}
			fallthrough
		case reflect.Map:
			onePtrOptimization = true
		case reflect.Struct:
			onePtrOptimization = isStructOnePtr(firstField.Type)
		}
	}
	structDescriptor := &StructDescriptor{
		onePtrEmbedded:     onePtrEmbedded,
		onePtrOptimization: onePtrOptimization,
		Type:               typ,
		Fields:             bindings,
	}
	for _, extension := range extensions {
		extension.UpdateStructDescriptor(structDescriptor)
	}
	processTags(structDescriptor, cfg)
	// merge normal & embedded bindings & sort with original order
	allBindings := sortableBindings(append(embeddedBindings, structDescriptor.Fields...))
	sort.Sort(allBindings)
	structDescriptor.Fields = allBindings
	return structDescriptor
}

func isStructOnePtr(typ reflect.Type) bool {
	if typ.NumField() == 1 {
		firstField := typ.Field(0)
		switch firstField.Type.Kind() {
		case reflect.Ptr:
			return true
		case reflect.Map:
			return true
		case reflect.Struct:
			return isStructOnePtr(firstField.Type)
		}
	}
	return false
}

type sortableBindings []*Binding

func (bindings sortableBindings) Len() int {
	return len(bindings)
}

func (bindings sortableBindings) Less(i, j int) bool {
	left := bindings[i].levels
	right := bindings[j].levels
	k := 0
	for {
		if left[k] < right[k] {
			return true
		} else if left[k] > right[k] {
			return false
		}
		k++
	}
}

func (bindings sortableBindings) Swap(i, j int) {
	bindings[i], bindings[j] = bindings[j], bindings[i]
}

func processTags(structDescriptor *StructDescriptor, cfg *frozenConfig) {
	for _, binding := range structDescriptor.Fields {
		shouldOmitEmpty := false
		tagParts := strings.Split(binding.Field.Tag.Get("json"), ",")
		for _, tagPart := range tagParts[1:] {
			if tagPart == "omitempty" {
				shouldOmitEmpty = true
			} else if tagPart == "string" {
				if binding.Field.Type.Kind() == reflect.String {
					binding.Decoder = &stringModeStringDecoder{binding.Decoder, cfg}
					binding.Encoder = &stringModeStringEncoder{binding.Encoder, cfg}
				} else {
					binding.Decoder = &stringModeNumberDecoder{binding.Decoder}
					binding.Encoder = &stringModeNumberEncoder{binding.Encoder}
				}
			}
		}
		binding.Decoder = &structFieldDecoder{binding.Field, binding.Decoder}
		binding.Encoder = &structFieldEncoder{binding.Field, binding.Encoder, shouldOmitEmpty}
	}
}

func calcFieldNames(originalFieldName string, tagProvidedFieldName string, wholeTag string) []string {
	// ignore?
	if wholeTag == "-" {
		return []string{}
	}
	// rename?
	var fieldNames []string
	if tagProvidedFieldName == "" {
		fieldNames = []string{originalFieldName}
	} else {
		fieldNames = []string{tagProvidedFieldName}
	}
	// private?
	isNotExported := unicode.IsLower(rune(originalFieldName[0]))
	if isNotExported {
		fieldNames = []string{}
	}
	return fieldNames
}
<commit_msg>fix(reflect): don't process ignored struct fields<commit_after>package jsoniter

import (
	"fmt"
	"reflect"
	"sort"
	"strings"
	"unicode"
	"unsafe"
)

var typeDecoders = map[string]ValDecoder{}
var fieldDecoders = map[string]ValDecoder{}
var typeEncoders = map[string]ValEncoder{}
var fieldEncoders = map[string]ValEncoder{}
var extensions = []Extension{}

type StructDescriptor struct {
	onePtrEmbedded     bool
	onePtrOptimization bool
	Type               reflect.Type
	Fields             []*Binding
}

func (structDescriptor *StructDescriptor) GetField(fieldName string) *Binding {
	for _, binding := range structDescriptor.Fields {
		if binding.Field.Name == fieldName {
			return binding
		}
	}
	return nil
}

type Binding struct {
	levels    []int
	Field     *reflect.StructField
	FromNames []string
	ToNames   []string
	Encoder   ValEncoder
	Decoder   ValDecoder
}

type Extension interface {
	UpdateStructDescriptor(structDescriptor *StructDescriptor)
	CreateDecoder(typ reflect.Type) ValDecoder
	CreateEncoder(typ reflect.Type) ValEncoder
	DecorateDecoder(typ reflect.Type, decoder ValDecoder) ValDecoder
	DecorateEncoder(typ reflect.Type, encoder ValEncoder) ValEncoder
}

type DummyExtension struct {
}

func (extension *DummyExtension) UpdateStructDescriptor(structDescriptor *StructDescriptor) {
}

func (extension *DummyExtension) CreateDecoder(typ reflect.Type) ValDecoder {
	return nil
}

func (extension *DummyExtension) CreateEncoder(typ reflect.Type) ValEncoder {
	return nil
}

func (extension *DummyExtension) DecorateDecoder(typ reflect.Type, decoder ValDecoder) ValDecoder {
	return decoder
}

func (extension *DummyExtension) DecorateEncoder(typ reflect.Type, encoder ValEncoder) ValEncoder {
	return encoder
}

type funcDecoder struct {
	fun DecoderFunc
}

func (decoder *funcDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) {
	decoder.fun(ptr, iter)
}

type funcEncoder struct {
	fun         EncoderFunc
	isEmptyFunc func(ptr unsafe.Pointer) bool
}

func (encoder *funcEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
	encoder.fun(ptr, stream)
}

func (encoder *funcEncoder) EncodeInterface(val interface{}, stream *Stream) {
	WriteToStream(val, stream, encoder)
}

func (encoder *funcEncoder) IsEmpty(ptr unsafe.Pointer) bool {
	if encoder.isEmptyFunc == nil {
		return false
	}
	return encoder.isEmptyFunc(ptr)
}

func RegisterTypeDecoderFunc(typ string, fun DecoderFunc) {
	typeDecoders[typ] = &funcDecoder{fun}
}

func RegisterTypeDecoder(typ string, decoder ValDecoder) {
	typeDecoders[typ] = decoder
}

func RegisterFieldDecoderFunc(typ string, field string, fun DecoderFunc) {
	RegisterFieldDecoder(typ, field, &funcDecoder{fun})
}

func RegisterFieldDecoder(typ string, field string, decoder ValDecoder) {
	fieldDecoders[fmt.Sprintf("%s/%s", typ, field)] = decoder
}

func RegisterTypeEncoderFunc(typ string, fun EncoderFunc, isEmptyFunc func(unsafe.Pointer) bool) {
	typeEncoders[typ] = &funcEncoder{fun, isEmptyFunc}
}

func RegisterTypeEncoder(typ string, encoder ValEncoder) {
	typeEncoders[typ] = encoder
}

func RegisterFieldEncoderFunc(typ string, field string, fun EncoderFunc, isEmptyFunc func(unsafe.Pointer) bool) {
	RegisterFieldEncoder(typ, field, &funcEncoder{fun, isEmptyFunc})
}

func RegisterFieldEncoder(typ string, field string, encoder ValEncoder) {
	fieldEncoders[fmt.Sprintf("%s/%s", typ, field)] = encoder
}

func RegisterExtension(extension Extension) {
	extensions = append(extensions, extension)
}

func getTypeDecoderFromExtension(typ reflect.Type) ValDecoder {
	decoder := _getTypeDecoderFromExtension(typ)
	if decoder != nil {
		for _, extension := range extensions {
			decoder = extension.DecorateDecoder(typ, decoder)
		}
	}
	return decoder
}
func _getTypeDecoderFromExtension(typ reflect.Type) ValDecoder {
	for _, extension := range extensions {
		decoder := extension.CreateDecoder(typ)
		if decoder != nil {
			return decoder
		}
	}
	typeName := typ.String()
	decoder := typeDecoders[typeName]
	if decoder != nil {
		return decoder
	}
	if typ.Kind() == reflect.Ptr {
		decoder := typeDecoders[typ.Elem().String()]
		if decoder != nil {
			return &optionalDecoder{typ.Elem(), decoder}
		}
	}
	return nil
}

func getTypeEncoderFromExtension(typ reflect.Type) ValEncoder {
	encoder := _getTypeEncoderFromExtension(typ)
	if encoder != nil {
		for _, extension := range extensions {
			encoder = extension.DecorateEncoder(typ, encoder)
		}
	}
	return encoder
}

func _getTypeEncoderFromExtension(typ reflect.Type) ValEncoder {
	for _, extension := range extensions {
		encoder := extension.CreateEncoder(typ)
		if encoder != nil {
			return encoder
		}
	}
	typeName := typ.String()
	encoder := typeEncoders[typeName]
	if encoder != nil {
		return encoder
	}
	if typ.Kind() == reflect.Ptr {
		encoder := typeEncoders[typ.Elem().String()]
		if encoder != nil {
			return &optionalEncoder{encoder}
		}
	}
	return nil
}

func describeStruct(cfg *frozenConfig, typ reflect.Type) (*StructDescriptor, error) {
	embeddedBindings := []*Binding{}
	bindings := []*Binding{}
	for i := 0; i < typ.NumField(); i++ {
		field := typ.Field(i)
		tag := field.Tag.Get("json")
		tagParts := strings.Split(tag, ",")
		if tag == "-" {
			continue
		}
		if field.Anonymous && (tag == "" || tagParts[0] == "") {
			if field.Type.Kind() == reflect.Struct {
				structDescriptor, err := describeStruct(cfg, field.Type)
				if err != nil {
					return nil, err
				}
				for _, binding := range structDescriptor.Fields {
					binding.levels = append([]int{i}, binding.levels...)
					omitempty := binding.Encoder.(*structFieldEncoder).omitempty
					binding.Encoder = &structFieldEncoder{&field, binding.Encoder, omitempty}
					binding.Decoder = &structFieldDecoder{&field, binding.Decoder}
					embeddedBindings = append(embeddedBindings, binding)
				}
				continue
			} else if field.Type.Kind() == reflect.Ptr && field.Type.Elem().Kind() == reflect.Struct {
				structDescriptor, err := describeStruct(cfg, field.Type.Elem())
				if err != nil {
					return nil, err
				}
				for _, binding := range structDescriptor.Fields {
					binding.levels = append([]int{i}, binding.levels...)
					omitempty := binding.Encoder.(*structFieldEncoder).omitempty
					binding.Encoder = &optionalEncoder{binding.Encoder}
					binding.Encoder = &structFieldEncoder{&field, binding.Encoder, omitempty}
					binding.Decoder = &deferenceDecoder{field.Type.Elem(), binding.Decoder}
					binding.Decoder = &structFieldDecoder{&field, binding.Decoder}
					embeddedBindings = append(embeddedBindings, binding)
				}
				continue
			}
		}
		fieldNames := calcFieldNames(field.Name, tagParts[0], tag)
		fieldCacheKey := fmt.Sprintf("%s/%s", typ.String(), field.Name)
		decoder := fieldDecoders[fieldCacheKey]
		if decoder == nil {
			var err error
			decoder, err = decoderOfType(cfg, field.Type)
			if err != nil {
				return nil, err
			}
		}
		encoder := fieldEncoders[fieldCacheKey]
		if encoder == nil {
			var err error
			encoder, err = encoderOfType(cfg, field.Type)
			if err != nil {
				return nil, err
			}
			// map is stored as pointer in the struct
			if field.Type.Kind() == reflect.Map {
				encoder = &optionalEncoder{encoder}
			}
		}
		binding := &Binding{
			Field:     &field,
			FromNames: fieldNames,
			ToNames:   fieldNames,
			Decoder:   decoder,
			Encoder:   encoder,
		}
		binding.levels = []int{i}
		bindings = append(bindings, binding)
	}
	return createStructDescriptor(cfg, typ, bindings, embeddedBindings), nil
}
func createStructDescriptor(cfg *frozenConfig, typ reflect.Type, bindings []*Binding, embeddedBindings []*Binding) *StructDescriptor {
	onePtrEmbedded := false
	onePtrOptimization := false
	if typ.NumField() == 1 {
		firstField := typ.Field(0)
		switch firstField.Type.Kind() {
		case reflect.Ptr:
			if firstField.Anonymous && firstField.Type.Elem().Kind() == reflect.Struct {
				onePtrEmbedded = true
			}
			fallthrough
		case reflect.Map:
			onePtrOptimization = true
		case reflect.Struct:
			onePtrOptimization = isStructOnePtr(firstField.Type)
		}
	}
	structDescriptor := &StructDescriptor{
		onePtrEmbedded:     onePtrEmbedded,
		onePtrOptimization: onePtrOptimization,
		Type:               typ,
		Fields:             bindings,
	}
	for _, extension := range extensions {
		extension.UpdateStructDescriptor(structDescriptor)
	}
	processTags(structDescriptor, cfg)
	// merge normal & embedded bindings & sort with original order
	allBindings := sortableBindings(append(embeddedBindings, structDescriptor.Fields...))
	sort.Sort(allBindings)
	structDescriptor.Fields = allBindings
	return structDescriptor
}

func isStructOnePtr(typ reflect.Type) bool {
	if typ.NumField() == 1 {
		firstField := typ.Field(0)
		switch firstField.Type.Kind() {
		case reflect.Ptr:
			return true
		case reflect.Map:
			return true
		case reflect.Struct:
			return isStructOnePtr(firstField.Type)
		}
	}
	return false
}

type sortableBindings []*Binding

func (bindings sortableBindings) Len() int {
	return len(bindings)
}

func (bindings sortableBindings) Less(i, j int) bool {
	left := bindings[i].levels
	right := bindings[j].levels
	k := 0
	for {
		if left[k] < right[k] {
			return true
		} else if left[k] > right[k] {
			return false
		}
		k++
	}
}

func (bindings sortableBindings) Swap(i, j int) {
	bindings[i], bindings[j] = bindings[j], bindings[i]
}

func processTags(structDescriptor *StructDescriptor, cfg *frozenConfig) {
	for _, binding := range structDescriptor.Fields {
		shouldOmitEmpty := false
		tagParts := strings.Split(binding.Field.Tag.Get("json"), ",")
		for _, tagPart := range tagParts[1:] {
			if tagPart == "omitempty" {
				shouldOmitEmpty = true
			} else if tagPart == "string" {
				if binding.Field.Type.Kind() == reflect.String {
					binding.Decoder = &stringModeStringDecoder{binding.Decoder, cfg}
					binding.Encoder = &stringModeStringEncoder{binding.Encoder, cfg}
				} else {
					binding.Decoder = &stringModeNumberDecoder{binding.Decoder}
					binding.Encoder = &stringModeNumberEncoder{binding.Encoder}
				}
			}
		}
		binding.Decoder = &structFieldDecoder{binding.Field, binding.Decoder}
		binding.Encoder = &structFieldEncoder{binding.Field, binding.Encoder, shouldOmitEmpty}
	}
}

func calcFieldNames(originalFieldName string, tagProvidedFieldName string, wholeTag string) []string {
	// ignore?
	if wholeTag == "-" {
		return []string{}
	}
	// rename?
	var fieldNames []string
	if tagProvidedFieldName == "" {
		fieldNames = []string{originalFieldName}
	} else {
		fieldNames = []string{tagProvidedFieldName}
	}
	// private?
	isNotExported := unicode.IsLower(rune(originalFieldName[0]))
	if isNotExported {
		fieldNames = []string{}
	}
	return fieldNames
}
<|endoftext|>


<commit_before>package gc

import (
	"net/http"
	"net/url"
)

// The User Agent to send to the upstream
var DefaultUserAgent = []string{""}

// Tweaks request `out` before sending it to the upstream
type RequestTweaker func(in *Request, out *http.Request)

type Upstream struct {
	Name      string
	Address   string
	Transport *http.Transport
	Headers   []string
	Tweaker   RequestTweaker
}

// Issues a request to the upstream
func (u *Upstream) RoundTrip(in *Request) (*http.Response, error) {
	return u.Transport.RoundTrip(u.createRequest(in))
}

func (u *Upstream) createRequest(in *Request) *http.Request {
	targetUrl, err := url.Parse(u.Address + in.URL.RequestURI())
	if err != nil {
		in.Error("upstream url %s %v", u.Address+in.URL.RequestURI(), err)
		targetUrl = in.URL
	}
	out := &http.Request{
		URL:           targetUrl,
		Close:         false,
		Proto:         "HTTP/1.1",
		ProtoMajor:    1,
		ProtoMinor:    1,
		Host:          in.Host,
		Body:          in.Body,
		Method:        in.Method,
		ContentLength: in.ContentLength,
		Header:        http.Header{"X-Request-Id": []string{in.Id}, "User-Agent": DefaultUserAgent},
	}

	for _, k := range u.Headers {
		value := in.Header[k]
		if len(value) > 0 {
			out.Header[k] = value
		}
	}

	if u.Tweaker != nil {
		u.Tweaker(in, out)
	}
	return out
}
<commit_msg>set X-Forwarded-For<commit_after>package gc

import (
	"net"
	"strings"
	"net/http"
	"net/url"
)

// The User Agent to send to the upstream
var DefaultUserAgent = []string{""}

// Tweaks request `out` before sending it to the upstream
type RequestTweaker func(in *Request, out *http.Request)

type Upstream struct {
	Name      string
	Address   string
	Transport *http.Transport
	Headers   []string
	Tweaker   RequestTweaker
}

// Issues a request to the upstream
func (u *Upstream) RoundTrip(in *Request) (*http.Response, error) {
	return u.Transport.RoundTrip(u.createRequest(in))
}

func (u *Upstream) createRequest(in *Request) *http.Request {
	targetUrl, err := url.Parse(u.Address + in.URL.RequestURI())
	if err != nil {
		in.Error("upstream url %s %v", u.Address+in.URL.RequestURI(), err)
		targetUrl = in.URL
	}
	out := &http.Request{
		URL:           targetUrl,
		Close:         false,
		Proto:         "HTTP/1.1",
		ProtoMajor:    1,
		ProtoMinor:    1,
		Host:          in.Host,
		Body:          in.Body,
		Method:        in.Method,
		ContentLength: in.ContentLength,
		Header:        http.Header{"X-Request-Id": []string{in.Id}, "User-Agent": DefaultUserAgent},
	}

	for _, k := range u.Headers {
		value := in.Header[k]
		if len(value) > 0 {
			out.Header[k] = value
		}
	}

	if clientIP, _, err := net.SplitHostPort(in.RemoteAddr); err == nil {
		if prior, ok := out.Header["X-Forwarded-For"]; ok {
			clientIP = strings.Join(prior, ", ") + ", " + clientIP
		}
		out.Header.Set("X-Forwarded-For", clientIP)
	}

	if u.Tweaker != nil {
		u.Tweaker(in, out)
	}
	return out
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package common

import (
	"fmt"
	"sort"

	"github.com/juju/errors"
	"github.com/juju/names"
	"github.com/juju/version"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	envtools "github.com/juju/juju/environs/tools"
	"github.com/juju/juju/network"
	"github.com/juju/juju/state"
	"github.com/juju/juju/state/binarystorage"
	coretools "github.com/juju/juju/tools"
)

var envtoolsFindTools = envtools.FindTools

// ToolsURLGetter is an interface providing the ToolsURL method.
type ToolsURLGetter interface {
	// ToolsURLs returns URLs for the tools with
	// the specified binary version.
	ToolsURLs(v version.Binary) ([]string, error)
}

type ModelConfigGetter interface {
	ModelConfig() (*config.Config, error)
}

// APIHostPortsGetter is an interface providing the APIHostPorts method.
type APIHostPortsGetter interface {
	// APIHostPorst returns the HostPorts for each API server.
	APIHostPorts() ([][]network.HostPort, error)
}

// ToolsStorageGetter is an interface providing the ToolsStorage method.
type ToolsStorageGetter interface {
	// ToolsStorage returns a binarystorage.StorageCloser.
	ToolsStorage() (binarystorage.StorageCloser, error)
}

// ToolsGetter implements a common Tools method for use by various
// facades.
type ToolsGetter struct {
	entityFinder       state.EntityFinder
	configGetter       ModelConfigGetter
	toolsStorageGetter ToolsStorageGetter
	urlGetter          ToolsURLGetter
	getCanRead         GetAuthFunc
}

// NewToolsGetter returns a new ToolsGetter. The GetAuthFunc will be
// used on each invocation of Tools to determine current permissions.
func NewToolsGetter(f state.EntityFinder, c ModelConfigGetter, s ToolsStorageGetter, t ToolsURLGetter, getCanRead GetAuthFunc) *ToolsGetter {
	return &ToolsGetter{f, c, s, t, getCanRead}
}

// Tools finds the tools necessary for the given agents.
func (t *ToolsGetter) Tools(args params.Entities) (params.ToolsResults, error) {
	result := params.ToolsResults{
		Results: make([]params.ToolsResult, len(args.Entities)),
	}
	canRead, err := t.getCanRead()
	if err != nil {
		return result, err
	}
	agentVersion, err := t.getGlobalAgentVersion()
	if err != nil {
		return result, err
	}
	toolsStorage, err := t.toolsStorageGetter.ToolsStorage()
	if err != nil {
		return result, err
	}
	defer toolsStorage.Close()

	for i, entity := range args.Entities {
		tag, err := names.ParseTag(entity.Tag)
		if err != nil {
			result.Results[i].Error = ServerError(ErrPerm)
			continue
		}
		agentToolsList, err := t.oneAgentTools(canRead, tag, agentVersion, toolsStorage)
		if err == nil {
			result.Results[i].ToolsList = agentToolsList
			// TODO(axw) Get rid of this in 1.22, when all upgraders
			// are known to ignore the flag.
			result.Results[i].DisableSSLHostnameVerification = true
		}
		result.Results[i].Error = ServerError(err)
	}
	return result, nil
}

func (t *ToolsGetter) getGlobalAgentVersion() (version.Number, error) {
	// Get the Agent Version requested in the Environment Config
	nothing := version.Number{}
	cfg, err := t.configGetter.ModelConfig()
	if err != nil {
		return nothing, err
	}
	agentVersion, ok := cfg.AgentVersion()
	if !ok {
		return nothing, errors.New("agent version not set in model config")
	}
	return agentVersion, nil
}

func (t *ToolsGetter) oneAgentTools(canRead AuthFunc, tag names.Tag, agentVersion version.Number, storage binarystorage.Storage) (coretools.List, error) {
	if !canRead(tag) {
		return nil, ErrPerm
	}
	entity, err := t.entityFinder.FindEntity(tag)
	if err != nil {
		return nil, err
	}
	tooler, ok := entity.(state.AgentTooler)
	if !ok {
		return nil, NotSupportedError(tag, "agent tools")
	}
	existingTools, err := tooler.AgentTools()
	if err != nil {
		return nil, err
	}
	toolsFinder := NewToolsFinder(t.configGetter, t.toolsStorageGetter, t.urlGetter)
	list, err := toolsFinder.findTools(params.FindToolsParams{
		Number:       agentVersion,
		MajorVersion: -1,
		MinorVersion: -1,
		Series:       existingTools.Version.Series,
		Arch:         existingTools.Version.Arch,
	})
	if err != nil {
		return nil, err
	}
	return list, nil
}

// ToolsSetter implements a common Tools method for use by various
// facades.
type ToolsSetter struct {
	st          state.EntityFinder
	getCanWrite GetAuthFunc
}

// NewToolsSetter returns a new ToolsGetter. The GetAuthFunc will be
// used on each invocation of Tools to determine current permissions.
func NewToolsSetter(st state.EntityFinder, getCanWrite GetAuthFunc) *ToolsSetter {
	return &ToolsSetter{
		st:          st,
		getCanWrite: getCanWrite,
	}
}

// SetTools updates the recorded tools version for the agents.
func (t *ToolsSetter) SetTools(args params.EntitiesVersion) (params.ErrorResults, error) {
	results := params.ErrorResults{
		Results: make([]params.ErrorResult, len(args.AgentTools)),
	}
	canWrite, err := t.getCanWrite()
	if err != nil {
		return results, errors.Trace(err)
	}
	for i, agentTools := range args.AgentTools {
		tag, err := names.ParseTag(agentTools.Tag)
		if err != nil {
			results.Results[i].Error = ServerError(ErrPerm)
			continue
		}
		err = t.setOneAgentVersion(tag, agentTools.Tools.Version, canWrite)
		results.Results[i].Error = ServerError(err)
	}
	return results, nil
}

func (t *ToolsSetter) setOneAgentVersion(tag names.Tag, vers version.Binary, canWrite AuthFunc) error {
	if !canWrite(tag) {
		return ErrPerm
	}
	entity0, err := t.st.FindEntity(tag)
	if err != nil {
		return err
	}
	entity, ok := entity0.(state.AgentTooler)
	if !ok {
		return NotSupportedError(tag, "agent tools")
	}
	return entity.SetAgentVersion(vers)
}

type ToolsFinder struct {
	configGetter       ModelConfigGetter
	toolsStorageGetter ToolsStorageGetter
	urlGetter          ToolsURLGetter
}

// NewToolsFinder returns a new ToolsFinder, returning tools
// with their URLs pointing at the API server.
func NewToolsFinder(c ModelConfigGetter, s ToolsStorageGetter, t ToolsURLGetter) *ToolsFinder {
	return &ToolsFinder{c, s, t}
}

// FindTools returns a List containing all tools matching the given parameters.
func (f *ToolsFinder) FindTools(args params.FindToolsParams) (params.FindToolsResult, error) {
	result := params.FindToolsResult{}
	list, err := f.findTools(args)
	if err != nil {
		result.Error = ServerError(err)
	} else {
		result.List = list
	}
	return result, nil
}

// findTools calls findMatchingTools and then rewrites the URLs
// using the provided ToolsURLGetter.
func (f *ToolsFinder) findTools(args params.FindToolsParams) (coretools.List, error) {
	list, err := f.findMatchingTools(args)
	if err != nil {
		return nil, err
	}
	// Rewrite the URLs so they point at the API server. If the
	// tools are not in tools storage, then the API server will
	// download and cache them if the client requests that version.
	var fullList coretools.List
	for _, baseTools := range list {
		urls, err := f.urlGetter.ToolsURLs(baseTools.Version)
		if err != nil {
			return nil, err
		}
		for _, url := range urls {
			tools := *baseTools
			tools.URL = url
			fullList = append(fullList, &tools)
		}
	}
	return fullList, nil
}

// findMatchingTools searches tools storage and simplestreams for tools matching the
// given parameters. If an exact match is specified (number, series and arch)
// and is found in tools storage, then simplestreams will not be searched.
func (f *ToolsFinder) findMatchingTools(args params.FindToolsParams) (coretools.List, error) {
	exactMatch := args.Number != version.Zero && args.Series != "" && args.Arch != ""
	storageList, err := f.matchingStorageTools(args)
	if err == nil && exactMatch {
		return storageList, nil
	} else if err != nil && err != coretools.ErrNoMatches {
		return nil, err
	}

	// Look for tools in simplestreams too, but don't replace
	// any versions found in storage.
	cfg, err := f.configGetter.ModelConfig()
	if err != nil {
		return nil, err
	}
	env, err := environs.New(cfg)
	if err != nil {
		return nil, err
	}
	filter := toolsFilter(args)
	stream := envtools.PreferredStream(&args.Number, cfg.Development(), cfg.AgentStream())
	simplestreamsList, err := envtoolsFindTools(
		env, args.MajorVersion, args.MinorVersion, stream, filter,
	)
	if len(storageList) == 0 && err != nil {
		return nil, err
	}

	list := storageList
	found := make(map[version.Binary]bool)
	for _, tools := range storageList {
		found[tools.Version] = true
	}
	for _, tools := range simplestreamsList {
		if !found[tools.Version] {
			list = append(list, tools)
		}
	}
	sort.Sort(list)
	return list, nil
}

// matchingStorageTools returns a coretools.List, with an entry for each
// metadata entry in the tools storage that matches the given parameters.
func (f *ToolsFinder) matchingStorageTools(args params.FindToolsParams) (coretools.List, error) {
	storage, err := f.toolsStorageGetter.ToolsStorage()
	if err != nil {
		return nil, err
	}
	defer storage.Close()
	allMetadata, err := storage.AllMetadata()
	if err != nil {
		return nil, err
	}
	list := make(coretools.List, len(allMetadata))
	for i, m := range allMetadata {
		vers, err := version.ParseBinary(m.Version)
		if err != nil {
			return nil, errors.Annotatef(err, "unexpectedly bad version %q in tools storage", m.Version)
		}
		list[i] = &coretools.Tools{
			Version: vers,
			Size:    m.Size,
			SHA256:  m.SHA256,
		}
	}
	list, err = list.Match(toolsFilter(args))
	if err != nil {
		return nil, err
	}
	var matching coretools.List
	for _, tools := range list {
		if args.MajorVersion != -1 && tools.Version.Major != args.MajorVersion {
			continue
		}
		if args.MinorVersion != -1 && tools.Version.Minor != args.MinorVersion {
			continue
		}
		matching = append(matching, tools)
	}
	if len(matching) == 0 {
		return nil, coretools.ErrNoMatches
	}
	return matching, nil
}

func toolsFilter(args params.FindToolsParams) coretools.Filter {
	return coretools.Filter{
		Number: args.Number,
		Arch:   args.Arch,
		Series: args.Series,
	}
}

type toolsURLGetter struct {
	modelUUID          string
	apiHostPortsGetter APIHostPortsGetter
}

// NewToolsURLGetter creates a new ToolsURLGetter that
// returns tools URLs pointing at an API server.
func NewToolsURLGetter(modelUUID string, a APIHostPortsGetter) *toolsURLGetter {
	return &toolsURLGetter{modelUUID, a}
}

func (t *toolsURLGetter) ToolsURLs(v version.Binary) ([]string, error) {
	addrs, err := apiAddresses(t.apiHostPortsGetter)
	if err != nil {
		return nil, err
	}
	if len(addrs) == 0 {
		return nil, errors.Errorf("no suitable API server address to pick from %v")
	}
	var urls []string
	for _, addr := range addrs {
		serverRoot := fmt.Sprintf("https://%s/model/%s", addr, t.modelUUID)
		url := ToolsURL(serverRoot, v)
		urls = append(urls, url)
	}
	return urls, nil
}

// ToolsURL returns a tools URL pointing the API server
// specified by the "serverRoot".
func ToolsURL(serverRoot string, v version.Binary) string {
	return fmt.Sprintf("%s/tools/%s", serverRoot, v.String())
}
<commit_msg>Fix a typo.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package common

import (
	"fmt"
	"sort"

	"github.com/juju/errors"
	"github.com/juju/names"
	"github.com/juju/version"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	envtools "github.com/juju/juju/environs/tools"
	"github.com/juju/juju/network"
	"github.com/juju/juju/state"
	"github.com/juju/juju/state/binarystorage"
	coretools "github.com/juju/juju/tools"
)

var envtoolsFindTools = envtools.FindTools

// ToolsURLGetter is an interface providing the ToolsURL method.
type ToolsURLGetter interface {
	// ToolsURLs returns URLs for the tools with
	// the specified binary version.
	ToolsURLs(v version.Binary) ([]string, error)
}

type ModelConfigGetter interface {
	ModelConfig() (*config.Config, error)
}

// APIHostPortsGetter is an interface providing the APIHostPorts method.
type APIHostPortsGetter interface {
	// APIHostPorst returns the HostPorts for each API server.
	APIHostPorts() ([][]network.HostPort, error)
}

// ToolsStorageGetter is an interface providing the ToolsStorage method.
type ToolsStorageGetter interface {
	// ToolsStorage returns a binarystorage.StorageCloser.
	ToolsStorage() (binarystorage.StorageCloser, error)
}

// ToolsGetter implements a common Tools method for use by various
// facades.
type ToolsGetter struct {
	entityFinder       state.EntityFinder
	configGetter       ModelConfigGetter
	toolsStorageGetter ToolsStorageGetter
	urlGetter          ToolsURLGetter
	getCanRead         GetAuthFunc
}

// NewToolsGetter returns a new ToolsGetter. The GetAuthFunc will be
// used on each invocation of Tools to determine current permissions.
func NewToolsGetter(f state.EntityFinder, c ModelConfigGetter, s ToolsStorageGetter, t ToolsURLGetter, getCanRead GetAuthFunc) *ToolsGetter {
	return &ToolsGetter{f, c, s, t, getCanRead}
}

// Tools finds the tools necessary for the given agents.
func (t *ToolsGetter) Tools(args params.Entities) (params.ToolsResults, error) {
	result := params.ToolsResults{
		Results: make([]params.ToolsResult, len(args.Entities)),
	}
	canRead, err := t.getCanRead()
	if err != nil {
		return result, err
	}
	agentVersion, err := t.getGlobalAgentVersion()
	if err != nil {
		return result, err
	}
	toolsStorage, err := t.toolsStorageGetter.ToolsStorage()
	if err != nil {
		return result, err
	}
	defer toolsStorage.Close()

	for i, entity := range args.Entities {
		tag, err := names.ParseTag(entity.Tag)
		if err != nil {
			result.Results[i].Error = ServerError(ErrPerm)
			continue
		}
		agentToolsList, err := t.oneAgentTools(canRead, tag, agentVersion, toolsStorage)
		if err == nil {
			result.Results[i].ToolsList = agentToolsList
			// TODO(axw) Get rid of this in 1.22, when all upgraders
			// are known to ignore the flag.
			result.Results[i].DisableSSLHostnameVerification = true
		}
		result.Results[i].Error = ServerError(err)
	}
	return result, nil
}

func (t *ToolsGetter) getGlobalAgentVersion() (version.Number, error) {
	// Get the Agent Version requested in the Environment Config
	nothing := version.Number{}
	cfg, err := t.configGetter.ModelConfig()
	if err != nil {
		return nothing, err
	}
	agentVersion, ok := cfg.AgentVersion()
	if !ok {
		return nothing, errors.New("agent version not set in model config")
	}
	return agentVersion, nil
}

func (t *ToolsGetter) oneAgentTools(canRead AuthFunc, tag names.Tag, agentVersion version.Number, storage binarystorage.Storage) (coretools.List, error) {
	if !canRead(tag) {
		return nil, ErrPerm
	}
	entity, err := t.entityFinder.FindEntity(tag)
	if err != nil {
		return nil, err
	}
	tooler, ok := entity.(state.AgentTooler)
	if !ok {
		return nil, NotSupportedError(tag, "agent tools")
	}
	existingTools, err := tooler.AgentTools()
	if err != nil {
		return nil, err
	}
	toolsFinder := NewToolsFinder(t.configGetter, t.toolsStorageGetter, t.urlGetter)
	list, err := toolsFinder.findTools(params.FindToolsParams{
		Number:       agentVersion,
		MajorVersion: -1,
		MinorVersion: -1,
		Series:       existingTools.Version.Series,
		Arch:         existingTools.Version.Arch,
	})
	if err != nil {
		return nil, err
	}
	return list, nil
}

// ToolsSetter implements a common Tools method for use by various
// facades.
type ToolsSetter struct {
	st          state.EntityFinder
	getCanWrite GetAuthFunc
}

// NewToolsSetter returns a new ToolsGetter. The GetAuthFunc will be
// used on each invocation of Tools to determine current permissions.
func NewToolsSetter(st state.EntityFinder, getCanWrite GetAuthFunc) *ToolsSetter {
	return &ToolsSetter{
		st:          st,
		getCanWrite: getCanWrite,
	}
}

// SetTools updates the recorded tools version for the agents.
func (t *ToolsSetter) SetTools(args params.EntitiesVersion) (params.ErrorResults, error) {
	results := params.ErrorResults{
		Results: make([]params.ErrorResult, len(args.AgentTools)),
	}
	canWrite, err := t.getCanWrite()
	if err != nil {
		return results, errors.Trace(err)
	}
	for i, agentTools := range args.AgentTools {
		tag, err := names.ParseTag(agentTools.Tag)
		if err != nil {
			results.Results[i].Error = ServerError(ErrPerm)
			continue
		}
		err = t.setOneAgentVersion(tag, agentTools.Tools.Version, canWrite)
		results.Results[i].Error = ServerError(err)
	}
	return results, nil
}

func (t *ToolsSetter) setOneAgentVersion(tag names.Tag, vers version.Binary, canWrite AuthFunc) error {
	if !canWrite(tag) {
		return ErrPerm
	}
	entity0, err := t.st.FindEntity(tag)
	if err != nil {
		return err
	}
	entity, ok := entity0.(state.AgentTooler)
	if !ok {
		return NotSupportedError(tag, "agent tools")
	}
	return entity.SetAgentVersion(vers)
}

type ToolsFinder struct {
	configGetter       ModelConfigGetter
	toolsStorageGetter ToolsStorageGetter
	urlGetter          ToolsURLGetter
}

// NewToolsFinder returns a new ToolsFinder, returning tools
// with their URLs pointing at the API server.
func NewToolsFinder(c ModelConfigGetter, s ToolsStorageGetter, t ToolsURLGetter) *ToolsFinder {
	return &ToolsFinder{c, s, t}
}

// FindTools returns a List containing all tools matching the given parameters.
func (f *ToolsFinder) FindTools(args params.FindToolsParams) (params.FindToolsResult, error) {
	result := params.FindToolsResult{}
	list, err := f.findTools(args)
	if err != nil {
		result.Error = ServerError(err)
	} else {
		result.List = list
	}
	return result, nil
}

// findTools calls findMatchingTools and then rewrites the URLs
// using the provided ToolsURLGetter.
func (f *ToolsFinder) findTools(args params.FindToolsParams) (coretools.List, error) {
	list, err := f.findMatchingTools(args)
	if err != nil {
		return nil, err
	}
	// Rewrite the URLs so they point at the API servers. If the
	// tools are not in tools storage, then the API server will
	// download and cache them if the client requests that version.
	var fullList coretools.List
	for _, baseTools := range list {
		urls, err := f.urlGetter.ToolsURLs(baseTools.Version)
		if err != nil {
			return nil, err
		}
		for _, url := range urls {
			tools := *baseTools
			tools.URL = url
			fullList = append(fullList, &tools)
		}
	}
	return fullList, nil
}

// findMatchingTools searches tools storage and simplestreams for tools matching the
// given parameters. If an exact match is specified (number, series and arch)
// and is found in tools storage, then simplestreams will not be searched.
func (f *ToolsFinder) findMatchingTools(args params.FindToolsParams) (coretools.List, error) {
	exactMatch := args.Number != version.Zero && args.Series != "" && args.Arch != ""
	storageList, err := f.matchingStorageTools(args)
	if err == nil && exactMatch {
		return storageList, nil
	} else if err != nil && err != coretools.ErrNoMatches {
		return nil, err
	}

	// Look for tools in simplestreams too, but don't replace
	// any versions found in storage.
	cfg, err := f.configGetter.ModelConfig()
	if err != nil {
		return nil, err
	}
	env, err := environs.New(cfg)
	if err != nil {
		return nil, err
	}
	filter := toolsFilter(args)
	stream := envtools.PreferredStream(&args.Number, cfg.Development(), cfg.AgentStream())
	simplestreamsList, err := envtoolsFindTools(
		env, args.MajorVersion, args.MinorVersion, stream, filter,
	)
	if len(storageList) == 0 && err != nil {
		return nil, err
	}

	list := storageList
	found := make(map[version.Binary]bool)
	for _, tools := range storageList {
		found[tools.Version] = true
	}
	for _, tools := range simplestreamsList {
		if !found[tools.Version] {
			list = append(list, tools)
		}
	}
	sort.Sort(list)
	return list, nil
}

// matchingStorageTools returns a coretools.List, with an entry for each
// metadata entry in the tools storage that matches the given parameters.
func (f *ToolsFinder) matchingStorageTools(args params.FindToolsParams) (coretools.List, error) {
	storage, err := f.toolsStorageGetter.ToolsStorage()
	if err != nil {
		return nil, err
	}
	defer storage.Close()
	allMetadata, err := storage.AllMetadata()
	if err != nil {
		return nil, err
	}
	list := make(coretools.List, len(allMetadata))
	for i, m := range allMetadata {
		vers, err := version.ParseBinary(m.Version)
		if err != nil {
			return nil, errors.Annotatef(err, "unexpectedly bad version %q in tools storage", m.Version)
		}
		list[i] = &coretools.Tools{
			Version: vers,
			Size:    m.Size,
			SHA256:  m.SHA256,
		}
	}
	list, err = list.Match(toolsFilter(args))
	if err != nil {
		return nil, err
	}
	var matching coretools.List
	for _, tools := range list {
		if args.MajorVersion != -1 && tools.Version.Major != args.MajorVersion {
			continue
		}
		if args.MinorVersion != -1 && tools.Version.Minor != args.MinorVersion {
			continue
		}
		matching = append(matching, tools)
	}
	if len(matching) == 0 {
		return nil, coretools.ErrNoMatches
	}
	return matching, nil
}

func toolsFilter(args params.FindToolsParams) coretools.Filter {
	return coretools.Filter{
		Number: args.Number,
		Arch:   args.Arch,
		Series: args.Series,
	}
}

type toolsURLGetter struct {
	modelUUID          string
	apiHostPortsGetter APIHostPortsGetter
}

// NewToolsURLGetter creates a new ToolsURLGetter that
// returns tools URLs pointing at an API server.
func NewToolsURLGetter(modelUUID string, a APIHostPortsGetter) *toolsURLGetter {
	return &toolsURLGetter{modelUUID, a}
}

func (t *toolsURLGetter) ToolsURLs(v version.Binary) ([]string, error) {
	addrs, err := apiAddresses(t.apiHostPortsGetter)
	if err != nil {
		return nil, err
	}
	if len(addrs) == 0 {
		return nil, errors.Errorf("no suitable API server address to pick from %v")
	}
	var urls []string
	for _, addr := range addrs {
		serverRoot := fmt.Sprintf("https://%s/model/%s", addr, t.modelUUID)
		url := ToolsURL(serverRoot, v)
		urls = append(urls, url)
	}
	return urls, nil
}

// ToolsURL returns a tools URL pointing the API server
// specified by the "serverRoot".
func ToolsURL(serverRoot string, v version.Binary) string {
	return fmt.Sprintf("%s/tools/%s", serverRoot, v.String())
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Soichiro Kashima
// Licensed under MIT license.

package fint_test

import (
	"github.com/ksoichiro/fint"
	"os"
	"testing"
)

func TestExecuteAsCommand(t *testing.T) {
	var err error
	os.Setenv("TERM", "dumb")
	err = fint.ExecuteAsCommand(&fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "default", Id: "objc"})
	if err == nil {
		t.Errorf("Expected error but not occurred")
	}

	os.Setenv("TERM", "xterm-256color")
	fint.ExecuteAsCommand(&fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "default", Id: "objc"})
	if err == nil {
		t.Errorf("Expected error but not occurred")
	}
}

func TestExecute(t *testing.T) {
	testExecuteNormal(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "default", Id: "objc"}, 20)
	testExecuteNormal(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "ja", Id: "objc"}, 20)
	testExecuteNormal(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample_Empty", ConfigPath: "conf/config.json", Locale: "ja", Id: "objc"}, 0)
	testExecuteNormal(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample_SingleError", ConfigPath: "conf/config.json", Locale: "ja", Id: "objc"}, 1)
}

func TestExecuteError(t *testing.T) {
	testExecuteError(t, &fint.Opt{SrcRoot: "", ConfigPath: "conf/config.json", Locale: "default", Id: "objc"},
		"fint: source directory is required.")
	testExecuteError(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "default", Id: ""},
		"fint: ID of the rule set is required.")
	testExecuteError(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "", Locale: "default", Id: "objc"},
		"open : no such file or directory")
	testExecuteError(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "default", Id: "foo"},
		"fint: no matching ruleset to [foo]")
}

func TestCheckSourceFile(t *testing.T) {
	filename := "testdata/non_existent_file"
	_, err := fint.CheckSourceFile(filename, fint.RuleSet{})
	if err == nil {
		t.Errorf("Expected error but not occurred")
	}
	msg := "fint: cannot open " + filename
	if err.Error() != msg {
		t.Errorf("Expected error message [%s] but was [%s]", msg, err.Error())
	}
}

func TestSetbufsize(t *testing.T) {
	var (
		filename = "testdata/objc/FintExample/FintExample/FEAppDelegate.m"
		msg      string
	)
	fint.Setbufsize(0)
	_, err := fint.CheckSourceFile(filename, fint.RuleSet{})
	if err != nil {
		t.Errorf("Unexpected error occurred: %v", err)
	}

	fint.Setbufsize(1)
	_, err = fint.CheckSourceFile(filename, fint.RuleSet{})
	if err == nil {
		t.Errorf("Expected error but not occurred")
	}
	msg = "fint: too long line: " + filename
	if err.Error() != msg {
		t.Errorf("Expected error message [%s] but was [%s]", msg, err.Error())
	}
}

func testExecuteNormal(t *testing.T, opt *fint.Opt, expectedViolations int) {
	v, _ := fint.Execute(opt)
	if len(v) != expectedViolations {
		t.Errorf("Expected violations are [%d] but [%d] found", expectedViolations, len(v))
	}
}

func testExecuteError(t *testing.T, opt *fint.Opt, msg string) {
	_, err := fint.Execute(opt)
	if err == nil {
		t.Errorf("Expected error but not occurred")
		return
	}
	if err.Error() != msg {
		t.Errorf("Expected error message [%s] but was [%s]", msg, err.Error())
	}
}
<commit_msg>Added single error test for command line execution.<commit_after>// Copyright (c) 2014 Soichiro Kashima
// Licensed under MIT license.

package fint_test

import (
	"github.com/ksoichiro/fint"
	"os"
	"testing"
)

func TestExecuteAsCommand(t *testing.T) {
	var err error
	os.Setenv("TERM", "dumb")
	err = fint.ExecuteAsCommand(&fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "default", Id: "objc"})
	if err == nil {
		t.Errorf("Expected error but not occurred")
	}

	os.Setenv("TERM", "xterm-256color")
	fint.ExecuteAsCommand(&fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "default", Id: "objc"})
	if err == nil {
		t.Errorf("Expected error but not occurred")
	}

	fint.ExecuteAsCommand(&fint.Opt{SrcRoot: "testdata/objc/FintExample_SingleError", ConfigPath: "conf/config.json", Locale: "default", Id: "objc"})
	if err == nil {
		t.Errorf("Expected error but not occurred")
	}
}

func TestExecute(t *testing.T) {
	testExecuteNormal(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "default", Id: "objc"}, 20)
	testExecuteNormal(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "ja", Id: "objc"}, 20)
	testExecuteNormal(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample_Empty", ConfigPath: "conf/config.json", Locale: "ja", Id: "objc"}, 0)
	testExecuteNormal(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample_SingleError", ConfigPath: "conf/config.json", Locale: "ja", Id: "objc"}, 1)
}

func TestExecuteError(t *testing.T) {
	testExecuteError(t, &fint.Opt{SrcRoot: "", ConfigPath: "conf/config.json", Locale: "default", Id: "objc"},
		"fint: source directory is required.")
	testExecuteError(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "default", Id: ""},
		"fint: ID of the rule set is required.")
	testExecuteError(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "", Locale: "default", Id: "objc"},
		"open : no such file or directory")
	testExecuteError(t, &fint.Opt{SrcRoot: "testdata/objc/FintExample", ConfigPath: "conf/config.json", Locale: "default", Id: "foo"},
		"fint: no matching ruleset to [foo]")
}

func TestCheckSourceFile(t *testing.T) {
	filename := "testdata/non_existent_file"
	_, err := fint.CheckSourceFile(filename, fint.RuleSet{})
	if err == nil {
		t.Errorf("Expected error but not occurred")
	}
	msg := "fint: cannot open " + filename
	if err.Error() != msg {
		t.Errorf("Expected error message [%s] but was [%s]", msg, err.Error())
	}
}

func TestSetbufsize(t *testing.T) {
	var (
		filename = "testdata/objc/FintExample/FintExample/FEAppDelegate.m"
		msg      string
	)
	fint.Setbufsize(0)
	_, err := fint.CheckSourceFile(filename, fint.RuleSet{})
	if err != nil {
		t.Errorf("Unexpected error occurred: %v", err)
	}

	fint.Setbufsize(1)
	_, err = fint.CheckSourceFile(filename, fint.RuleSet{})
	if err == nil {
		t.Errorf("Expected error but not occurred")
	}
	msg = "fint: too long line: " + filename
	if err.Error() != msg {
		t.Errorf("Expected error message [%s] but was [%s]", msg, err.Error())
	}
}

func testExecuteNormal(t *testing.T, opt *fint.Opt, expectedViolations int) {
	v, _ := fint.Execute(opt)
	if len(v) != expectedViolations {
		t.Errorf("Expected violations are [%d] but [%d] found", expectedViolations, len(v))
	}
}

func testExecuteError(t *testing.T, opt *fint.Opt, msg string) {
	_, err := fint.Execute(opt)
	if err == nil {
		t.Errorf("Expected error but not occurred")
		return
	}
	if err.Error() != msg {
		t.Errorf("Expected error message [%s] but was [%s]", msg, err.Error())
	}
}
<|endoftext|>


<commit_before>package amv

import (
	"strings"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
)

func getOpenGraph(ctx *aero.Context, amv *arn.AMV) *arn.OpenGraph {
	openGraph := &arn.OpenGraph{
		Tags: map[string]string{
			"og:title":     amv.Title.ByUser(nil) + " (AMV)",
			"og:url":       "https://" + ctx.App.Config.Domain + amv.Link(),
			"og:site_name": ctx.App.Config.Domain,
			"og:type":      "video.other",
		},
		// Meta: map[string]string{},
	}

	openGraph.Tags["og:description"] = strings.Join(amv.Tags, ", ")

	if amv.File != "" {
		openGraph.Tags["og:video"] = "https://" + ctx.App.Config.Domain + "/videos/amvs/" + amv.File
		openGraph.Tags["og:video:type"] = "video/webm"
		openGraph.Tags["og:video:width"] = "640"
		openGraph.Tags["og:video:height"] = "360"

		// openGraph.Meta["twitter:player"] = openGraph.Tags["og:video"]
		// openGraph.Meta["twitter:player:width"] = openGraph.Tags["og:video:width"]
		// openGraph.Meta["twitter:player:height"] = openGraph.Tags["og:video:height"]
		// openGraph.Meta["twitter:player:stream"] = openGraph.Tags["og:video"]
		// openGraph.Meta["twitter:player:stream:content_type"] = openGraph.Tags["og:video:type"]
	}

	return openGraph
}
<commit_msg>Added back Twitter player meta tags<commit_after>package amv

import (
	"strings"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
)

func getOpenGraph(ctx *aero.Context, amv *arn.AMV) *arn.OpenGraph {
	openGraph := &arn.OpenGraph{
		Tags: map[string]string{
			"og:title":     amv.Title.ByUser(nil) + " (AMV)",
			"og:url":       "https://" + ctx.App.Config.Domain + amv.Link(),
			"og:site_name": ctx.App.Config.Domain,
			"og:type":      "video.other",
		},
		Meta: map[string]string{},
	}

	openGraph.Tags["og:description"] = strings.Join(amv.Tags, ", ")

	if amv.File != "" {
		openGraph.Tags["og:video"] = "https://" + ctx.App.Config.Domain + "/videos/amvs/" + amv.File
		openGraph.Tags["og:video:type"] = "video/webm"
		openGraph.Tags["og:video:width"] = "640"
		openGraph.Tags["og:video:height"] = "360"

		openGraph.Meta["twitter:player"] = openGraph.Tags["og:video"]
		openGraph.Meta["twitter:player:width"] = openGraph.Tags["og:video:width"]
		openGraph.Meta["twitter:player:height"] = openGraph.Tags["og:video:height"]
		openGraph.Meta["twitter:player:stream"] = openGraph.Tags["og:video"]
		openGraph.Meta["twitter:player:stream:content_type"] = openGraph.Tags["og:video:type"]
	}

	return openGraph
}
<|endoftext|>


<commit_before>package httpretry

import (
	"bytes"
	"io"
	"net/http"
	"net/http/httptest"
	"strconv"
	"testing"
)

func TestRetry(t *testing.T) {
	requests := []func(w http.ResponseWriter, r *http.Request){
		func(w http.ResponseWriter, r *http.Request) {
			head := w.Header()
			head.Set("Accept-Ranges", "bytes")
			head.Set("Content-Type", "text/plain")
			head.Set("Content-Length", "5")
			w.WriteHeader(200)
			w.Write([]byte("ab"))
		},
		func(w http.ResponseWriter, r *http.Request) {
			head := w.Header()
			head.Set("Content-Range", "bytes 2-4/4")
			head.Set("Accept-Ranges", "bytes")
			head.Set("Content-Type", "text/plain")
			head.Set("Content-Length", "3")
			w.WriteHeader(206)
			w.Write([]byte("cd"))
		},
		func(w http.ResponseWriter, r *http.Request) {
			head := w.Header()
			head.Set("Content-Type", "text/plain")
			head.Set("Content-Length", "4")
			w.WriteHeader(500)
			w.Write([]byte("boom"))
		},
		func(w http.ResponseWriter, r *http.Request) {
			head := w.Header()
			head.Set("Content-Range", "bytes 4-4/4")
			head.Set("Accept-Ranges", "bytes")
			head.Set("Content-Type", "text/plain")
			head.Set("Content-Length", "1")
			w.WriteHeader(206)
			w.Write([]byte("e"))
		},
	}
	i := 0

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if i < len(requests) {
			requests[i](w, r)
			i += 1
		} else {
			head := w.Header()
			head.Set("Content-Type", "text/plain")
			head.Set("Content-Length", "7")
			w.WriteHeader(404)
			w.Write([]byte("missing"))
		}
	}))
	defer ts.Close()

	req, err := http.NewRequest("GET", ts.URL, nil)
	if err != nil {
		t.Fatal(err)
	}

	code, head, reader := Getter(req, nil)

	if code != 200 {
		t.Errorf("Unexpected status %d", code)
	}

	if ctype := head.Get("Content-Type"); ctype != "text/plain" {
		t.Errorf("Unexpected Content Type: %s", ctype)
	}

	buf := &bytes.Buffer{}
	written, err := io.Copy(buf, reader)
	if err != nil {
		t.Errorf("Copy error: %s", err)
	}

	if written != 5 {
		t.Errorf("Wrote %d", written)
	}

	if b := buf.String(); b != "abcde" {
		t.Errorf("Got %s", b)
	}

	reader.Close()
}

func TestSingleSuccess(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writeTestData(w, 200, "ok")
	}))
	defer ts.Close()

	req, err := http.NewRequest("GET", ts.URL, nil)
	if err != nil {
		t.Fatal(err)
	}

	code, head, reader := Getter(req, nil)

	if code != 200 {
		t.Errorf("Unexpected status %d", code)
	}

	if ctype := head.Get("Content-Type"); ctype != "text/plain" {
		t.Errorf("Unexpected Content Type: %s", ctype)
	}

	buf := &bytes.Buffer{}
	written, err := io.Copy(buf, reader)
	if err != nil {
		t.Errorf("Copy error: %s", err)
	}

	if written != 2 {
		t.Errorf("Wrote %d", written)
	}

	if b := buf.String(); b != "ok" {
		t.Errorf("Got %s", b)
	}

	reader.Close()
}

func TestSkipRetryWithoutAcceptRange(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		head := w.Header()
		head.Set("Content-Type", "text/plain")
		head.Set("Content-Length", "2")
		w.WriteHeader(200)
		w.Write([]byte("o"))
	}))
	defer ts.Close()

	req, err := http.NewRequest("GET", ts.URL, nil)
	if err != nil {
		t.Fatal(err)
	}

	code, head, reader := Getter(req, nil)

	if code != 200 {
		t.Errorf("Unexpected status %d", code)
	}

	if ctype := head.Get("Content-Type"); ctype != "text/plain" {
		t.Errorf("Unexpected Content Type: %s", ctype)
	}

	buf := &bytes.Buffer{}
	written, err := io.Copy(buf, reader)
	if err != nil {
		t.Errorf("Copy error: %s", err)
	}

	if written != 1 {
		t.Errorf("Wrote %d", written)
	}

	if b := buf.String(); b != "o" {
		t.Errorf("Got %s", b)
	}

	reader.Close()
}

func TestRetryWith400(t *testing.T) {
	status := 200
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writeTestData(w, status, "client error")
	}))
	defer ts.Close()

	req, err := http.NewRequest("GET", ts.URL, nil)
	if err != nil {
		t.Fatal(err)
	}

	for status = 400; status < 500; status++ {
		code, head, reader := Getter(req, nil)
		reader.Close()
		if code != status {
			t.Errorf("Expected status %d, got %d", status, code)
		}

		if ctype := head.Get("Content-Type"); ctype != "text/plain" {
			t.Fatalf("Unexpected Content Type: %s", ctype)
		}
	}
}

func writeTestData(w http.ResponseWriter, status int, body string) {
	by := []byte(body)
	head := w.Header()
	head.Set("Accept-Ranges", "bytes")
	head.Set("Content-Type", "text/plain")
	head.Set("Content-Length", strconv.Itoa(len(by)))
	w.WriteHeader(status)
	w.Write(by)
}
<commit_msg>test retries with timeouts<commit_after>package httpretry

import (
	"bytes"
	"io"
	"net/http"
	"net/http/httptest"
	"strconv"
	"testing"
	"time"
)

func TestRetry(t *testing.T) {
	t.Parallel()
	requests := []func(w http.ResponseWriter, r *http.Request){
		func(w http.ResponseWriter, r *http.Request) {
			time.Sleep(time.Second)
			writeTestData(w, 404, "never reached")
		},
		func(w http.ResponseWriter, r *http.Request) {
			head := w.Header()
			head.Set("Accept-Ranges", "bytes")
			head.Set("Content-Type", "text/plain")
			head.Set("Content-Length", "5")
			w.WriteHeader(200)
			w.Write([]byte("ab"))
		},
		func(w http.ResponseWriter, r *http.Request) {
			head := w.Header()
			head.Set("Content-Range", "bytes 2-4/4")
			head.Set("Accept-Ranges", "bytes")
			head.Set("Content-Type", "text/plain")
			head.Set("Content-Length", "3")
			w.WriteHeader(206)
			w.Write([]byte("cd"))
		},
		func(w http.ResponseWriter, r *http.Request) {
			time.Sleep(time.Second)
			writeTestData(w, 404, "never reached")
		},
		func(w http.ResponseWriter, r *http.Request) {
			head := w.Header()
			head.Set("Content-Type", "text/plain")
			head.Set("Content-Length", "4")
			w.WriteHeader(500)
			w.Write([]byte("boom"))
		},
		func(w http.ResponseWriter, r *http.Request) {
			head := w.Header()
			head.Set("Content-Range", "bytes 4-4/4")
			head.Set("Accept-Ranges", "bytes")
			head.Set("Content-Type", "text/plain")
			head.Set("Content-Length", "1")
			w.WriteHeader(206)
			w.Write([]byte("e"))
		},
	}
	i := 0

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if i < len(requests) {
			requests[i](w, r)
			i += 1
		} else {
			head := w.Header()
			head.Set("Content-Type", "text/plain")
			head.Set("Content-Length", "7")
			w.WriteHeader(404)
			w.Write([]byte("missing"))
		}
	}))
	defer ts.Close()

	req, err := http.NewRequest("GET", ts.URL, nil)
	if err != nil {
		t.Fatal(err)
	}

	code, head, reader := Getter(req, nil)

	if code != 200 {
		t.Errorf("Unexpected status %d", code)
	}

	if ctype := head.Get("Content-Type"); ctype != "text/plain" {
		t.Errorf("Unexpected Content Type: %s", ctype)
	}

	buf := &bytes.Buffer{}
	written, err := io.Copy(buf, reader)
	if err != nil {
		t.Errorf("Copy error: %s", err)
	}

	if written != 5 {
		t.Errorf("Wrote %d", written)
	}

	if b := buf.String(); b != "abcde" {
		t.Errorf("Got %s", b)
	}

	reader.Close()
}

func TestSingleSuccess(t *testing.T) {
	t.Parallel()
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writeTestData(w, 200, "ok")
	}))
	defer ts.Close()

	req, err := http.NewRequest("GET", ts.URL, nil)
	if err != nil {
		t.Fatal(err)
	}

	code, head, reader := Getter(req, nil)

	if code != 200 {
		t.Errorf("Unexpected status %d", code)
	}

	if ctype := head.Get("Content-Type"); ctype != "text/plain" {
		t.Errorf("Unexpected Content Type: %s", ctype)
	}

	buf := &bytes.Buffer{}
	written, err := io.Copy(buf, reader)
	if err != nil {
		t.Errorf("Copy error: %s", err)
	}

	if written != 2 {
		t.Errorf("Wrote %d", written)
	}

	if b := buf.String(); b != "ok" {
		t.Errorf("Got %s", b)
	}

	reader.Close()
}

func TestSkipRetryWithoutAcceptRange(t *testing.T) {
	t.Parallel()
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		head := w.Header()
		head.Set("Content-Type", "text/plain")
		head.Set("Content-Length", "2")
		w.WriteHeader(200)
		w.Write([]byte("o"))
	}))
	defer ts.Close()

	req, err := http.NewRequest("GET", ts.URL, nil)
	if err != nil {
		t.Fatal(err)
	}

	code, head, reader := Getter(req, nil)

	if code != 200 {
		t.Errorf("Unexpected status %d", code)
	}

	if ctype := head.Get("Content-Type"); ctype != "text/plain" {
		t.Errorf("Unexpected Content Type: %s", ctype)
	}

	buf := &bytes.Buffer{}
	written, err := io.Copy(buf, reader)
	if err != nil {
		t.Errorf("Copy error: %s", err)
	}

	if written != 1 {
		t.Errorf("Wrote %d", written)
	}

	if b := buf.String(); b != "o" {
		t.Errorf("Got %s", b)
	}

	reader.Close()
}

func TestRetryWith400(t *testing.T) {
	t.Parallel()
	status := 200
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writeTestData(w, status, "client error")
	}))
	defer ts.Close()

	req, err := http.NewRequest("GET", ts.URL, nil)
	if err != nil {
		t.Fatal(err)
	}

	for status = 400; status < 500; status++ {
		code, head, reader := Getter(req, nil)
		reader.Close()
		if code != status {
			t.Errorf("Expected status %d, got %d", status, code)
		}

		if ctype := head.Get("Content-Type"); ctype != "text/plain" {
			t.Fatalf("Unexpected Content Type: %s", ctype)
		}
	}
}

func writeTestData(w http.ResponseWriter, status int, body string) {
	by := []byte(body)
	head := w.Header()
	head.Set("Accept-Ranges", "bytes")
	head.Set("Content-Type", "text/plain")
	head.Set("Content-Length", strconv.Itoa(len(by)))
	w.WriteHeader(status)
	w.Write(by)
}

func init() {
	tport := http.DefaultTransport.(*http.Transport)
	tport.ResponseHeaderTimeout = 500 * time.Millisecond
}
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"flag"
	"fmt"
	"strconv"
)

// boolValue needs to implement the boolFlag internal interface in flag
// to be able to capture bool fields and values
// type boolFlag interface {
//	  Value
//	  IsBoolFlag() bool
// }
type boolValue struct {
	destination *bool
	count       *int
}

func newBoolValue(val bool, p *bool, count *int) *boolValue {
	*p = val
	return &boolValue{
		destination: p,
		count:       count,
	}
}

func (b *boolValue) Set(s string) error {
	v, err := strconv.ParseBool(s)
	if err != nil {
		err = errors.New("parse error")
		return err
	}
	*b.destination = v
	if b.count != nil {
		*b.count = *b.count + 1
	}
	return err
}

func (b *boolValue) Get() interface{} { return *b.destination }

func (b *boolValue) String() string {
	if b.destination != nil {
		return strconv.FormatBool(*b.destination)
	}
	return strconv.FormatBool(false)
}

func (b *boolValue) IsBoolFlag() bool { return true }

// TakesValue returns true of the flag takes a value, otherwise false
func (f *BoolFlag) TakesValue() bool {
	return false
}

// GetUsage returns the usage string for the flag
func (f *BoolFlag) GetUsage() string {
	return f.Usage
}

// GetCategory returns the category for the flag
func (f *BoolFlag) GetCategory() string {
	return f.Category
}

// GetValue returns the flags value as string representation and an empty
// string if the flag takes no value at all.
func (f *BoolFlag) GetValue() string {
	return ""
}

// GetDefaultText returns the default text for this flag
func (f *BoolFlag) GetDefaultText() string {
	if f.DefaultText != "" {
		return f.DefaultText
	}
	return fmt.Sprintf("%v", f.Value)
}

// GetEnvVars returns the env vars for this flag
func (f *BoolFlag) GetEnvVars() []string {
	return f.EnvVars
}

// Apply populates the flag given the flag set and environment
func (f *BoolFlag) Apply(set *flag.FlagSet) error {
	if val, source, found := flagFromEnvOrFile(f.EnvVars, f.FilePath); found {
		if val != "" {
			valBool, err := strconv.ParseBool(val)

			if err != nil {
				return fmt.Errorf("could not parse %q as bool value from %s for flag %s: %s", val, source, f.Name, err)
			}

			f.Value = valBool
		} else {
			// empty value implies that the env is defined but set to empty string, we have to assume that this is
			// what the user wants. If user doesnt want this then the env needs to be deleted or the flag removed from
			// file
			f.Value = false
		}
		f.HasBeenSet = true
	}

	for _, name := range f.Names() {
		var value flag.Value
		if f.Destination != nil {
			value = newBoolValue(f.Value, f.Destination, f.Count)
		} else {
			t := new(bool)
			value = newBoolValue(f.Value, t, f.Count)
		}
		set.Var(value, name, f.Usage)
	}

	return nil
}

// Get returns the flag’s value in the given Context.
func (f *BoolFlag) Get(ctx *Context) bool {
	return ctx.Bool(f.Name)
}

// Bool looks up the value of a local BoolFlag, returns
// false if not found
func (cCtx *Context) Bool(name string) bool {
	if fs := cCtx.lookupFlagSet(name); fs != nil {
		return lookupBool(name, fs)
	}
	return false
}

func lookupBool(name string, set *flag.FlagSet) bool {
	f := set.Lookup(name)
	if f != nil {
		parsed, err := strconv.ParseBool(f.Value.String())
		if err != nil {
			return false
		}
		return parsed
	}
	return false
}
<commit_msg>Run gofmt<commit_after>package cli

import (
	"errors"
	"flag"
	"fmt"
	"strconv"
)

// boolValue needs to implement the boolFlag internal interface in flag
// to be able to capture bool fields and values
//
//	type boolFlag interface {
//		  Value
//		  IsBoolFlag() bool
//	}
type boolValue struct {
	destination *bool
	count       *int
}

func newBoolValue(val bool, p *bool, count *int) *boolValue {
	*p = val
	return &boolValue{
		destination: p,
		count:       count,
	}
}

func (b *boolValue) Set(s string) error {
	v, err := strconv.ParseBool(s)
	if err != nil {
		err = errors.New("parse error")
		return err
	}
	*b.destination = v
	if b.count != nil {
		*b.count = *b.count + 1
	}
	return err
}

func (b *boolValue) Get() interface{} { return *b.destination }

func (b *boolValue) String() string {
	if b.destination != nil {
		return strconv.FormatBool(*b.destination)
	}
	return strconv.FormatBool(false)
}

func (b *boolValue) IsBoolFlag() bool { return true }

// TakesValue returns true of the flag takes a value, otherwise false
func (f *BoolFlag) TakesValue() bool {
	return false
}

// GetUsage returns the usage string for the flag
func (f *BoolFlag) GetUsage() string {
	return f.Usage
}

// GetCategory returns the category for the flag
func (f *BoolFlag) GetCategory() string {
	return f.Category
}

// GetValue returns the flags value as string representation and an empty
// string if the flag takes no value at all.
func (f *BoolFlag) GetValue() string {
	return ""
}

// GetDefaultText returns the default text for this flag
func (f *BoolFlag) GetDefaultText() string {
	if f.DefaultText != "" {
		return f.DefaultText
	}
	return fmt.Sprintf("%v", f.Value)
}

// GetEnvVars returns the env vars for this flag
func (f *BoolFlag) GetEnvVars() []string {
	return f.EnvVars
}

// Apply populates the flag given the flag set and environment
func (f *BoolFlag) Apply(set *flag.FlagSet) error {
	if val, source, found := flagFromEnvOrFile(f.EnvVars, f.FilePath); found {
		if val != "" {
			valBool, err := strconv.ParseBool(val)

			if err != nil {
				return fmt.Errorf("could not parse %q as bool value from %s for flag %s: %s", val, source, f.Name, err)
			}

			f.Value = valBool
		} else {
			// empty value implies that the env is defined but set to empty string, we have to assume that this is
			// what the user wants. If user doesnt want this then the env needs to be deleted or the flag removed from
			// file
			f.Value = false
		}
		f.HasBeenSet = true
	}

	for _, name := range f.Names() {
		var value flag.Value
		if f.Destination != nil {
			value = newBoolValue(f.Value, f.Destination, f.Count)
		} else {
			t := new(bool)
			value = newBoolValue(f.Value, t, f.Count)
		}
		set.Var(value, name, f.Usage)
	}

	return nil
}

// Get returns the flag’s value in the given Context.
func (f *BoolFlag) Get(ctx *Context) bool {
	return ctx.Bool(f.Name)
}

// Bool looks up the value of a local BoolFlag, returns
// false if not found
func (cCtx *Context) Bool(name string) bool {
	if fs := cCtx.lookupFlagSet(name); fs != nil {
		return lookupBool(name, fs)
	}
	return false
}

func lookupBool(name string, set *flag.FlagSet) bool {
	f := set.Lookup(name)
	if f != nil {
		parsed, err := strconv.ParseBool(f.Value.String())
		if err != nil {
			return false
		}
		return parsed
	}
	return false
}
<|endoftext|>


<commit_before>package filecache

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

const VERSION = "1.0.2"

// File size constants for use with FileCache.MaxSize.
// For example, cache.MaxSize = 64 * Megabyte
const (
	Kilobyte = 1024
	Megabyte = 1024 * 1024
	Gigabyte = 1024 * 1024 * 1024
)

var (
	DefaultExpireItem int   = 300 // 5 minutes
	DefaultMaxSize    int64 = 16 * Megabyte
	DefaultMaxItems   int   = 32
	DefaultEvery      int   = 60 // 1 minute
)

var (
	InvalidCacheItem = errors.New("invalid cache item")
	ItemIsDirectory  = errors.New("can't cache a directory")
	ItemNotInCache   = errors.New("item not in cache")
	ItemTooLarge     = errors.New("item too large for cache")
	WriteIncomplete  = errors.New("incomplete write of cache item")
)

var SquelchItemNotInCache = true

// Mumber of items to buffer adding to the file cache.
var NewCachePipeSize = 4

type cacheItem struct {
	content    []byte
	Size       int64
	Lastaccess time.Time
	Modified   time.Time
}

func (itm *cacheItem) GetReader() io.Reader {
	b := bytes.NewReader(itm.Access())
	return b
}

func (itm *cacheItem) Access() []byte {
	itm.Lastaccess = time.Now()
	return itm.content
}

func cacheFile(path string, maxSize int64) (itm *cacheItem, err error) {
	fi, err := os.Stat(path)
	if err != nil {
		return
	} else if fi.Mode().IsDir() {
		return nil, ItemIsDirectory
	} else if fi.Size() > maxSize {
		return nil, ItemTooLarge
	}

	content, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	itm = &cacheItem{
		content:    content,
		Size:       fi.Size(),
		Modified:   fi.ModTime(),
		Lastaccess: time.Now(),
	}
	return
}

// FileCache represents a cache in memory.
// An ExpireItem value of 0 means that items should not be expired based
// on time in memory.
type FileCache struct {
	dur        time.Duration
	items      map[string]*cacheItem
	in         chan string
	MaxItems   int   // Maximum number of files to cache
	MaxSize    int64 // Maximum file size to store
	ExpireItem int   // Seconds a file should be cached for
	Every      int   // Run an expiration check Every seconds
}

// NewDefaultCache returns a new FileCache with sane defaults.
func NewDefaultCache() *FileCache {
	return &FileCache{time.Since(time.Now()),
		nil, nil,
		DefaultMaxItems,
		DefaultMaxSize,
		DefaultExpireItem,
		DefaultEvery,
	}
}

// add_item is an internal function for adding an item to the cache.
func (cache *FileCache) add_item(name string) (err error) {
	if cache.items == nil {
		return
	}
	ok := cache.InCache(name)
	expired := cache.itemExpired(name)
	if ok && !expired {
		return nil
	} else if ok {
		delete(cache.items, name)
	}

	itm, err := cacheFile(name, cache.MaxSize)
	if cache.items != nil && itm != nil {
		cache.items[name] = itm
	} else {
		return
	}
	if !cache.InCache(name) {
		return ItemNotInCache
	}
	return nil
}

// item_listener is a goroutine that listens for incoming files and caches
// them.
func (cache *FileCache) item_listener() {
	for {
		name, ok := <-cache.in
		if !ok {
			return
		}
		cache.add_item(name)
	}
}

// expire_oldest is used to expire the oldest item in the cache.
// The force argument is used to indicate it should remove at least one
// entry; for example, if a large number of files are cached at once, none
// may appear older than another.
func (cache *FileCache) expire_oldest(force bool) {
	oldest := time.Now()
	oldest_name := ""

	for name, itm := range cache.items {
		if force && oldest_name == "" {
			oldest = itm.Lastaccess
			oldest_name = name
		} else if itm.Lastaccess.Before(oldest) {
			oldest = itm.Lastaccess
			oldest_name = name
		}
	}
	if oldest_name != "" {
		delete(cache.items, oldest_name)
	}
}

// vacuum is a background goroutine responsible for cleaning the cache.
// It runs periodically, every cache.Every seconds. If cache.Every is set
// to 0, it will not run.
func (cache *FileCache) vacuum() {
	if cache.Every < 1 {
		return
	}

	for _ = range time.Tick(cache.dur) {
		if cache.items == nil {
			return
		}
		for name, _ := range cache.items {
			if cache.itemExpired(name) {
				delete(cache.items, name)
			}
		}
		for size := cache.Size(); size > cache.MaxItems; size = cache.Size() {
			cache.expire_oldest(true)
		}
	}
}

// FileChanged returns true if file should be expired based on mtime.
// If the file has changed on disk or no longer exists, it should be
// expired.
func (cache *FileCache) changed(name string) bool {
	itm, ok := cache.items[name]
	if !ok || itm == nil {
		return true
	}
	fi, err := os.Stat(name)
	if err != nil {
		return true
	} else if !itm.Modified.Equal(fi.ModTime()) {
		return true
	}
	return false
}

// Expired returns true if the item has not been accessed recently.
func (cache *FileCache) expired(name string) bool {
	itm, ok := cache.items[name]
	if !ok {
		return true
	}
	dur := time.Now().Sub(itm.Lastaccess)
	sec, err := strconv.Atoi(fmt.Sprintf("%0.0f", dur.Seconds()))
	if err != nil {
		return true
	} else if sec >= cache.ExpireItem {
		return true
	}
	return false
}

// itemExpired returns true if an item is expired.
func (cache *FileCache) itemExpired(name string) bool {
	if cache.changed(name) {
		return true
	} else if cache.ExpireItem != 0 && cache.expired(name) {
		return true
	}
	return false
}

// Active returns true if the cache has been started, and false otherwise.
func (cache *FileCache) Active() bool {
	if cache.in == nil || cache.items == nil {
		return false
	}
	return true
}

// Size returns the number of entries in the cache.
func (cache *FileCache) Size() int {
	return len(cache.items)
}

// FileSize returns the sum of the file sizes stored in the cache
func (cache *FileCache) FileSize() (totalSize int64) {
	for _, itm := range cache.items {
		totalSize += itm.Size
	}
	return
}

// StoredFiles returns the list of files stored in the cache.
func (cache *FileCache) StoredFiles() (fileList []string) {
	fileList = make([]string, 0)
	for name, _ := range cache.items {
		fileList = append(fileList, name)
	}
	return
}

// InCache returns true if the item is in the cache.
func (cache *FileCache) InCache(name string) bool {
	if cache.changed(name) {
		delete(cache.items, name)
		return false
	}
	_, ok := cache.items[name]
	return ok
}

// WriteItem writes the cache item to the specified io.Writer.
func (cache *FileCache) WriteItem(w io.Writer, name string) (err error) {
	itm, ok := cache.items[name]
	if !ok {
		if !SquelchItemNotInCache {
			err = ItemNotInCache
		}
		return
	}
	r := itm.GetReader()
	itm.Lastaccess = time.Now()
	n, err := io.Copy(w, r)
	if err != nil {
		return
	} else if int64(n) != itm.Size {
		err = WriteIncomplete
		return
	}
	return
}

// GetItem returns the content of the item and a bool if name is present.
// GetItem should be used when you are certain an object is in the cache,
// or if you want to use the cache only.
func (cache *FileCache) GetItem(name string) (content []byte, ok bool) {
	itm, ok := cache.items[name]
	if !ok {
		return
	}
	content = itm.Access()
	return
}

// GetItemString is the same as GetItem, except returning a string.
func (cache *FileCache) GetItemString(name string) (content string, ok bool) {
	itm, ok := cache.items[name]
	if !ok {
		return
	}
	content = string(itm.Access())
	return
}

// ReadFile retrieves the file named by 'name'.
// If the file is not in the cache, load the file and cache the file in the
// background. If the file was not in the cache and the read was successful,
// the error ItemNotInCache is returned to indicate that the item was pulled
// from the filesystem and not the cache, unless the SquelchItemNotInCache
// global option is set; in that case, returns no error.
func (cache *FileCache) ReadFile(name string) (content []byte, err error) {
	if cache.InCache(name) {
		content, _ = cache.GetItem(name)
	} else {
		go cache.Cache(name)
		content, err = ioutil.ReadFile(name)
		if err == nil && !SquelchItemNotInCache {
			err = ItemNotInCache
		}
	}
	return
}

// ReadFileString is the same as ReadFile, except returning a string.
func (cache *FileCache) ReadFileString(name string) (content string, err error) {
	raw, err := cache.ReadFile(name)
	if err == nil {
		content = string(raw)
	}
	return
}

// WriteFile writes the file named by 'name' to the specified io.Writer.
// If the file is in the cache, it is loaded from the cache; otherwise,
// it is read from the filesystem and the file is cached in the background.
func (cache *FileCache) WriteFile(w io.Writer, name string) (err error) {
	if cache.InCache(name) {
		err = cache.WriteItem(w, name)
	} else {
		var fi os.FileInfo
		fi, err = os.Stat(name)
		if err != nil {
			return
		} else if fi.IsDir() {
			return ItemIsDirectory
		}
		go cache.Cache(name)
		var file *os.File
		file, err = os.Open(name)
		if err != nil {
			return
		}
		defer file.Close()
		_, err = io.Copy(w, file)

	}
	return
}

func (cache *FileCache) HttpWriteFile(w http.ResponseWriter, r *http.Request) {
	path, err := url.QueryUnescape(r.URL.String())
	if err != nil {
		http.ServeFile(w, r, r.URL.Path)
	} else if len(path) > 1 {
		path = path[1:len(path)]
	} else {
		http.ServeFile(w, r, ".")
		return
	}

	if cache.InCache(path) {
		itm := cache.items[path]
		ctype := http.DetectContentType(itm.Access())
		mtype := mime.TypeByExtension(filepath.Ext(path))
		if mtype != "" && mtype != ctype {
			ctype = mtype
		}
		w.Header().Set("content-length", fmt.Sprintf("%d", itm.Size))
		w.Header().Set("content-disposition",
			fmt.Sprintf("filename=%s", filepath.Base(path)))
		w.Header().Set("content-type", ctype)
		w.Write(itm.Access())
		return
	}
	go cache.Cache(path)
	http.ServeFile(w, r, path)
}

// HttpHandler returns a valid HTTP handler for the given cache.
func HttpHandler(cache *FileCache) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		cache.HttpWriteFile(w, r)
	}
}

// Cache will store the file named by 'name' to the cache.
// This function doesn't return anything as it passes the file onto the
// incoming pipe; the file will be cached asynchronously. Errors will
// not be returned.
func (cache *FileCache) Cache(name string) {
	if cache.Size() == cache.MaxItems {
		cache.expire_oldest(true)
	}
	cache.in <- name
}

// CacheNow immediately caches the file named by 'name'.
func (cache *FileCache) CacheNow(name string) (err error) {
	if cache.Size() == cache.MaxItems {
		cache.expire_oldest(true)
	}
	return cache.add_item(name)
}

// Start activates the file cache; it will start up the background caching
// and automatic cache expiration goroutines and initialise the internal
// data structures.
func (cache *FileCache) Start() error {
	if cache.in != nil {
		close(cache.in)
	}
	dur, err := time.ParseDuration(fmt.Sprintf("%ds", cache.Every))
	if err != nil {
		return err
	}
	cache.dur = dur
	cache.items = make(map[string]*cacheItem, 0)
	cache.in = make(chan string, NewCachePipeSize)
	go cache.item_listener()
	go cache.vacuum()
	return nil
}

// Stop turns off the file cache.
// This closes the concurrent caching mechanism, destroys the cache, and
// the background scanner that it should stop.
// If there are any items or cache operations ongoing while Stop() is called,
// it is undefined how they will behave.
func (cache *FileCache) Stop() {
	if cache.in != nil {
		close(cache.in)
	}
	if cache.items != nil {
		for name, _ := range cache.items {
			delete(cache.items, name)
		}
		cache.items = nil
	}
}

// RemoveItem immediately removes the item from the cache if it is present.
// It returns a boolean indicating whether anything was removed, and an error
// if an error has occurred.
func (cache *FileCache) Remove(name string) (ok bool, err error) {
	_, ok = cache.items[name]
	if !ok {
		return
	}
	delete(cache.items, name)
	_, valid := cache.items[name]
	if valid {
		ok = false
	}
	return
}
<commit_msg>add Mutexes to filecache<commit_after>package filecache

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"
)

const VERSION = "1.0.2"

// File size constants for use with FileCache.MaxSize.
// For example, cache.MaxSize = 64 * Megabyte
const (
	Kilobyte = 1024
	Megabyte = 1024 * 1024
	Gigabyte = 1024 * 1024 * 1024
)

var (
	DefaultExpireItem int   = 300 // 5 minutes
	DefaultMaxSize    int64 = 16 * Megabyte
	DefaultMaxItems   int   = 32
	DefaultEvery      int   = 60 // 1 minute
)

var (
	InvalidCacheItem = errors.New("invalid cache item")
	ItemIsDirectory  = errors.New("can't cache a directory")
	ItemNotInCache   = errors.New("item not in cache")
	ItemTooLarge     = errors.New("item too large for cache")
	WriteIncomplete  = errors.New("incomplete write of cache item")
)

var SquelchItemNotInCache = true

// Mumber of items to buffer adding to the file cache.
var NewCachePipeSize = 4

type cacheItem struct {
	content    []byte
	lock       *sync.Mutex
	Size       int64
	Lastaccess time.Time
	Modified   time.Time
}

func (itm *cacheItem) Lock() {
        itm.lock.Lock()
}

func (itm *cacheItem) Unlock() {
        itm.lock.Unlock()
}

func (itm *cacheItem) GetReader() io.Reader {
	b := bytes.NewReader(itm.Access())
	return b
}

func (itm *cacheItem) Access() []byte {
	itm.Lastaccess = time.Now()
	return itm.content
}

func cacheFile(path string, maxSize int64) (itm *cacheItem, err error) {
	fi, err := os.Stat(path)
	if err != nil {
		return
	} else if fi.Mode().IsDir() {
		return nil, ItemIsDirectory
	} else if fi.Size() > maxSize {
		return nil, ItemTooLarge
	}

	content, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	itm = &cacheItem{
		content:    content,
		lock:       new(sync.Mutex),
		Size:       fi.Size(),
		Modified:   fi.ModTime(),
		Lastaccess: time.Now(),
	}
	return
}

// FileCache represents a cache in memory.
// An ExpireItem value of 0 means that items should not be expired based
// on time in memory.
type FileCache struct {
	dur        time.Duration
	items      map[string]*cacheItem
	in         chan string
	mutex      *sync.Mutex
	MaxItems   int   // Maximum number of files to cache
	MaxSize    int64 // Maximum file size to store
	ExpireItem int   // Seconds a file should be cached for
	Every      int   // Run an expiration check Every seconds
}

// NewDefaultCache returns a new FileCache with sane defaults.
func NewDefaultCache() *FileCache {
	return &FileCache{
		dur:        time.Since(time.Now()),
		items:      nil,
		in:         nil,
		mutex:      new(sync.Mutex),
		MaxItems:   DefaultMaxItems,
		MaxSize:    DefaultMaxSize,
		ExpireItem: DefaultExpireItem,
		Every:      DefaultEvery,
	}
}

func (cache *FileCache) lock() {
        cache.mutex.lock()
}

func (cache *FileCache) unlock() {
        cache.mutex.unlock()
}

// add_item is an internal function for adding an item to the cache.
func (cache *FileCache) add_item(name string) (err error) {
	if cache.items == nil {
		return
	}
	ok := cache.InCache(name)
	expired := cache.itemExpired(name)
	if ok && !expired {
		return nil
	} else if ok {
		delete(cache.items, name)
	}

	itm, err := cacheFile(name, cache.MaxSize)
	if cache.items != nil && itm != nil {
		cache.items[name] = itm
	} else {
		return
	}
	if !cache.InCache(name) {
		return ItemNotInCache
	}
	return nil
}

// item_listener is a goroutine that listens for incoming files and caches
// them.
func (cache *FileCache) item_listener() {
	for {
		name, ok := <-cache.in
		if !ok {
			return
		}
		cache.add_item(name)
	}
}

// expire_oldest is used to expire the oldest item in the cache.
// The force argument is used to indicate it should remove at least one
// entry; for example, if a large number of files are cached at once, none
// may appear older than another.
func (cache *FileCache) expire_oldest(force bool) {
	oldest := time.Now()
	oldest_name := ""

	for name, itm := range cache.items {
		if force && oldest_name == "" {
			oldest = itm.Lastaccess
			oldest_name = name
		} else if itm.Lastaccess.Before(oldest) {
			oldest = itm.Lastaccess
			oldest_name = name
		}
	}
	if oldest_name != "" {
		delete(cache.items, oldest_name)
	}
}

// vacuum is a background goroutine responsible for cleaning the cache.
// It runs periodically, every cache.Every seconds. If cache.Every is set
// to 0, it will not run.
func (cache *FileCache) vacuum() {
	if cache.Every < 1 {
		return
	}

	for _ = range time.Tick(cache.dur) {
		if cache.items == nil {
			return
		}
		for name, _ := range cache.items {
			if cache.itemExpired(name) {
				delete(cache.items, name)
			}
		}
		for size := cache.Size(); size > cache.MaxItems; size = cache.Size() {
			cache.expire_oldest(true)
		}
	}
}

// FileChanged returns true if file should be expired based on mtime.
// If the file has changed on disk or no longer exists, it should be
// expired.
func (cache *FileCache) changed(name string) bool {
	itm, ok := cache.items[name]
	if !ok || itm == nil {
		return true
	}
	fi, err := os.Stat(name)
	if err != nil {
		return true
	} else if !itm.Modified.Equal(fi.ModTime()) {
		return true
	}
	return false
}

// Expired returns true if the item has not been accessed recently.
func (cache *FileCache) expired(name string) bool {
	itm, ok := cache.items[name]
	if !ok {
		return true
	}
	dur := time.Now().Sub(itm.Lastaccess)
	sec, err := strconv.Atoi(fmt.Sprintf("%0.0f", dur.Seconds()))
	if err != nil {
		return true
	} else if sec >= cache.ExpireItem {
		return true
	}
	return false
}

// itemExpired returns true if an item is expired.
func (cache *FileCache) itemExpired(name string) bool {
	if cache.changed(name) {
		return true
	} else if cache.ExpireItem != 0 && cache.expired(name) {
		return true
	}
	return false
}

// Active returns true if the cache has been started, and false otherwise.
func (cache *FileCache) Active() bool {
	if cache.in == nil || cache.items == nil {
		return false
	}
	return true
}

// Size returns the number of entries in the cache.
func (cache *FileCache) Size() int {
	return len(cache.items)
}

// FileSize returns the sum of the file sizes stored in the cache
func (cache *FileCache) FileSize() (totalSize int64) {
	for _, itm := range cache.items {
		totalSize += itm.Size
	}
	return
}

// StoredFiles returns the list of files stored in the cache.
func (cache *FileCache) StoredFiles() (fileList []string) {
	fileList = make([]string, 0)
	for name, _ := range cache.items {
		fileList = append(fileList, name)
	}
	return
}

// InCache returns true if the item is in the cache.
func (cache *FileCache) InCache(name string) bool {
	if cache.changed(name) {
		delete(cache.items, name)
		return false
	}
	_, ok := cache.items[name]
	return ok
}

// WriteItem writes the cache item to the specified io.Writer.
func (cache *FileCache) WriteItem(w io.Writer, name string) (err error) {
	itm, ok := cache.items[name]
	if !ok {
		if !SquelchItemNotInCache {
			err = ItemNotInCache
		}
		return
	}
	r := itm.GetReader()
	itm.Lastaccess = time.Now()
	n, err := io.Copy(w, r)
	if err != nil {
		return
	} else if int64(n) != itm.Size {
		err = WriteIncomplete
		return
	}
	return
}

// GetItem returns the content of the item and a bool if name is present.
// GetItem should be used when you are certain an object is in the cache,
// or if you want to use the cache only.
func (cache *FileCache) GetItem(name string) (content []byte, ok bool) {
	itm, ok := cache.items[name]
	if !ok {
		return
	}
	content = itm.Access()
	return
}

// GetItemString is the same as GetItem, except returning a string.
func (cache *FileCache) GetItemString(name string) (content string, ok bool) {
	itm, ok := cache.items[name]
	if !ok {
		return
	}
	content = string(itm.Access())
	return
}

// ReadFile retrieves the file named by 'name'.
// If the file is not in the cache, load the file and cache the file in the
// background. If the file was not in the cache and the read was successful,
// the error ItemNotInCache is returned to indicate that the item was pulled
// from the filesystem and not the cache, unless the SquelchItemNotInCache
// global option is set; in that case, returns no error.
func (cache *FileCache) ReadFile(name string) (content []byte, err error) {
	if cache.InCache(name) {
		content, _ = cache.GetItem(name)
	} else {
		go cache.Cache(name)
		content, err = ioutil.ReadFile(name)
		if err == nil && !SquelchItemNotInCache {
			err = ItemNotInCache
		}
	}
	return
}

// ReadFileString is the same as ReadFile, except returning a string.
func (cache *FileCache) ReadFileString(name string) (content string, err error) {
	raw, err := cache.ReadFile(name)
	if err == nil {
		content = string(raw)
	}
	return
}

// WriteFile writes the file named by 'name' to the specified io.Writer.
// If the file is in the cache, it is loaded from the cache; otherwise,
// it is read from the filesystem and the file is cached in the background.
func (cache *FileCache) WriteFile(w io.Writer, name string) (err error) {
	if cache.InCache(name) {
		err = cache.WriteItem(w, name)
	} else {
		var fi os.FileInfo
		fi, err = os.Stat(name)
		if err != nil {
			return
		} else if fi.IsDir() {
			return ItemIsDirectory
		}
		go cache.Cache(name)
		var file *os.File
		file, err = os.Open(name)
		if err != nil {
			return
		}
		defer file.Close()
		_, err = io.Copy(w, file)

	}
	return
}

func (cache *FileCache) HttpWriteFile(w http.ResponseWriter, r *http.Request) {
	path, err := url.QueryUnescape(r.URL.String())
	if err != nil {
		http.ServeFile(w, r, r.URL.Path)
	} else if len(path) > 1 {
		path = path[1:len(path)]
	} else {
		http.ServeFile(w, r, ".")
		return
	}

	if cache.InCache(path) {
		itm := cache.items[path]
		ctype := http.DetectContentType(itm.Access())
		mtype := mime.TypeByExtension(filepath.Ext(path))
		if mtype != "" && mtype != ctype {
			ctype = mtype
		}
		w.Header().Set("content-length", fmt.Sprintf("%d", itm.Size))
		w.Header().Set("content-disposition",
			fmt.Sprintf("filename=%s", filepath.Base(path)))
		w.Header().Set("content-type", ctype)
		w.Write(itm.Access())
		return
	}
	go cache.Cache(path)
	http.ServeFile(w, r, path)
}

// HttpHandler returns a valid HTTP handler for the given cache.
func HttpHandler(cache *FileCache) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		cache.HttpWriteFile(w, r)
	}
}

// Cache will store the file named by 'name' to the cache.
// This function doesn't return anything as it passes the file onto the
// incoming pipe; the file will be cached asynchronously. Errors will
// not be returned.
func (cache *FileCache) Cache(name string) {
	if cache.Size() == cache.MaxItems {
		cache.expire_oldest(true)
	}
	cache.in <- name
}

// CacheNow immediately caches the file named by 'name'.
func (cache *FileCache) CacheNow(name string) (err error) {
	if cache.Size() == cache.MaxItems {
		cache.expire_oldest(true)
	}
	return cache.add_item(name)
}

// Start activates the file cache; it will start up the background caching
// and automatic cache expiration goroutines and initialise the internal
// data structures.
func (cache *FileCache) Start() error {
	if cache.in != nil {
		close(cache.in)
	}
	dur, err := time.ParseDuration(fmt.Sprintf("%ds", cache.Every))
	if err != nil {
		return err
	}
	cache.dur = dur
	cache.items = make(map[string]*cacheItem, 0)
	cache.in = make(chan string, NewCachePipeSize)
	go cache.item_listener()
	go cache.vacuum()
	return nil
}

// Stop turns off the file cache.
// This closes the concurrent caching mechanism, destroys the cache, and
// the background scanner that it should stop.
// If there are any items or cache operations ongoing while Stop() is called,
// it is undefined how they will behave.
func (cache *FileCache) Stop() {
	if cache.in != nil {
		close(cache.in)
	}
	if cache.items != nil {
		for name, _ := range cache.items {
			delete(cache.items, name)
		}
		cache.items = nil
	}
}

// RemoveItem immediately removes the item from the cache if it is present.
// It returns a boolean indicating whether anything was removed, and an error
// if an error has occurred.
func (cache *FileCache) Remove(name string) (ok bool, err error) {
	_, ok = cache.items[name]
	if !ok {
		return
	}
	delete(cache.items, name)
	_, valid := cache.items[name]
	if valid {
		ok = false
	}
	return
}
<|endoftext|>