<commit_before>package sconf

import (
	"bytes"
	"fmt"
	"strconv"
	"regexp"
	"strings"
	"io/ioutil"
	"sort"
	"github.com/vaughan0/go-ini"
)

type TierConf struct {
	reg *regexp.Regexp
	conf map[string]map[string]string

}

func NewTierConf() *TierConf {

	return &TierConf{
		conf: make(map[string]map[string]string),
		reg: regexp.MustCompile("\\$\\{.*?\\}"),

	}
}

func (m *TierConf) StringCheck() (string, error) {
	keys := make([]string, 0)
	for s, _ := range m.conf {
		keys = append(keys, s)
	}
	sort.Strings(keys)
	rv := ""
	for _, s := range keys {
		rv += fmt.Sprintf("[%s]\n", s)

		ps := make([]string, 0)
		for s, _ := range m.conf[s] {
			ps = append(ps, s)
		}
		sort.Strings(ps)

		for _, p := range ps {
			v, err := m.ToString(s, p)
			if err != nil {
				return "", err
			}
			rv += fmt.Sprintf("%s=%s\n", p, v)
		}
		rv += fmt.Sprintf("\n")
	}

	return rv, nil
}

func (m *TierConf) GetConf() map[string]map[string]string {
	return m.conf
}

func (m *TierConf) LoadFromConf(cfg map[string]map[string]string) {
	for name, section := range cfg {
		if _, ok := m.conf[name]; !ok {
			m.conf[name] = make(map[string]string)
		}

		for k, v := range section {
			m.conf[name][k] = v
		}
	}

}

func (m *TierConf) LoadFromFile(conf string) error {
	configs := strings.Split(conf, ",")

	for _, c := range configs {
		if err := m.LoadFromOneFile(c); err != nil {
			return err
		}
	}

	return nil

}

func (m *TierConf) LoadFromOneFile(conf string) error {
	data, err := ioutil.ReadFile(conf)
	if err != nil {
		return err
	} else {
		return m.Load(data)
	}

}

func (m *TierConf) Load(cfg []byte) error {
	file, err := ini.Load(bytes.NewReader(cfg))

	if err != nil {
		return err
	}

	for name, section := range file {
		if _, ok := m.conf[name]; !ok {
			m.conf[name] = make(map[string]string)
		}

		for k, v := range section {
			m.conf[name][k] = v
		}
	}

	return nil

}

func (m *TierConf) toString(history []string, section string, property string) (string, error) {
	s, err := m.ToSection(section)

	if err != nil {
		return "", err

	} else {
		if p, ok := s[property]; ok {
			v, perr := m.parseVar(history, p)
			if perr != nil {
				return "", perr
			} else {
				return v, nil
			}
		} else {
			return "", fmt.Errorf("property empty:%s.%s", section, property)
		}

	}

}

func (m *TierConf) parseVar(history []string, value string) (string, error) {

	ids := m.reg.FindAllStringIndex(value, -1)

	var rv string = ""

	lastpos := 0
	for _, index := range(ids) {
		rv += value[lastpos:index[0]]
		pv := value[index[0]:index[1]]
		v := strings.Trim(pv, " \t${}")

		tmp := strings.Index(v, ".")

		if tmp == -1 {
			rv += pv
		} else {
			trims := strings.Trim(v[:tmp], " \t")
			trimp := strings.Trim(v[tmp+1:], " \t")
			// 检查循环引用
			newhis := fmt.Sprintf("%s.%s", trims, trimp)
			for _, his := range history {
				if newhis == his {
					return "", fmt.Errorf("cyclic reference:${%s}", his)
				}
			}
			history = append(history, newhis)
			newval, err := m.toString(history, trims, trimp)
			history = history[:len(history)-1]
			if err != nil {
				if strings.Index(err.Error(), "cyclic reference") != -1 {
					return "", err
				} else {
					newval = pv
				}
			}

			rv += newval

			//fmt.Println(v[:tmp], v[tmp:], pv, ids, history)
		}

		lastpos = index[1]
	}

	rv += value[lastpos:]

	return rv, nil

}

func (m *TierConf) ToSection(section string) (map[string]string, error) {
	if s, ok := m.conf[section]; ok {
		return s, nil
	} else {
		return nil, fmt.Errorf("section empty:%s", section)
	}

}

func (m *TierConf) ToString(section string, property string) (string, error) {
	return m.toString(nil, section, property)

}

func (m *TierConf) ToStringWithDefault(section string, property string, deft string) string {
	v, err := m.ToString(section, property)

	if err != nil {
		return deft
	} else {
		return v
	}

}

func (m *TierConf) ToInt(section string, property string) (int, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		return strconv.Atoi(v)
	}

}

func (m *TierConf) ToInt32(section string, property string) (int32, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseInt(v, 10, 32)
		return int32(i), err
	}

}

func (m *TierConf) ToInt64(section string, property string) (int64, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseInt(v, 10, 64)
		return i, err
	}

}

func (m *TierConf) ToUint64(section string, property string) (uint64, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseUint(v, 10, 64)
		return i, err
	}

}

func (m *TierConf) ToUint32(section string, property string) (uint32, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseUint(v, 10, 32)
		return uint32(i), err
	}

}

func (m *TierConf) ToFloat64(section string, property string) (float64, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseFloat(v, 64)
		return i, err
	}

}

func (m *TierConf) ToFloat32(section string, property string) (float32, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseFloat(v, 32)
		return float32(i), err
	}

}

func (m *TierConf) ToBool(section string, property string) (bool, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return false, err
	} else {
		return strconv.ParseBool(v)
	}

}

func (m *TierConf) ToBoolWithDefault(section string, property string, deft bool) bool {
	v, err := m.ToBool(section, property)

	if err != nil {
		return deft
	} else {
		return v
	}

}

func (m *TierConf) ToIntWithDefault(section string, property string, deft int) int {
	v, err := m.ToInt(section, property)
	if err != nil {
		return deft

	} else {
		return v
	}
}

func (m *TierConf) ToSliceString(section string, property string, sep string) ([]string, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return nil, err
	} else {
		ss := strings.Split(v, sep)

		for i := 0; i < len(ss); i++ {
			ss[i] = strings.Trim(ss[i], " \t")
		}
		return ss, nil
	}

}

func (m *TierConf) ToSliceInt(section string, property string, sep string) ([]int, error) {
	s, err := m.ToSliceString(section, property, sep)
	if err != nil {
		return nil, err
	} else {
		ints := make([]int, 0)
		for _, v := range(s) {
			tmp, err := strconv.Atoi(v)
			if err != nil {
				return nil, err
			} else {
				ints = append(ints, tmp)
			}

		}

		return ints, nil
	}

}
<commit_msg>增加sconf反序列化的功能<commit_after>package sconf

import (
	"bytes"
	"fmt"
	"strconv"
	"regexp"
	"strings"
	"io/ioutil"
	"sort"
	"reflect"

	"github.com/vaughan0/go-ini"
)

type TierConf struct {
	reg *regexp.Regexp
	conf map[string]map[string]string

}

func NewTierConf() *TierConf {

	return &TierConf{
		conf: make(map[string]map[string]string),
		reg: regexp.MustCompile("\\$\\{.*?\\}"),

	}
}

func (m *TierConf) StringCheck() (string, error) {
	keys := make([]string, 0)
	for s, _ := range m.conf {
		keys = append(keys, s)
	}
	sort.Strings(keys)
	rv := ""
	for _, s := range keys {
		rv += fmt.Sprintf("[%s]\n", s)

		ps := make([]string, 0)
		for s, _ := range m.conf[s] {
			ps = append(ps, s)
		}
		sort.Strings(ps)

		for _, p := range ps {
			v, err := m.ToString(s, p)
			if err != nil {
				return "", err
			}
			rv += fmt.Sprintf("%s=%s\n", p, v)
		}
		rv += fmt.Sprintf("\n")
	}

	return rv, nil
}

func (m *TierConf) GetConf() map[string]map[string]string {
	return m.conf
}

func (m *TierConf) LoadFromConf(cfg map[string]map[string]string) {
	for name, section := range cfg {
		if _, ok := m.conf[name]; !ok {
			m.conf[name] = make(map[string]string)
		}

		for k, v := range section {
			m.conf[name][k] = v
		}
	}

}

func (m *TierConf) LoadFromFile(conf string) error {
	configs := strings.Split(conf, ",")

	for _, c := range configs {
		if err := m.LoadFromOneFile(c); err != nil {
			return err
		}
	}

	return nil

}

func (m *TierConf) LoadFromOneFile(conf string) error {
	data, err := ioutil.ReadFile(conf)
	if err != nil {
		return err
	} else {
		return m.Load(data)
	}

}

func (m *TierConf) Load(cfg []byte) error {
	file, err := ini.Load(bytes.NewReader(cfg))

	if err != nil {
		return err
	}

	for name, section := range file {
		if _, ok := m.conf[name]; !ok {
			m.conf[name] = make(map[string]string)
		}

		for k, v := range section {
			m.conf[name][k] = v
		}
	}

	return nil

}

func (m *TierConf) unmarshalSliceSet(ss []string, v reflect.Value) error {
	lens := len(ss)
	sv := reflect.MakeSlice(v.Type(), lens, lens)
	for i := 0; i < lens; i++ {
		e := sv.Index(i)
		err := m.unmarshalSinSet(ss[i], e)
		if err != nil {
			return err
		}
		v.Set(sv)
	}

	return nil

}

func (m *TierConf) unmarshalSinSet(s string, v reflect.Value) error {
	switch v.Kind() {
	// 不支持[]byte
	case reflect.String:
		v.SetString(s)

	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		i, err := strconv.ParseInt(s, 10, 64)
		if err != nil {
			return err
		}
		v.SetInt(i)

	case reflect.Uint, reflect.Uintptr, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		i, err := strconv.ParseUint(s, 10, 64)
		if err != nil {
			return err
		}
		v.SetUint(i)

	case reflect.Float32, reflect.Float64:
		i, err := strconv.ParseFloat(s, 32)
		if err != nil {
			return err
		}
		v.SetFloat(i)

	case reflect.Bool:
		i, err := strconv.ParseBool(s)
		if err != nil {
			return err
		}
		v.SetBool(i)

	default:
		return fmt.Errorf("not support:%s", v.Kind())

	}

	return nil

}

func (m *TierConf) unmarshalSet(tag reflect.StructTag, s string, v reflect.Value) error {

	switch v.Kind() {

	case reflect.Slice:
		sep := tag.Get("sep")
		if len(sep) == 0 {
			sep = ","
		}
		err := m.unmarshalSliceSet(strings.Split(s, sep), v)
		if err != nil {
			return err
		}

	default:
		err := m.unmarshalSinSet(s, v)
		if err != nil {
			return err
		}

	}

	return nil

}

func (m *TierConf) unmarshalMap(cfg map[string]string, vf reflect.Value) error {

	tvf := vf.Type()
	// 类型判断
	if tvf.Kind() != reflect.Struct && tvf.Kind() != reflect.Ptr {
		return fmt.Errorf("field cannot struct or ptr")
	}

	// vf.Elem().Kind() != reflect.Struct  nil时候是Invalid
	if tvf.Kind() == reflect.Ptr && tvf.Elem().Kind() != reflect.Struct {
		return fmt.Errorf("field ptr no point to struct")
	}

	var vstruct reflect.Value
	if tvf.Kind() == reflect.Ptr {
		if vf.IsNil() {
			vf.Set(reflect.New(tvf.Elem()))
		}

		vstruct = vf.Elem()
	} else {
		vstruct = vf
	}

	tstruct := vstruct.Type()
	for sk, sv := range cfg {
		for i := 0; i < tstruct.NumField(); i++ {
			f := tstruct.Field(i)

			tag := f.Tag.Get("sconf")
			if len(tag) == 0 {
				tag = f.Name
			}
			if !strings.EqualFold(sk, tag) {
				//fmt.Println("not equal", sk, tag, f.Name)
				continue
			}
			//fmt.Println("field struct equal", f, sk, tag, f.Name)

			vf := vstruct.Field(i)

			if !vf.CanSet() {
				return fmt.Errorf("field cannot set:%s", f.Name)
			}

			if err := m.unmarshalSet(f.Tag, sv, vf); err != nil {
				return err
			}

		}
	}

	//fmt.Println("[[", vf.Interface(), "]]")

	return nil

}

func (m *TierConf) Unmarshal(v interface{}) error {
	cfg := m.conf
	value := reflect.ValueOf(v)
	k := value.Kind()
	if reflect.Ptr != k {
		return fmt.Errorf("unmarshal to obj isn't ptr:%s", k)
	}

	vstruct := value.Elem()

	k = vstruct.Kind()
	if reflect.Struct != k {
		return fmt.Errorf("unmarshal to obj elem isn't struct:%s", k)
	}

	// ===============================
	tstruct := value.Type().Elem()

	for sk, sv := range cfg {
		for i := 0; i < tstruct.NumField(); i++ {
			f := tstruct.Field(i)

			tag := f.Tag.Get("sconf")
			if len(tag) == 0 {
				tag = f.Name
			}
			if !strings.EqualFold(sk, tag) {
				//fmt.Println("not equal", sk, tag, f.Name)
				continue
			}
			//fmt.Println("equal", f, sk, tag, f.Name)

			vf := vstruct.Field(i)

			if !vf.CanSet() {
				return fmt.Errorf("field cannot set:%s", f.Name)
			}

			if err := m.unmarshalMap(sv, vf); err != nil {
				return err
			}

			//fmt.Println("MMMM", vf.Interface())

		}

	}

	/*

	f, ok := tstruct.FieldByNameFunc(func(fieldName string) bool {
		fmt.Println("FieldByNameFunc", fieldName)
		return true
	})
	fmt.Println("FieldByNameFunc result", f, ok)
	*/

	return nil
}

func (m *TierConf) toString(history []string, section string, property string) (string, error) {
	s, err := m.ToSection(section)

	if err != nil {
		return "", err

	} else {
		if p, ok := s[property]; ok {
			v, perr := m.parseVar(history, p)
			if perr != nil {
				return "", perr
			} else {
				return v, nil
			}
		} else {
			return "", fmt.Errorf("property empty:%s.%s", section, property)
		}

	}

}

func (m *TierConf) parseVar(history []string, value string) (string, error) {

	ids := m.reg.FindAllStringIndex(value, -1)

	var rv string = ""

	lastpos := 0
	for _, index := range(ids) {
		rv += value[lastpos:index[0]]
		pv := value[index[0]:index[1]]
		v := strings.Trim(pv, " \t${}")

		tmp := strings.Index(v, ".")

		if tmp == -1 {
			rv += pv
		} else {
			trims := strings.Trim(v[:tmp], " \t")
			trimp := strings.Trim(v[tmp+1:], " \t")
			// 检查循环引用
			newhis := fmt.Sprintf("%s.%s", trims, trimp)
			for _, his := range history {
				if newhis == his {
					return "", fmt.Errorf("cyclic reference:${%s}", his)
				}
			}
			history = append(history, newhis)
			newval, err := m.toString(history, trims, trimp)
			history = history[:len(history)-1]
			if err != nil {
				if strings.Index(err.Error(), "cyclic reference") != -1 {
					return "", err
				} else {
					newval = pv
				}
			}

			rv += newval

			//fmt.Println(v[:tmp], v[tmp:], pv, ids, history)
		}

		lastpos = index[1]
	}

	rv += value[lastpos:]

	return rv, nil

}

func (m *TierConf) ToSection(section string) (map[string]string, error) {
	if s, ok := m.conf[section]; ok {
		return s, nil
	} else {
		return nil, fmt.Errorf("section empty:%s", section)
	}

}

func (m *TierConf) ToString(section string, property string) (string, error) {
	return m.toString(nil, section, property)

}

func (m *TierConf) ToStringWithDefault(section string, property string, deft string) string {
	v, err := m.ToString(section, property)

	if err != nil {
		return deft
	} else {
		return v
	}

}

func (m *TierConf) ToInt(section string, property string) (int, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		return strconv.Atoi(v)
	}

}

func (m *TierConf) ToInt32(section string, property string) (int32, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseInt(v, 10, 32)
		return int32(i), err
	}

}

func (m *TierConf) ToInt64(section string, property string) (int64, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseInt(v, 10, 64)
		return i, err
	}

}

func (m *TierConf) ToUint64(section string, property string) (uint64, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseUint(v, 10, 64)
		return i, err
	}

}

func (m *TierConf) ToUint32(section string, property string) (uint32, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseUint(v, 10, 32)
		return uint32(i), err
	}

}

func (m *TierConf) ToFloat64(section string, property string) (float64, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseFloat(v, 64)
		return i, err
	}

}

func (m *TierConf) ToFloat32(section string, property string) (float32, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return 0, err
	} else {
		i, err := strconv.ParseFloat(v, 32)
		return float32(i), err
	}

}

func (m *TierConf) ToBool(section string, property string) (bool, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return false, err
	} else {
		return strconv.ParseBool(v)
	}

}

func (m *TierConf) ToBoolWithDefault(section string, property string, deft bool) bool {
	v, err := m.ToBool(section, property)

	if err != nil {
		return deft
	} else {
		return v
	}

}

func (m *TierConf) ToIntWithDefault(section string, property string, deft int) int {
	v, err := m.ToInt(section, property)
	if err != nil {
		return deft

	} else {
		return v
	}
}

func (m *TierConf) ToSliceString(section string, property string, sep string) ([]string, error) {
	v, err := m.ToString(section, property)

	if err != nil {
		return nil, err
	} else {
		ss := strings.Split(v, sep)

		for i := 0; i < len(ss); i++ {
			ss[i] = strings.Trim(ss[i], " \t")
		}
		return ss, nil
	}

}

func (m *TierConf) ToSliceInt(section string, property string, sep string) ([]int, error) {
	s, err := m.ToSliceString(section, property, sep)
	if err != nil {
		return nil, err
	} else {
		ints := make([]int, 0)
		for _, v := range(s) {
			tmp, err := strconv.Atoi(v)
			if err != nil {
				return nil, err
			} else {
				ints = append(ints, tmp)
			}

		}

		return ints, nil
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logutil

import (
	"flag"
)

func init() {
	threshold := flag.Lookup("stderrthreshold")
	if threshold == nil {
		panic("the logging module doesn't specify a stderrthreshold flag")
	}
	const warningLevel = "1"
	if err := threshold.Value.Set(warningLevel); err != nil {
		panic(err)
	}
	threshold.DefValue = warningLevel
}
<commit_msg>No panic when could not find glog flag.<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logutil

import (
	"flag"
)

func init() {
	threshold := flag.Lookup("stderrthreshold")
	if threshold == nil {
		// the logging module doesn't specify a stderrthreshold flag
		return
	}

	const warningLevel = "1"
	if err := threshold.Value.Set(warningLevel); err != nil {
		return
	}
	threshold.DefValue = warningLevel
}
<|endoftext|>


<commit_before>// Code generated by go-bindata.
// sources:
// config/emoji.toml
// DO NOT EDIT!

package main

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func bindataRead(data []byte, name string) ([]byte, error) {
	gz, err := gzip.NewReader(bytes.NewBuffer(data))
	if err != nil {
		return nil, fmt.Errorf("Read %q: %v", name, err)
	}

	var buf bytes.Buffer
	_, err = io.Copy(&buf, gz)
	clErr := gz.Close()

	if err != nil {
		return nil, fmt.Errorf("Read %q: %v", name, err)
	}
	if clErr != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

type asset struct {
	bytes []byte
	info  os.FileInfo
}

type bindataFileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
}

func (fi bindataFileInfo) Name() string {
	return fi.name
}
func (fi bindataFileInfo) Size() int64 {
	return fi.size
}
func (fi bindataFileInfo) Mode() os.FileMode {
	return fi.mode
}
func (fi bindataFileInfo) ModTime() time.Time {
	return fi.modTime
}
func (fi bindataFileInfo) IsDir() bool {
	return false
}
func (fi bindataFileInfo) Sys() interface{} {
	return nil
}

var _configEmojiToml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x4c\x90\x5d\x6e\xec\x20\x0c\x46\xdf\x67\x15\xa3\x59\xc6\x48\x77\x05\xb7\x3b\xa8\xaa\x91\xe3\x18\x62\x05\x70\x64\x4c\xd3\xee\xbe\x06\xa6\x3f\x8f\x39\x27\x3a\xfe\xc4\x2b\x4a\xce\x6c\xff\xb9\xac\xf5\xed\x12\x44\x33\xd8\xf5\xdf\xf5\x76\x07\xb5\xfb\xed\x72\x90\x0e\x56\x90\x06\x55\x40\xda\x44\x2b\xb9\x5b\x05\xeb\x80\x8b\xc8\x5e\x1d\x2c\x2d\x06\xfe\x98\xa8\x45\x07\xa8\x0d\x19\xd2\xec\xe5\xa5\xa5\xde\x71\xae\x94\xe5\x7d\x06\x03\x6b\x27\x46\xd5\x66\xec\xdc\x18\xb7\x07\x6e\x84\xfb\x23\x83\xee\x2e\x2b\x61\x53\xb6\xcf\xe1\x93\x60\x67\x8d\xe7\x17\x1f\xd5\x78\x90\x93\x8f\x81\x50\x4a\x35\x3f\x6c\x2c\xa5\x97\x21\xfe\xae\x7c\x06\xb9\xb0\x7d\xef\x32\x58\xc1\x51\x92\x18\xb9\xc4\x81\xea\x41\xb0\x93\xfe\x0c\x7d\xf9\xe3\x72\xb3\xbe\x37\x10\x58\x53\x7a\xfe\xee\xd5\x44\xfd\x05\xfc\x76\xe0\xd8\x14\xfa\xf1\x29\x8b\x9c\x21\x79\xce\xed\x57\x00\x00\x00\xff\xff\xa3\x40\xf2\x55\x6c\x01\x00\x00")

func configEmojiTomlBytes() ([]byte, error) {
	return bindataRead(
		_configEmojiToml,
		"config/emoji.toml",
	)
}

func configEmojiToml() (*asset, error) {
	bytes, err := configEmojiTomlBytes()
	if err != nil {
		return nil, err
	}

	info := bindataFileInfo{name: "config/emoji.toml", size: 364, mode: os.FileMode(420), modTime: time.Unix(1460421386, 0)}
	a := &asset{bytes: bytes, info: info}
	return a, nil
}

// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
	cannonicalName := strings.Replace(name, "\\", "/", -1)
	if f, ok := _bindata[cannonicalName]; ok {
		a, err := f()
		if err != nil {
			return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
		}
		return a.bytes, nil
	}
	return nil, fmt.Errorf("Asset %s not found", name)
}

// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
func MustAsset(name string) []byte {
	a, err := Asset(name)
	if err != nil {
		panic("asset: Asset(" + name + "): " + err.Error())
	}

	return a
}

// AssetInfo loads and returns the asset info for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func AssetInfo(name string) (os.FileInfo, error) {
	cannonicalName := strings.Replace(name, "\\", "/", -1)
	if f, ok := _bindata[cannonicalName]; ok {
		a, err := f()
		if err != nil {
			return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
		}
		return a.info, nil
	}
	return nil, fmt.Errorf("AssetInfo %s not found", name)
}

// AssetNames returns the names of the assets.
func AssetNames() []string {
	names := make([]string, 0, len(_bindata))
	for name := range _bindata {
		names = append(names, name)
	}
	return names
}

// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
	"config/emoji.toml": configEmojiToml,
}

// AssetDir returns the file names below a certain
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
//     data/
//       foo.txt
//       img/
//         a.png
//         b.png
// then AssetDir("data") would return []string{"foo.txt", "img"}
// AssetDir("data/img") would return []string{"a.png", "b.png"}
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
// AssetDir("") will return []string{"data"}.
func AssetDir(name string) ([]string, error) {
	node := _bintree
	if len(name) != 0 {
		cannonicalName := strings.Replace(name, "\\", "/", -1)
		pathList := strings.Split(cannonicalName, "/")
		for _, p := range pathList {
			node = node.Children[p]
			if node == nil {
				return nil, fmt.Errorf("Asset %s not found", name)
			}
		}
	}
	if node.Func != nil {
		return nil, fmt.Errorf("Asset %s not found", name)
	}
	rv := make([]string, 0, len(node.Children))
	for childName := range node.Children {
		rv = append(rv, childName)
	}
	return rv, nil
}

type bintree struct {
	Func     func() (*asset, error)
	Children map[string]*bintree
}

var _bintree = &bintree{nil, map[string]*bintree{
	"config": &bintree{nil, map[string]*bintree{
		"emoji.toml": &bintree{configEmojiToml, map[string]*bintree{}},
	}},
}}

// RestoreAsset restores an asset under the given directory
func RestoreAsset(dir, name string) error {
	data, err := Asset(name)
	if err != nil {
		return err
	}
	info, err := AssetInfo(name)
	if err != nil {
		return err
	}
	err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
	if err != nil {
		return err
	}
	err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
	if err != nil {
		return err
	}
	return nil
}

// RestoreAssets restores an asset under the given directory recursively
func RestoreAssets(dir, name string) error {
	children, err := AssetDir(name)
	// File
	if err != nil {
		return RestoreAsset(dir, name)
	}
	// Dir
	for _, child := range children {
		err = RestoreAssets(dir, filepath.Join(name, child))
		if err != nil {
			return err
		}
	}
	return nil
}

func _filePath(dir, name string) string {
	cannonicalName := strings.Replace(name, "\\", "/", -1)
	return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}
<commit_msg>:snowflake: Change the default configuration<commit_after>// Code generated by go-bindata.
// sources:
// config/emoji.toml
// DO NOT EDIT!

package main

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func bindataRead(data []byte, name string) ([]byte, error) {
	gz, err := gzip.NewReader(bytes.NewBuffer(data))
	if err != nil {
		return nil, fmt.Errorf("Read %q: %v", name, err)
	}

	var buf bytes.Buffer
	_, err = io.Copy(&buf, gz)
	clErr := gz.Close()

	if err != nil {
		return nil, fmt.Errorf("Read %q: %v", name, err)
	}
	if clErr != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

type asset struct {
	bytes []byte
	info  os.FileInfo
}

type bindataFileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
}

func (fi bindataFileInfo) Name() string {
	return fi.name
}
func (fi bindataFileInfo) Size() int64 {
	return fi.size
}
func (fi bindataFileInfo) Mode() os.FileMode {
	return fi.mode
}
func (fi bindataFileInfo) ModTime() time.Time {
	return fi.modTime
}
func (fi bindataFileInfo) IsDir() bool {
	return false
}
func (fi bindataFileInfo) Sys() interface{} {
	return nil
}

var _configEmojiToml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x4c\x90\x5d\x6e\xec\x20\x0c\x46\xdf\x67\x15\xa3\x59\xc6\x48\x77\x05\xb7\x3b\xa8\xaa\x91\xe3\x18\x62\x05\x70\x64\x4c\xd3\xee\xbe\x06\xa6\x3f\x8f\x39\x27\x3a\xfe\xc4\x2b\x4a\xce\x6c\xff\xb9\xac\xf5\xed\x12\x44\x33\xd8\xf5\xdf\xf5\x76\x07\xb5\xfb\xed\x72\x90\x0e\x56\x90\x06\x55\x40\xda\x44\x2b\xb9\x5b\x05\xeb\x80\x8b\xc8\x5e\x1d\x2c\x2d\x06\xfe\x98\xa8\x45\x07\xa8\x0d\x19\xd2\xec\xe5\xa5\xa5\xde\x71\xae\x94\xe5\x7d\x06\x03\x6b\x27\x46\xd5\x66\xec\xdc\x18\xb7\x07\x6e\x84\xfb\x23\x83\xee\x2e\x2b\x61\x53\xb6\xcf\xe1\x93\x60\x67\x8d\xe7\x17\x1f\xd5\x78\x90\x93\x8f\x81\x50\x4a\x35\x3f\x6c\x2c\xa5\x97\x21\xfe\xae\x7c\x06\xb9\xb0\x7d\xef\x32\x58\xc1\x51\x92\x18\xb9\xc4\x81\xea\x41\xb0\x93\xfe\x0c\x7d\xf9\xe3\x72\xb3\xbe\x37\x10\x58\x53\x7a\xfe\xee\xd5\x44\xfd\x05\xfc\x76\xe0\xd8\x14\xfa\xf1\x29\x8b\x9c\x21\x79\xce\xed\x57\x00\x00\x00\xff\xff\xa3\x40\xf2\x55\x6c\x01\x00\x00")

func configEmojiTomlBytes() ([]byte, error) {
	return bindataRead(
		_configEmojiToml,
		"config/emoji.toml",
	)
}

func configEmojiToml() (*asset, error) {
	bytes, err := configEmojiTomlBytes()
	if err != nil {
		return nil, err
	}

	info := bindataFileInfo{name: "config/emoji.toml", size: 364, mode: os.FileMode(420), modTime: time.Unix(1460421386, 0)}
	a := &asset{bytes: bytes, info: info}
	return a, nil
}

// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
	cannonicalName := strings.Replace(name, "\\", "/", -1)
	if f, ok := _bindata[cannonicalName]; ok {
		a, err := f()
		if err != nil {
			return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
		}
		return a.bytes, nil
	}
	return nil, fmt.Errorf("Asset %s not found", name)
}

// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
func MustAsset(name string) []byte {
	a, err := Asset(name)
	if err != nil {
		panic("asset: Asset(" + name + "): " + err.Error())
	}

	return a
}

// AssetInfo loads and returns the asset info for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func AssetInfo(name string) (os.FileInfo, error) {
	cannonicalName := strings.Replace(name, "\\", "/", -1)
	if f, ok := _bindata[cannonicalName]; ok {
		a, err := f()
		if err != nil {
			return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
		}
		return a.info, nil
	}
	return nil, fmt.Errorf("AssetInfo %s not found", name)
}

// AssetNames returns the names of the assets.
func AssetNames() []string {
	names := make([]string, 0, len(_bindata))
	for name := range _bindata {
		names = append(names, name)
	}
	return names
}

// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
	"config/emoji.toml": configEmojiToml,
}

// AssetDir returns the file names below a certain
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
//     data/
//       foo.txt
//       img/
//         a.png
//         b.png
// then AssetDir("data") would return []string{"foo.txt", "img"}
// AssetDir("data/img") would return []string{"a.png", "b.png"}
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
// AssetDir("") will return []string{"data"}.
func AssetDir(name string) ([]string, error) {
	node := _bintree
	if len(name) != 0 {
		cannonicalName := strings.Replace(name, "\\", "/", -1)
		pathList := strings.Split(cannonicalName, "/")
		for _, p := range pathList {
			node = node.Children[p]
			if node == nil {
				return nil, fmt.Errorf("Asset %s not found", name)
			}
		}
	}
	if node.Func != nil {
		return nil, fmt.Errorf("Asset %s not found", name)
	}
	rv := make([]string, 0, len(node.Children))
	for childName := range node.Children {
		rv = append(rv, childName)
	}
	return rv, nil
}

type bintree struct {
	Func     func() (*asset, error)
	Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
	"config": &bintree{nil, map[string]*bintree{
		"emoji.toml": &bintree{configEmojiToml, map[string]*bintree{}},
	}},
}}

// RestoreAsset restores an asset under the given directory
func RestoreAsset(dir, name string) error {
	data, err := Asset(name)
	if err != nil {
		return err
	}
	info, err := AssetInfo(name)
	if err != nil {
		return err
	}
	err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
	if err != nil {
		return err
	}
	err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
	if err != nil {
		return err
	}
	return nil
}

// RestoreAssets restores an asset under the given directory recursively
func RestoreAssets(dir, name string) error {
	children, err := AssetDir(name)
	// File
	if err != nil {
		return RestoreAsset(dir, name)
	}
	// Dir
	for _, child := range children {
		err = RestoreAssets(dir, filepath.Join(name, child))
		if err != nil {
			return err
		}
	}
	return nil
}

func _filePath(dir, name string) string {
	cannonicalName := strings.Replace(name, "\\", "/", -1)
	return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}

<|endoftext|>


<commit_before>/*
Copyright 2014 Huawei Technologies Co., Ltd. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package handler

import (
	"encoding/json"
	"net/http"

	"github.com/Huawei/containerops/pilotage/module"

	"gopkg.in/macaron.v1"
)

//GetSettingV1Handler is get user's system default setting.
func GetSettingV1Handler(ctx *macaron.Context) (int, []byte) {
	result, _ := json.Marshal(map[string]string{"message": ""})

	namespace := ctx.Params(":namespace")
	if namespace == "" {
		result, _ = json.Marshal(map[string]string{"errMsg": "namespace can't be empty"})
		return http.StatusBadRequest, result
	}

	repository := ctx.Params(":repository")
	if repository == "" {
		result, _ = json.Marshal(map[string]string{"errMsg": "repository can't be empty"})
		return http.StatusBadRequest, result
	}

	setting, err := module.GetUserSetting(namespace, repository)
	if err != nil {
		result, _ = json.Marshal(map[string]string{"errMsg": "error when get user setting"})
		return http.StatusBadRequest, result
	}

	settingMap := make(map[string]interface{})
	if setting.Setting != "" {
		err = json.Unmarshal([]byte(setting.Setting), &settingMap)
		if err != nil {
			result, _ = json.Marshal(map[string]string{"errMsg": "error when get setting info"})
			return http.StatusBadRequest, result
		}
	}

	result, _ = json.Marshal(map[string]interface{}{"setting": settingMap})
	return http.StatusOK, result
}

//PutSettingV1Handler is set user's system default setting.
func PutSettingV1Handler(ctx *macaron.Context) (int, []byte) {
	result, _ := json.Marshal(map[string]string{"message": ""})

	body := new(struct {
		Setting map[string]interface{} `json:"setting"`
	})

	namespace := ctx.Params(":namespace")
	if namespace == "" {
		result, _ = json.Marshal(map[string]string{"errMsg": "namespace can't be empty"})
		return http.StatusBadRequest, result
	}

	repository := ctx.Params(":repository")
	if repository == "" {
		result, _ = json.Marshal(map[string]string{"errMsg": "repository can't be empty"})
		return http.StatusBadRequest, result
	}

	reqBody, err := ctx.Req.Body().Bytes()
	if err != nil {
		result, _ = json.Marshal(map[string]string{"errMsg": "error when get request body:" + err.Error()})
		return http.StatusBadRequest, result
	}

	err = json.Unmarshal(reqBody, &body)
	if err != nil {
		result, _ = json.Marshal(map[string]string{"errMsg": "error when unmarshal request body:" + err.Error()})
		return http.StatusBadRequest, result
	}

	setting, err := module.GetUserSetting(namespace, repository)
	if err != nil {
		result, _ = json.Marshal(map[string]string{"errMsg": "error when get user setting"})
		return http.StatusBadRequest, result
	}

	err = setting.SetUserSetting(namespace, repository, body.Setting)
	if err != nil {
		result, _ = json.Marshal(map[string]string{"errMsg": "error when save user setting"})
		return http.StatusBadRequest, result
	}

	result, _ = json.Marshal(map[string]string{"message": "success"})
	return http.StatusOK, result
}
<commit_msg>system config set default key<commit_after>/*
Copyright 2014 Huawei Technologies Co., Ltd. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package handler

import (
	"encoding/json"
	"net/http"

	"github.com/Huawei/containerops/pilotage/module"

	"gopkg.in/macaron.v1"
)

//GetSettingV1Handler is get user's system default setting.
func GetSettingV1Handler(ctx *macaron.Context) (int, []byte) {
	result, _ := json.Marshal(map[string]string{"message": ""})

	namespace := ctx.Params(":namespace")
	if namespace == "" {
		result, _ = json.Marshal(map[string]string{"errMsg": "namespace can't be empty"})
		return http.StatusBadRequest, result
	}

	repository := ctx.Params(":repository")
	if repository == "" {
		result, _ = json.Marshal(map[string]string{"errMsg": "repository can't be empty"})
		return http.StatusBadRequest, result
	}

	setting, err := module.GetUserSetting(namespace, repository)
	if err != nil {
		result, _ = json.Marshal(map[string]string{"errMsg": "error when get user setting"})
		return http.StatusBadRequest, result
	}

	settingMap := make(map[string]interface{})
	settingMap["KUBE_APISERVER_IP"] = ""
	settingMap["KUBE_NODE_IP"] = ""

	if setting.Setting != "" {
		err = json.Unmarshal([]byte(setting.Setting), &settingMap)
		if err != nil {
			result, _ = json.Marshal(map[string]string{"errMsg": "error when get setting info"})
			return http.StatusBadRequest, result
		}
	}

	result, _ = json.Marshal(map[string]interface{}{"setting": settingMap})
	return http.StatusOK, result
}

//PutSettingV1Handler is set user's system default setting.
func PutSettingV1Handler(ctx *macaron.Context) (int, []byte) {
	result, _ := json.Marshal(map[string]string{"message": ""})

	body := new(struct {
		Setting map[string]interface{} `json:"setting"`
	})

	namespace := ctx.Params(":namespace")
	if namespace == "" {
		result, _ = json.Marshal(map[string]string{"errMsg": "namespace can't be empty"})
		return http.StatusBadRequest, result
	}

	repository := ctx.Params(":repository")
	if repository == "" {
		result, _ = json.Marshal(map[string]string{"errMsg": "repository can't be empty"})
		return http.StatusBadRequest, result
	}

	reqBody, err := ctx.Req.Body().Bytes()
	if err != nil {
		result, _ = json.Marshal(map[string]string{"errMsg": "error when get request body:" + err.Error()})
		return http.StatusBadRequest, result
	}

	err = json.Unmarshal(reqBody, &body)
	if err != nil {
		result, _ = json.Marshal(map[string]string{"errMsg": "error when unmarshal request body:" + err.Error()})
		return http.StatusBadRequest, result
	}

	setting, err := module.GetUserSetting(namespace, repository)
	if err != nil {
		result, _ = json.Marshal(map[string]string{"errMsg": "error when get user setting"})
		return http.StatusBadRequest, result
	}

	err = setting.SetUserSetting(namespace, repository, body.Setting)
	if err != nil {
		result, _ = json.Marshal(map[string]string{"errMsg": "error when save user setting"})
		return http.StatusBadRequest, result
	}

	result, _ = json.Marshal(map[string]string{"message": "success"})
	return http.StatusOK, result
}
<|endoftext|>


<commit_before>package sendgrid

import (
	"context"
	"errors"
	"net/http"
	"strconv"
	"time"

	"github.com/sendgrid/rest"
	"github.com/sendgrid/sendgrid-go/helpers/mail"
)

// Version is this client library's current version
const (
	Version        = "3.10.4"
	rateLimitRetry = 5
	rateLimitSleep = 1100
)

type options struct {
	Auth     string
	Endpoint string
	Host     string
	Subuser  string
}

// Client is the Twilio SendGrid Go client
type Client struct {
	rest.Request
}

func (o *options) baseURL() string {
	return o.Host + o.Endpoint
}

// requestNew create Request
// @return [Request] a default request object
func requestNew(options options) rest.Request {
	requestHeaders := map[string]string{
		"Authorization": options.Auth,
		"User-Agent":    "sendgrid/" + Version + ";go",
		"Accept":        "application/json",
	}

	if len(options.Subuser) != 0 {
		requestHeaders["On-Behalf-Of"] = options.Subuser
	}

	return rest.Request{
		BaseURL: options.baseURL(),
		Headers: requestHeaders,
	}
}

// Send sends an email through Twilio SendGrid
func (cl *Client) Send(email *mail.SGMailV3) (*rest.Response, error) {
	return cl.SendWithContext(context.Background(), email)
}

// SendWithContext sends an email through Twilio SendGrid with context.Context.
func (cl *Client) SendWithContext(ctx context.Context, email *mail.SGMailV3) (*rest.Response, error) {
	cl.Body = mail.GetRequestBody(email)
	return MakeRequestWithContext(ctx, cl.Request)
}

// DefaultClient is used if no custom HTTP client is defined
var DefaultClient = rest.DefaultClient

// API sets up the request to the Twilio SendGrid API, this is main interface.
// Please use the MakeRequest or MakeRequestAsync functions instead.
// (deprecated)
func API(request rest.Request) (*rest.Response, error) {
	return MakeRequest(request)
}

// MakeRequest attempts a Twilio SendGrid request synchronously.
func MakeRequest(request rest.Request) (*rest.Response, error) {
	return MakeRequestWithContext(context.Background(), request)
}

// MakeRequestWithContext attempts a Twilio SendGrid request synchronously with context.Context.
func MakeRequestWithContext(ctx context.Context, request rest.Request) (*rest.Response, error) {
	return DefaultClient.SendWithContext(ctx, request)
}

// MakeRequestRetry a synchronous request, but retry in the event of a rate
// limited response.
func MakeRequestRetry(request rest.Request) (*rest.Response, error) {
	return MakeRequestRetryWithContext(context.Background(), request)
}

// MakeRequestRetryWithContext a synchronous request with context.Context, but retry in the event of a rate
// limited response.
func MakeRequestRetryWithContext(ctx context.Context, request rest.Request) (*rest.Response, error) {
	retry := 0
	var response *rest.Response
	var err error

	for {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}
		response, err = MakeRequestWithContext(ctx, request)
		if err != nil {
			return nil, err
		}

		if response.StatusCode != http.StatusTooManyRequests {
			return response, nil
		}

		if retry > rateLimitRetry {
			return nil, errors.New("rate limit retry exceeded")
		}
		retry++

		resetTime := time.Now().Add(rateLimitSleep * time.Millisecond)

		reset, ok := response.Headers["X-RateLimit-Reset"]
		if ok && len(reset) > 0 {
			t, err := strconv.Atoi(reset[0])
			if err == nil {
				resetTime = time.Unix(int64(t), 0)
			}
		}
		time.Sleep(resetTime.Sub(time.Now()))
	}
}

// MakeRequestAsync attempts a request asynchronously in a new go
// routine. This function returns two channels: responses
// and errors. This function will retry in the case of a
// rate limit.
func MakeRequestAsync(request rest.Request) (chan *rest.Response, chan error) {
	return MakeRequestAsyncWithContext(context.Background(), request)
}

// MakeRequestAsyncWithContext attempts a request asynchronously in a new go
// routine with context.Context. This function returns two channels: responses
// and errors. This function will retry in the case of a
// rate limit.
func MakeRequestAsyncWithContext(ctx context.Context, request rest.Request) (chan *rest.Response, chan error) {
	r := make(chan *rest.Response)
	e := make(chan error)

	go func() {
		response, err := MakeRequestRetryWithContext(ctx, request)
		if err != nil {
			e <- err
		}
		if response != nil {
			r <- response
		}
	}()

	return r, e
}
<commit_msg>Release v3.10.5<commit_after>package sendgrid

import (
	"context"
	"errors"
	"net/http"
	"strconv"
	"time"

	"github.com/sendgrid/rest"
	"github.com/sendgrid/sendgrid-go/helpers/mail"
)

// Version is this client library's current version
const (
	Version        = "3.10.5"
	rateLimitRetry = 5
	rateLimitSleep = 1100
)

type options struct {
	Auth     string
	Endpoint string
	Host     string
	Subuser  string
}

// Client is the Twilio SendGrid Go client
type Client struct {
	rest.Request
}

func (o *options) baseURL() string {
	return o.Host + o.Endpoint
}

// requestNew create Request
// @return [Request] a default request object
func requestNew(options options) rest.Request {
	requestHeaders := map[string]string{
		"Authorization": options.Auth,
		"User-Agent":    "sendgrid/" + Version + ";go",
		"Accept":        "application/json",
	}

	if len(options.Subuser) != 0 {
		requestHeaders["On-Behalf-Of"] = options.Subuser
	}

	return rest.Request{
		BaseURL: options.baseURL(),
		Headers: requestHeaders,
	}
}

// Send sends an email through Twilio SendGrid
func (cl *Client) Send(email *mail.SGMailV3) (*rest.Response, error) {
	return cl.SendWithContext(context.Background(), email)
}

// SendWithContext sends an email through Twilio SendGrid with context.Context.
func (cl *Client) SendWithContext(ctx context.Context, email *mail.SGMailV3) (*rest.Response, error) {
	cl.Body = mail.GetRequestBody(email)
	return MakeRequestWithContext(ctx, cl.Request)
}

// DefaultClient is used if no custom HTTP client is defined
var DefaultClient = rest.DefaultClient

// API sets up the request to the Twilio SendGrid API, this is main interface.
// Please use the MakeRequest or MakeRequestAsync functions instead.
// (deprecated)
func API(request rest.Request) (*rest.Response, error) {
	return MakeRequest(request)
}

// MakeRequest attempts a Twilio SendGrid request synchronously.
func MakeRequest(request rest.Request) (*rest.Response, error) {
	return MakeRequestWithContext(context.Background(), request)
}

// MakeRequestWithContext attempts a Twilio SendGrid request synchronously with context.Context.
func MakeRequestWithContext(ctx context.Context, request rest.Request) (*rest.Response, error) {
	return DefaultClient.SendWithContext(ctx, request)
}

// MakeRequestRetry a synchronous request, but retry in the event of a rate
// limited response.
func MakeRequestRetry(request rest.Request) (*rest.Response, error) {
	return MakeRequestRetryWithContext(context.Background(), request)
}

// MakeRequestRetryWithContext a synchronous request with context.Context, but retry in the event of a rate
// limited response.
func MakeRequestRetryWithContext(ctx context.Context, request rest.Request) (*rest.Response, error) {
	retry := 0
	var response *rest.Response
	var err error

	for {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}
		response, err = MakeRequestWithContext(ctx, request)
		if err != nil {
			return nil, err
		}

		if response.StatusCode != http.StatusTooManyRequests {
			return response, nil
		}

		if retry > rateLimitRetry {
			return nil, errors.New("rate limit retry exceeded")
		}
		retry++

		resetTime := time.Now().Add(rateLimitSleep * time.Millisecond)

		reset, ok := response.Headers["X-RateLimit-Reset"]
		if ok && len(reset) > 0 {
			t, err := strconv.Atoi(reset[0])
			if err == nil {
				resetTime = time.Unix(int64(t), 0)
			}
		}
		time.Sleep(resetTime.Sub(time.Now()))
	}
}

// MakeRequestAsync attempts a request asynchronously in a new go
// routine. This function returns two channels: responses
// and errors. This function will retry in the case of a
// rate limit.
func MakeRequestAsync(request rest.Request) (chan *rest.Response, chan error) {
	return MakeRequestAsyncWithContext(context.Background(), request)
}

// MakeRequestAsyncWithContext attempts a request asynchronously in a new go
// routine with context.Context. This function returns two channels: responses
// and errors. This function will retry in the case of a
// rate limit.
func MakeRequestAsyncWithContext(ctx context.Context, request rest.Request) (chan *rest.Response, chan error) {
	r := make(chan *rest.Response)
	e := make(chan error)

	go func() {
		response, err := MakeRequestRetryWithContext(ctx, request)
		if err != nil {
			e <- err
		}
		if response != nil {
			r <- response
		}
	}()

	return r, e
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"runtime/debug"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/httplog"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/version"
	"github.com/golang/glog"
)

// mux is an object that can register http handlers.
type mux interface {
	Handle(pattern string, handler http.Handler)
	HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request))
}

// defaultAPIServer exposes nested objects for testability.
type defaultAPIServer struct {
	http.Handler
	group *APIGroup
}

// Handle returns a Handler function that expose the provided storage interfaces
// as RESTful resources at prefix, serialized by codec, and also includes the support
// http resources.
func Handle(storage map[string]RESTStorage, codec runtime.Codec, prefix string) http.Handler {
	group := NewAPIGroup(storage, codec)

	mux := http.NewServeMux()
	group.InstallREST(mux, prefix)
	InstallSupport(mux)

	return &defaultAPIServer{RecoverPanics(mux), group}
}

// APIGroup is a http.Handler that exposes multiple RESTStorage objects
// It handles URLs of the form:
// /${storage_key}[/${object_name}]
// Where 'storage_key' points to a RESTStorage object stored in storage.
//
// TODO: consider migrating this to go-restful which is a more full-featured version of the same thing.
type APIGroup struct {
	handler RESTHandler
}

// NewAPIGroup returns an object that will serve a set of REST resources and their
// associated operations.  The provided codec controls serialization and deserialization.
// This is a helper method for registering multiple sets of REST handlers under different
// prefixes onto a server.
// TODO: add multitype codec serialization
func NewAPIGroup(storage map[string]RESTStorage, codec runtime.Codec) *APIGroup {
	return &APIGroup{RESTHandler{
		storage: storage,
		codec:   codec,
		ops:     NewOperations(),
		// Delay just long enough to handle most simple write operations
		asyncOpWait: time.Millisecond * 25,
	}}
}

// InstallREST registers the REST handlers (storage, watch, and operations) into a mux.
// It is expected that the provided prefix will serve all operations. Path MUST NOT end
// in a slash.
func (g *APIGroup) InstallREST(mux mux, paths ...string) {
	restHandler := &g.handler
	watchHandler := &WatchHandler{g.handler.storage, g.handler.codec}
	redirectHandler := &RedirectHandler{g.handler.storage, g.handler.codec}
	opHandler := &OperationHandler{g.handler.ops, g.handler.codec}

	for _, prefix := range paths {
		prefix = strings.TrimRight(prefix, "/")
		proxyHandler := &ProxyHandler{prefix + "/proxy/", g.handler.storage, g.handler.codec}
		mux.Handle(prefix+"/", http.StripPrefix(prefix, restHandler))
		mux.Handle(prefix+"/watch/", http.StripPrefix(prefix+"/watch/", watchHandler))
		mux.Handle(prefix+"/proxy/", http.StripPrefix(prefix+"/proxy/", proxyHandler))
		mux.Handle(prefix+"/redirect/", http.StripPrefix(prefix+"/redirect/", redirectHandler))
		mux.Handle(prefix+"/operations", http.StripPrefix(prefix+"/operations", opHandler))
		mux.Handle(prefix+"/operations/", http.StripPrefix(prefix+"/operations/", opHandler))
	}
}

// InstallSupport registers the APIServer support functions into a mux.
func InstallSupport(mux mux) {
	healthz.InstallHandler(mux)
	mux.Handle("/logs/", http.StripPrefix("/logs/", http.FileServer(http.Dir("/var/log/"))))
	mux.Handle("/proxy/minion/", http.StripPrefix("/proxy/minion", http.HandlerFunc(handleProxyMinion)))
	mux.HandleFunc("/version", handleVersion)
	mux.HandleFunc("/", handleIndex)
}

// RecoverPanics wraps an http Handler to recover and log panics.
func RecoverPanics(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		defer func() {
			if x := recover(); x != nil {
				w.WriteHeader(http.StatusInternalServerError)
				fmt.Fprint(w, "apis panic. Look in log for details.")
				glog.Infof("APIServer panic'd on %v %v: %#v\n%s\n", req.Method, req.RequestURI, x, debug.Stack())
			}
		}()
		defer httplog.NewLogged(req, &w).StacktraceWhen(
			httplog.StatusIsNot(
				http.StatusOK,
				http.StatusAccepted,
				http.StatusTemporaryRedirect,
				http.StatusConflict,
				http.StatusNotFound,
			),
		).Log()

		// Dispatch to the internal handler
		handler.ServeHTTP(w, req)
	})
}

// handleVersion writes the server's version information.
func handleVersion(w http.ResponseWriter, req *http.Request) {
	writeRawJSON(http.StatusOK, version.Get(), w)
}

// writeJSON renders an object as JSON to the response.
func writeJSON(statusCode int, codec runtime.Codec, object runtime.Object, w http.ResponseWriter) {
	output, err := codec.Encode(object)
	if err != nil {
		errorJSON(err, codec, w)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(statusCode)
	w.Write(output)
}

// errorJSON renders an error to the response.
func errorJSON(err error, codec runtime.Codec, w http.ResponseWriter) {
	status := errToAPIStatus(err)
	writeJSON(status.Code, codec, status, w)
}

// writeRawJSON writes a non-API object in JSON.
func writeRawJSON(statusCode int, object interface{}, w http.ResponseWriter) {
	output, err := json.Marshal(object)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(statusCode)
	w.Write(output)
}

func parseTimeout(str string) time.Duration {
	if str != "" {
		timeout, err := time.ParseDuration(str)
		if err == nil {
			return timeout
		}
		glog.Errorf("Failed to parse: %#v '%s'", err, str)
	}
	return 30 * time.Second
}

func readBody(req *http.Request) ([]byte, error) {
	defer req.Body.Close()
	return ioutil.ReadAll(req.Body)
}

// splitPath returns the segments for a URL path.
func splitPath(path string) []string {
	path = strings.Trim(path, "/")
	if path == "" {
		return []string{}
	}
	return strings.Split(path, "/")
}
<commit_msg>Don't stack trace on 301s<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"runtime/debug"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/httplog"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/version"
	"github.com/golang/glog"
)

// mux is an object that can register http handlers.
type mux interface {
	Handle(pattern string, handler http.Handler)
	HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request))
}

// defaultAPIServer exposes nested objects for testability.
type defaultAPIServer struct {
	http.Handler
	group *APIGroup
}

const (
	StatusUnprocessableEntity = 422
)

// Handle returns a Handler function that expose the provided storage interfaces
// as RESTful resources at prefix, serialized by codec, and also includes the support
// http resources.
func Handle(storage map[string]RESTStorage, codec runtime.Codec, prefix string) http.Handler {
	group := NewAPIGroup(storage, codec)

	mux := http.NewServeMux()
	group.InstallREST(mux, prefix)
	InstallSupport(mux)

	return &defaultAPIServer{RecoverPanics(mux), group}
}

// APIGroup is a http.Handler that exposes multiple RESTStorage objects
// It handles URLs of the form:
// /${storage_key}[/${object_name}]
// Where 'storage_key' points to a RESTStorage object stored in storage.
//
// TODO: consider migrating this to go-restful which is a more full-featured version of the same thing.
type APIGroup struct {
	handler RESTHandler
}

// NewAPIGroup returns an object that will serve a set of REST resources and their
// associated operations.  The provided codec controls serialization and deserialization.
// This is a helper method for registering multiple sets of REST handlers under different
// prefixes onto a server.
// TODO: add multitype codec serialization
func NewAPIGroup(storage map[string]RESTStorage, codec runtime.Codec) *APIGroup {
	return &APIGroup{RESTHandler{
		storage: storage,
		codec:   codec,
		ops:     NewOperations(),
		// Delay just long enough to handle most simple write operations
		asyncOpWait: time.Millisecond * 25,
	}}
}

// InstallREST registers the REST handlers (storage, watch, and operations) into a mux.
// It is expected that the provided prefix will serve all operations. Path MUST NOT end
// in a slash.
func (g *APIGroup) InstallREST(mux mux, paths ...string) {
	restHandler := &g.handler
	watchHandler := &WatchHandler{g.handler.storage, g.handler.codec}
	redirectHandler := &RedirectHandler{g.handler.storage, g.handler.codec}
	opHandler := &OperationHandler{g.handler.ops, g.handler.codec}

	for _, prefix := range paths {
		prefix = strings.TrimRight(prefix, "/")
		proxyHandler := &ProxyHandler{prefix + "/proxy/", g.handler.storage, g.handler.codec}
		mux.Handle(prefix+"/", http.StripPrefix(prefix, restHandler))
		mux.Handle(prefix+"/watch/", http.StripPrefix(prefix+"/watch/", watchHandler))
		mux.Handle(prefix+"/proxy/", http.StripPrefix(prefix+"/proxy/", proxyHandler))
		mux.Handle(prefix+"/redirect/", http.StripPrefix(prefix+"/redirect/", redirectHandler))
		mux.Handle(prefix+"/operations", http.StripPrefix(prefix+"/operations", opHandler))
		mux.Handle(prefix+"/operations/", http.StripPrefix(prefix+"/operations/", opHandler))
	}
}

// InstallSupport registers the APIServer support functions into a mux.
func InstallSupport(mux mux) {
	healthz.InstallHandler(mux)
	mux.Handle("/logs/", http.StripPrefix("/logs/", http.FileServer(http.Dir("/var/log/"))))
	mux.Handle("/proxy/minion/", http.StripPrefix("/proxy/minion", http.HandlerFunc(handleProxyMinion)))
	mux.HandleFunc("/version", handleVersion)
	mux.HandleFunc("/", handleIndex)
}

// RecoverPanics wraps an http Handler to recover and log panics.
func RecoverPanics(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		defer func() {
			if x := recover(); x != nil {
				w.WriteHeader(http.StatusInternalServerError)
				fmt.Fprint(w, "apis panic. Look in log for details.")
				glog.Infof("APIServer panic'd on %v %v: %#v\n%s\n", req.Method, req.RequestURI, x, debug.Stack())
			}
		}()
		defer httplog.NewLogged(req, &w).StacktraceWhen(
			httplog.StatusIsNot(
				http.StatusOK,
				http.StatusAccepted,
				http.StatusMovedPermanently,
				http.StatusTemporaryRedirect,
				http.StatusConflict,
				http.StatusNotFound,
				StatusUnprocessableEntity,
			),
		).Log()

		// Dispatch to the internal handler
		handler.ServeHTTP(w, req)
	})
}

// handleVersion writes the server's version information.
func handleVersion(w http.ResponseWriter, req *http.Request) {
	writeRawJSON(http.StatusOK, version.Get(), w)
}

// writeJSON renders an object as JSON to the response.
func writeJSON(statusCode int, codec runtime.Codec, object runtime.Object, w http.ResponseWriter) {
	output, err := codec.Encode(object)
	if err != nil {
		errorJSON(err, codec, w)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(statusCode)
	w.Write(output)
}

// errorJSON renders an error to the response.
func errorJSON(err error, codec runtime.Codec, w http.ResponseWriter) {
	status := errToAPIStatus(err)
	writeJSON(status.Code, codec, status, w)
}

// writeRawJSON writes a non-API object in JSON.
func writeRawJSON(statusCode int, object interface{}, w http.ResponseWriter) {
	output, err := json.Marshal(object)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(statusCode)
	w.Write(output)
}

func parseTimeout(str string) time.Duration {
	if str != "" {
		timeout, err := time.ParseDuration(str)
		if err == nil {
			return timeout
		}
		glog.Errorf("Failed to parse: %#v '%s'", err, str)
	}
	return 30 * time.Second
}

func readBody(req *http.Request) ([]byte, error) {
	defer req.Body.Close()
	return ioutil.ReadAll(req.Body)
}

// splitPath returns the segments for a URL path.
func splitPath(path string) []string {
	path = strings.Trim(path, "/")
	if path == "" {
		return []string{}
	}
	return strings.Split(path, "/")
}
<|endoftext|>


<commit_before>// Copyright (c) 2011 The Grumble Authors
// The use of this source code is goverened by a BSD-style
// license that can be found in the LICENSE-file.

// This package implements a simple disk-persisted content-addressed
// blobstore.
package blobstore

import (
	"crypto/sha1"
	"encoding/hex"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
)

type BlobStore struct {
	dir     string
	lockfn  string
	makeall bool
}

var (
	ErrLocked          = os.NewError("lockfile acquired by another process")
	ErrLockAcquirement = os.NewError("unable to acquire lockfile")
	ErrBadFile         = os.NewError("a bad file exists in the blobstore directory. unable to create container directores.")
	ErrNoSuchKey       = os.NewError("no such key")
	ErrInvalidKey      = os.NewError("invalid key")
)

var (
	defaultStore *BlobStore
	defaultMutex sync.Mutex
)

// Open an existing, or create a new BlobStore at path.
// Path must point to a directory, and must already exist.
// See NewBlobStore for more information.
func Open(path string, makeall bool) (err os.Error) {
	defaultMutex.Lock()
	defer defaultMutex.Unlock()

	if defaultStore != nil {
		panic("Default BlobStore already open")
	}

	defaultStore, err = NewBlobStore(path, makeall)
	return
}

// Close the open default BlobStore. This removes the lockfile allowing
// other processes to open the BlobStore.
func Close() (err os.Error) {
	if defaultStore == nil {
		panic("DefaultStore not open")
	}

	err = defaultStore.Close()
	if err != nil {
		return
	}

	defaultStore = nil
	return
}

// Lookup a blob by its key and return a buffer containing the contents
// of the blob.
func Get(key string) (buf []byte, err os.Error) {
	return defaultStore.Get(key)
}

// Store a blob. If the blob was successfully stored, the returned key
// can be used to retrieve the buf from the BlobStore.
func Put(buf []byte) (key string, err os.Error) {
	return defaultStore.Put(buf)
}

// Open an existing, or create a new BlobStore residing at path.
// Path must point to a directory, and must already exist.
//
// The makeall argument determines whether the BlobStore should
// create all possible blob-container directories a priori.
// This can take up a bit of disk space since the metadata for
// those directories can take up a lot of space. However, tt saves
// some I/O operations when writing blobs. (Since the BlobStore
// knows that all directories will exist, it does not need to check
// whether they do, and create them if they do not.).
func NewBlobStore(path string, makeall bool) (bs *BlobStore, err os.Error) {
	// Does the directory exist?
	dir, err := os.Open(path)
	if err != nil {
		return
	}
	dir.Close()

	// Try to acquire an exclusive lock on the blobstore.
	lockfn := filepath.Join(path, "lock")
	err = AcquireLockFile(lockfn)
	if err != nil {
		return nil, err
	}
	// Make sure to remove the lockfile if we return with an error.
	// It would be impossible for users to remove it (they wouldn't
	// know the filename.)
	defer func() {
		if err != nil {
			ReleaseLockFile(lockfn)
		}
	}()

	if makeall {
		for i := 0; i < 256; i++ {
			outer := filepath.Join(path, hex.EncodeToString([]byte{byte(i)}))
			err = os.Mkdir(outer, 0700)
			if e, ok := err.(*os.PathError); ok {
				switch e.Error {
				case os.EEXIST:
					// The file alread exists. Stat it to check whether it is indeed
					// a directory.
					fi, err := os.Stat(outer)
					if err != nil {
						return nil, err
					}
					if !fi.IsDirectory() {
						return nil, ErrBadFile
					}
				case os.ENOTDIR:
					return nil, ErrBadFile
				}
			} else if err != nil {
				return nil, err
			}
			for j := 0; j < 256; j++ {
				inner := filepath.Join(outer, hex.EncodeToString([]byte{byte(j)}))
				err = os.Mkdir(inner, 0700)
				if e, ok := err.(*os.PathError); ok {
					switch e.Error {
					case os.EEXIST:
						// The file alread exists. Stat it to check whether it is indeed
						// a directory.
						fi, err := os.Stat(inner)
						if err != nil {
							return nil, err
						}
						if !fi.IsDirectory() {
							return nil, ErrBadFile
						}
					case os.ENOTDIR:
						return nil, ErrBadFile
					}
				} else if err != nil {
					return nil, err
				}
			}
		}
	}

	bs = &BlobStore{
		dir:     path,
		lockfn:  lockfn,
		makeall: makeall,
	}
	return bs, nil
}

// Close an open BlobStore. This removes the lockfile allowing
// other processes to open the BlobStore.
func (bs *BlobStore) Close() (err os.Error) {
	return os.Remove(bs.lockfn)
}

// Checks that a given key is a valid key for the BlobStore.
// If it is, it returns the three components that make up the on-disk path
// the given key can be found or should be stored at.
func getKeyComponents(key string) (dir1, dir2, fn string, err os.Error) {
	// SHA1 digests are 40 bytes long when hex-encoded
	if len(key) != 40 {
		err = ErrInvalidKey
		return
	}
	// Check whether the string is valid hex-encoding.
	_, err = hex.DecodeString(key)
	if err != nil {
		err = ErrInvalidKey
		return
	}

	return key[0:2], key[2:4], key[4:], nil
}

// Lookup the path hat a key would have on disk.
// Returns an error if the key is not a valid BlobStore key.
func (bs *BlobStore) pathForKey(key string) (fn string, err os.Error) {
	dir1, dir2, rest, err := getKeyComponents(key)
	if err != nil {
		return
	}

	fn = filepath.Join(bs.dir, dir1, dir2, rest)
	return
}

// Lookup a blob by its key and return a buffer containing the contents
// of the blob.
func (bs *BlobStore) Get(key string) (buf []byte, err os.Error) {
	fn, err := bs.pathForKey(key)
	if err != nil {
		return
	}

	file, err := os.Open(fn)
	if e, ok := err.(*os.PathError); ok && e.Error == os.ENOENT {
		err = ErrNoSuchKey
		return
	} else if err != nil {
		return
	}

	br, err := newBlobReader(file, key)
	if err != nil {
		file.Close()
		return
	}
	defer br.Close()

	buf, err = ioutil.ReadAll(br)
	if err != nil {
		return
	}

	return
}

// Store a blob. If the blob was successfully stored, the returned key
// can be used to retrieve the buf from the BlobStore.
func (bs *BlobStore) Put(buf []byte) (key string, err os.Error) {
	// Calculate the key for the blob.  We can't really delay it more than this,
	// since we need to know the key for the blob to check whether it's already on
	// disk.
	h := sha1.New()
	h.Write(buf)
	key = hex.EncodeToString(h.Sum())

	// Get the components that make up the on-disk
	// path for the blob.
	dir1, dir2, rest, err := getKeyComponents(key)
	if err != nil {
		return
	}

	blobpath := filepath.Join(bs.dir, dir1, dir2, rest)
	blobdir := filepath.Join(bs.dir, dir1, dir2)

	// Check if the blob already exists.
	file, err := os.Open(blobpath)
	if err == nil {
		// File exists. Job's done.
		file.Close()
		return
	} else {
		if e, ok := err.(*os.PathError); ok && e.Error == os.ENOENT {
			// No such file exists on disk. Ready to rock!
		} else {
			return
		}
	}

	if !bs.makeall {
		// Make sure the leading directories exist...
		err = os.MkdirAll(blobdir, 0700)
		if err != nil {
			return
		}
	}

	// Create a temporary file to write to. Once we're done, we
	// can atomically rename the file to the correct key.
	file, err = ioutil.TempFile(blobdir, rest)
	if err != nil {
		return
	}

	tmpfn := file.Name()

	_, err = file.Write(buf)
	if err != nil {
		return
	}

	err = file.Sync()
	if err != nil {
		return "", err
	}

	err = file.Close()
	if err != nil {
		return "", err
	}

	err = os.Rename(tmpfn, blobpath)
	if err != nil {
		os.Remove(tmpfn)
		return
	}

	return key, nil
}
<commit_msg>Make pkg/blobstore tests pass on Windows.<commit_after>// Copyright (c) 2011 The Grumble Authors
// The use of this source code is goverened by a BSD-style
// license that can be found in the LICENSE-file.

// This package implements a simple disk-persisted content-addressed
// blobstore.
package blobstore

import (
	"crypto/sha1"
	"encoding/hex"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
)

type BlobStore struct {
	dir     string
	lockfn  string
	makeall bool
}

const (
	win32AlreadyExists = 183
)

var (
	ErrLocked          = os.NewError("lockfile acquired by another process")
	ErrLockAcquirement = os.NewError("unable to acquire lockfile")
	ErrBadFile         = os.NewError("a bad file exists in the blobstore directory. unable to create container directores.")
	ErrNoSuchKey       = os.NewError("no such key")
	ErrInvalidKey      = os.NewError("invalid key")
)

var (
	defaultStore *BlobStore
	defaultMutex sync.Mutex
)

// Open an existing, or create a new BlobStore at path.
// Path must point to a directory, and must already exist.
// See NewBlobStore for more information.
func Open(path string, makeall bool) (err os.Error) {
	defaultMutex.Lock()
	defer defaultMutex.Unlock()

	if defaultStore != nil {
		panic("Default BlobStore already open")
	}

	defaultStore, err = NewBlobStore(path, makeall)
	return
}

// Close the open default BlobStore. This removes the lockfile allowing
// other processes to open the BlobStore.
func Close() (err os.Error) {
	if defaultStore == nil {
		panic("DefaultStore not open")
	}

	err = defaultStore.Close()
	if err != nil {
		return
	}

	defaultStore = nil
	return
}

// Lookup a blob by its key and return a buffer containing the contents
// of the blob.
func Get(key string) (buf []byte, err os.Error) {
	return defaultStore.Get(key)
}

// Store a blob. If the blob was successfully stored, the returned key
// can be used to retrieve the buf from the BlobStore.
func Put(buf []byte) (key string, err os.Error) {
	return defaultStore.Put(buf)
}

// Open an existing, or create a new BlobStore residing at path.
// Path must point to a directory, and must already exist.
//
// The makeall argument determines whether the BlobStore should
// create all possible blob-container directories a priori.
// This can take up a bit of disk space since the metadata for
// those directories can take up a lot of space. However, tt saves
// some I/O operations when writing blobs. (Since the BlobStore
// knows that all directories will exist, it does not need to check
// whether they do, and create them if they do not.).
func NewBlobStore(path string, makeall bool) (bs *BlobStore, err os.Error) {
	// Does the directory exist?
	dir, err := os.Open(path)
	if err != nil {
		return
	}
	dir.Close()

	// Try to acquire an exclusive lock on the blobstore.
	lockfn := filepath.Join(path, "lock")
	err = AcquireLockFile(lockfn)
	if err != nil {
		return nil, err
	}
	// Make sure to remove the lockfile if we return with an error.
	// It would be impossible for users to remove it (they wouldn't
	// know the filename.)
	defer func() {
		if err != nil {
			ReleaseLockFile(lockfn)
		}
	}()

	if makeall {
		for i := 0; i < 256; i++ {
			outer := filepath.Join(path, hex.EncodeToString([]byte{byte(i)}))
			err = os.Mkdir(outer, 0700)
			if e, ok := err.(*os.PathError); ok {
				if isExistError(e) {
					// The file alread exists. Stat it to check whether it is indeed
					// a directory.
					fi, err := os.Stat(outer)
					if err != nil {
						return nil, err
					}
					if !fi.IsDirectory() {
						return nil, ErrBadFile
					}
				} else if e.Error == os.ENOTDIR {
					return nil, ErrBadFile
				}
			} else if err != nil {
				return nil, err
			}
			for j := 0; j < 256; j++ {
				inner := filepath.Join(outer, hex.EncodeToString([]byte{byte(j)}))
				err = os.Mkdir(inner, 0700)
				if e, ok := err.(*os.PathError); ok {
					if isExistError(e) {
						// The file alread exists. Stat it to check whether it is indeed
						// a directory.
						fi, err := os.Stat(inner)
						if err != nil {
							return nil, err
						}
						if !fi.IsDirectory() {
							return nil, ErrBadFile
						}
					} else if e.Error == os.ENOTDIR {
						return nil, ErrBadFile
					}
				} else if err != nil {
					return nil, err
				}
			}
		}
	}

	bs = &BlobStore{
		dir:     path,
		lockfn:  lockfn,
		makeall: makeall,
	}
	return bs, nil
}

// Close an open BlobStore. This removes the lockfile allowing
// other processes to open the BlobStore.
func (bs *BlobStore) Close() (err os.Error) {
	return os.Remove(bs.lockfn)
}

// Checks that a given key is a valid key for the BlobStore.
// If it is, it returns the three components that make up the on-disk path
// the given key can be found or should be stored at.
func getKeyComponents(key string) (dir1, dir2, fn string, err os.Error) {
	// SHA1 digests are 40 bytes long when hex-encoded
	if len(key) != 40 {
		err = ErrInvalidKey
		return
	}
	// Check whether the string is valid hex-encoding.
	_, err = hex.DecodeString(key)
	if err != nil {
		err = ErrInvalidKey
		return
	}

	return key[0:2], key[2:4], key[4:], nil
}

// Lookup the path hat a key would have on disk.
// Returns an error if the key is not a valid BlobStore key.
func (bs *BlobStore) pathForKey(key string) (fn string, err os.Error) {
	dir1, dir2, rest, err := getKeyComponents(key)
	if err != nil {
		return
	}

	fn = filepath.Join(bs.dir, dir1, dir2, rest)
	return
}

// Lookup a blob by its key and return a buffer containing the contents
// of the blob.
func (bs *BlobStore) Get(key string) (buf []byte, err os.Error) {
	fn, err := bs.pathForKey(key)
	if err != nil {
		return
	}

	file, err := os.Open(fn)
	if e, ok := err.(*os.PathError); ok && (e.Error == os.ENOENT || e.Error == os.ENOTDIR) {
		err = ErrNoSuchKey
		return
	} else if err != nil {
		return
	}

	br, err := newBlobReader(file, key)
	if err != nil {
		file.Close()
		return
	}
	defer br.Close()

	buf, err = ioutil.ReadAll(br)
	if err != nil {
		return
	}

	return
}

// Store a blob. If the blob was successfully stored, the returned key
// can be used to retrieve the buf from the BlobStore.
func (bs *BlobStore) Put(buf []byte) (key string, err os.Error) {
	// Calculate the key for the blob.  We can't really delay it more than this,
	// since we need to know the key for the blob to check whether it's already on
	// disk.
	h := sha1.New()
	h.Write(buf)
	key = hex.EncodeToString(h.Sum())

	// Get the components that make up the on-disk
	// path for the blob.
	dir1, dir2, rest, err := getKeyComponents(key)
	if err != nil {
		return
	}

	blobpath := filepath.Join(bs.dir, dir1, dir2, rest)
	blobdir := filepath.Join(bs.dir, dir1, dir2)

	// Check if the blob already exists.
	file, err := os.Open(blobpath)
	if err == nil {
		// File exists. Job's done.
		file.Close()
		return
	} else {
		if e, ok := err.(*os.PathError); ok && (e.Error == os.ENOENT || e.Error == os.ENOTDIR) {
			// No such file exists on disk. Ready to rock!
		} else {
			return
		}
	}

	if !bs.makeall {
		// Make sure the leading directories exist...
		err = os.MkdirAll(filepath.ToSlash(blobdir), 0700)
		if err != nil {
			return
		}
	}

	// Create a temporary file to write to. Once we're done, we
	// can atomically rename the file to the correct key.
	file, err = ioutil.TempFile(blobdir, rest)
	if err != nil {
		return
	}

	tmpfn := file.Name()

	_, err = file.Write(buf)
	if err != nil {
		return
	}

	err = file.Sync()
	if err != nil {
		return "", err
	}

	err = file.Close()
	if err != nil {
		return "", err
	}

	err = os.Rename(tmpfn, blobpath)
	if err != nil {
		os.Remove(tmpfn)
		return
	}

	return key, nil
}

// Check whether an os.PathError is an EXIST error.
// On Unix, it checks for EEXIST. On Windows, it checks for EEXIST
// and Errno code 183 (ERROR_ALREADY_EXISTS)
func isExistError(err *os.PathError) (exists bool) {
	if e, ok := err.Error.(os.Errno); ok && e == win32AlreadyExists {
		exists = true
	}
	if err.Error == os.EEXIST {
		exists = true
	}
	return
}
<|endoftext|>


<commit_before>package clocky

import (
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"time"

	"appengine"
	"appengine/memcache"
	"appengine/taskqueue"
	"appengine/urlfetch"
)

type Source struct {
	URL                 string
	Refresh, Expiration int
}

var Sources = map[string]Source{
	"nextbus": Source{
		URL: ("http://webservices.nextbus.com/service/publicXMLFeed?" +
			"command=predictionsForMultiStops&a=sf-muni" +
			"&stops=47|null|6825" +
			"&stops=49|null|6825" +
			"&stops=90|null|6825" +
			"&stops=10|null|5859" +
			"&stops=12|null|5859" +
			"&stops=1|null|4016" +
			"&stops=1|null|6297" +
			"&stops=27|null|5165"),
		Refresh:    10,
		Expiration: 300,
	},
	"forecast": Source{
		URL: ("http://forecast.weather.gov/MapClick.php?" +
			"lat=37.79570&lon=-122.42100&FcstType=dwml&unit=1"),
		Refresh:    3600,
		Expiration: 8 * 3600,
	},
	// NDBC latest observations for all points.  This file is much
	// smaller than the file for any individual station, because
	// the latter contains 45 days of 6-minute observations.
	// http://www.ndbc.noaa.gov/measdes.shtml
	"conditions": Source{
		URL:        "http://www.ndbc.noaa.gov/data/latest_obs/latest_obs.txt",
		Refresh:    360,
		Expiration: 1800,
	},
}

func fetch(c appengine.Context, key string) os.Error {
	s, ok := Sources[key]
	if !ok {
		return fmt.Errorf("%q not found", key)
	}

	c.Debugf("fetching %s data", key)
	transport := urlfetch.Transport{Context: c, DeadlineSeconds: 60}
	req, err := http.NewRequest("GET", s.URL, strings.NewReader(""))
	if err != nil {
		return err
	}
	resp, err := transport.RoundTrip(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("fetch: bad status %d for %s", resp.StatusCode, s.URL)
	}
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	item := &memcache.Item{
		Key:        key,
		Value:      contents,
		Expiration: int32(s.Expiration),
	}
	if err := memcache.Set(c, item); err != nil {
		return err
	}

	// We keep the last updated time in memcache.  It's not
	// updated atomically with the page, so it's only used to
	// limit the rate of fetches from the data servers.  Don't use
	// it for display; use the data creation times in the data
	// instead.  It doesn't matter to the user that we fetched a
	// weather forecast 3 minutes ago if the forecast is 48
	// minutes old.
	item = &memcache.Item{
		Key:   key + "_fresh",
		Value: []byte(strconv.Itoa64(time.Seconds())),
	}
	if err := memcache.Set(c, item); err != nil {
		return err
	}

	c.Infof("cached %d bytes of %s data", len(contents), key)
	return nil
}

func freshen(c appengine.Context, key string) os.Error {
	s, ok := Sources[key]
	if !ok {
		return fmt.Errorf("%q not found", key)
	}

	item, err := memcache.Get(c, key+"_fresh")
	if err == memcache.ErrCacheMiss {
		return fetch(c, key)
	} else if err != nil {
		return err
	}
	fresh, err := strconv.Atoi64(string(item.Value))
	if err != nil {
		return err
	}
	stale := fresh + int64(s.Refresh)
	if stale > time.Seconds() {
		return nil
	}

	t := &taskqueue.Task{Path: "/fetch/" + key}
	if _, err := taskqueue.Add(c, t, "fetch-"+key); err != nil {
		return err
	}

	return nil
}

func freshenAll(c appengine.Context) os.Error {
	ch := make(chan os.Error)
	for key, _ := range Sources {
		go func(key string) { ch <- freshen(c, key) }(key)
	}
	for _ = range Sources {
		if err := <-ch; err != nil {
			return err
		}
	}
	return nil
}

func freshenAllHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if err := freshenAll(c); err != nil {
		c.Errorf("%s", err)
		http.Error(w, err.String(), http.StatusInternalServerError)
		return
	}
	io.WriteString(w, "ok\n")
}

func init() {
	http.HandleFunc("/freshen", freshenAllHandler)
	http.HandleFunc("/_ah/warmup", freshenAllHandler)

	for key, _ := range Sources {
		h := func(key string) func(w http.ResponseWriter, r *http.Request) {
			return func(w http.ResponseWriter, r *http.Request) {
				c := appengine.NewContext(r)
				if err := fetch(c, key); err != nil {
					c.Errorf("%s", err)
					http.Error(w, err.String(), http.StatusInternalServerError)
					return
				}
			}
		}(key)
		http.HandleFunc("/fetch/"+key, h)
	}
}
<commit_msg>Return a body from /_ah/warmup.<commit_after>package clocky

import (
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"time"

	"appengine"
	"appengine/memcache"
	"appengine/taskqueue"
	"appengine/urlfetch"
)

type Source struct {
	URL                 string
	Refresh, Expiration int
}

var Sources = map[string]Source{
	"nextbus": Source{
		URL: ("http://webservices.nextbus.com/service/publicXMLFeed?" +
			"command=predictionsForMultiStops&a=sf-muni" +
			"&stops=47|null|6825" +
			"&stops=49|null|6825" +
			"&stops=90|null|6825" +
			"&stops=10|null|5859" +
			"&stops=12|null|5859" +
			"&stops=1|null|4016" +
			"&stops=1|null|6297" +
			"&stops=27|null|5165"),
		Refresh:    10,
		Expiration: 300,
	},
	"forecast": Source{
		URL: ("http://forecast.weather.gov/MapClick.php?" +
			"lat=37.79570&lon=-122.42100&FcstType=dwml&unit=1"),
		Refresh:    3600,
		Expiration: 8 * 3600,
	},
	// NDBC latest observations for all points.  This file is much
	// smaller than the file for any individual station, because
	// the latter contains 45 days of 6-minute observations.
	// http://www.ndbc.noaa.gov/measdes.shtml
	"conditions": Source{
		URL:        "http://www.ndbc.noaa.gov/data/latest_obs/latest_obs.txt",
		Refresh:    360,
		Expiration: 1800,
	},
}

func fetch(c appengine.Context, key string) os.Error {
	s, ok := Sources[key]
	if !ok {
		return fmt.Errorf("%q not found", key)
	}

	c.Debugf("fetching %s data", key)
	transport := urlfetch.Transport{Context: c, DeadlineSeconds: 60}
	req, err := http.NewRequest("GET", s.URL, strings.NewReader(""))
	if err != nil {
		return err
	}
	resp, err := transport.RoundTrip(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("fetch: bad status %d for %s", resp.StatusCode, s.URL)
	}
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	item := &memcache.Item{
		Key:        key,
		Value:      contents,
		Expiration: int32(s.Expiration),
	}
	if err := memcache.Set(c, item); err != nil {
		return err
	}

	// We keep the last updated time in memcache.  It's not
	// updated atomically with the page, so it's only used to
	// limit the rate of fetches from the data servers.  Don't use
	// it for display; use the data creation times in the data
	// instead.  It doesn't matter to the user that we fetched a
	// weather forecast 3 minutes ago if the forecast is 48
	// minutes old.
	item = &memcache.Item{
		Key:   key + "_fresh",
		Value: []byte(strconv.Itoa64(time.Seconds())),
	}
	if err := memcache.Set(c, item); err != nil {
		return err
	}

	c.Infof("cached %d bytes of %s data", len(contents), key)
	return nil
}

func freshen(c appengine.Context, key string) os.Error {
	s, ok := Sources[key]
	if !ok {
		return fmt.Errorf("%q not found", key)
	}

	item, err := memcache.Get(c, key+"_fresh")
	if err == memcache.ErrCacheMiss {
		return fetch(c, key)
	} else if err != nil {
		return err
	}
	fresh, err := strconv.Atoi64(string(item.Value))
	if err != nil {
		return err
	}
	stale := fresh + int64(s.Refresh)
	if stale > time.Seconds() {
		return nil
	}

	t := &taskqueue.Task{Path: "/fetch/" + key}
	if _, err := taskqueue.Add(c, t, "fetch-"+key); err != nil {
		return err
	}

	return nil
}

func freshenAll(c appengine.Context) os.Error {
	ch := make(chan os.Error)
	for key, _ := range Sources {
		go func(key string) { ch <- freshen(c, key) }(key)
	}
	for _ = range Sources {
		if err := <-ch; err != nil {
			return err
		}
	}
	return nil
}

func freshenAllHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if err := freshenAll(c); err != nil {
		c.Errorf("%s", err)
		http.Error(w, err.String(), http.StatusInternalServerError)
		return
	}
	io.WriteString(w, "ok\n")
}

func init() {
	http.HandleFunc("/freshen", freshenAllHandler)
	http.HandleFunc("/_ah/warmup", freshenAllHandler)

	for key, _ := range Sources {
		h := func(key string) func(w http.ResponseWriter, r *http.Request) {
			return func(w http.ResponseWriter, r *http.Request) {
				c := appengine.NewContext(r)
				if err := fetch(c, key); err != nil {
					c.Errorf("%s", err)
					http.Error(w, err.String(), http.StatusInternalServerError)
					return
				}
				io.WriteString(w, "ok\n")
			}
		}(key)
		http.HandleFunc("/fetch/"+key, h)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"reflect"
	"testing"
	"time"
)

func TestPurgeFile(t *testing.T) {
	dir, err := ioutil.TempDir("", "purgefile")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	for i := 0; i < 5; i++ {
		_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
		if err != nil {
			t.Fatal(err)
		}
	}

	stop := make(chan struct{})

	// keep at most 3 most recent files
	errch := PurgeFile(dir, "test", 3, time.Millisecond, stop)

	// create 5 more files
	for i := 5; i < 10; i++ {
		_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
		if err != nil {
			t.Fatal(err)
		}
		time.Sleep(10 * time.Millisecond)
	}

	// purge routine should purge 7 out of 10 files and only keep the
	// 3 most recent ones.
	// wait for purging for at most 100ms.
	var fnames []string
	for i := 0; i < 10; i++ {
		fnames, err = ReadDir(dir)
		if err != nil {
			t.Fatal(err)
		}
		if len(fnames) <= 3 {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	wnames := []string{"7.test", "8.test", "9.test"}
	if !reflect.DeepEqual(fnames, wnames) {
		t.Errorf("filenames = %v, want %v", fnames, wnames)
	}

	// no error should be reported from purge routine
	select {
	case err := <-errch:
		t.Errorf("unexpected purge error %v", err)
	case <-time.After(time.Millisecond):
	}
	close(stop)
}

func TestPurgeFileHoldingLock(t *testing.T) {
	dir, err := ioutil.TempDir("", "purgefile")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	for i := 0; i < 10; i++ {
		_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
		if err != nil {
			t.Fatal(err)
		}
	}

	// create a purge barrier at 5
	l, err := NewLock(path.Join(dir, fmt.Sprintf("%d.test", 5)))
	err = l.Lock()
	if err != nil {
		t.Fatal(err)
	}

	stop := make(chan struct{})
	errch := PurgeFile(dir, "test", 3, time.Millisecond, stop)
	time.Sleep(20 * time.Millisecond)

	fnames, err := ReadDir(dir)
	if err != nil {
		t.Fatal(err)
	}
	wnames := []string{"5.test", "6.test", "7.test", "8.test", "9.test"}
	if !reflect.DeepEqual(fnames, wnames) {
		t.Errorf("filenames = %v, want %v", fnames, wnames)
	}
	select {
	case err := <-errch:
		t.Errorf("unexpected purge error %v", err)
	case <-time.After(time.Millisecond):
	}

	// remove the purge barrier
	err = l.Unlock()
	if err != nil {
		t.Fatal(err)
	}
	err = l.Destroy()
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(20 * time.Millisecond)

	fnames, err = ReadDir(dir)
	if err != nil {
		t.Fatal(err)
	}
	wnames = []string{"7.test", "8.test", "9.test"}
	if !reflect.DeepEqual(fnames, wnames) {
		t.Errorf("filenames = %v, want %v", fnames, wnames)
	}
	select {
	case err := <-errch:
		t.Errorf("unexpected purge error %v", err)
	case <-time.After(time.Millisecond):
	}

	close(stop)
}
<commit_msg>pkg/fileutil: make purge test more reliable<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"reflect"
	"testing"
	"time"
)

func TestPurgeFile(t *testing.T) {
	dir, err := ioutil.TempDir("", "purgefile")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	for i := 0; i < 5; i++ {
		_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
		if err != nil {
			t.Fatal(err)
		}
	}

	stop := make(chan struct{})

	// keep at most 3 most recent files
	errch := PurgeFile(dir, "test", 3, time.Millisecond, stop)

	// create 5 more files
	for i := 5; i < 10; i++ {
		_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
		if err != nil {
			t.Fatal(err)
		}
		time.Sleep(10 * time.Millisecond)
	}

	// purge routine should purge 7 out of 10 files and only keep the
	// 3 most recent ones.
	// wait for purging for at most 100ms.
	var fnames []string
	for i := 0; i < 10; i++ {
		fnames, err = ReadDir(dir)
		if err != nil {
			t.Fatal(err)
		}
		if len(fnames) <= 3 {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	wnames := []string{"7.test", "8.test", "9.test"}
	if !reflect.DeepEqual(fnames, wnames) {
		t.Errorf("filenames = %v, want %v", fnames, wnames)
	}

	// no error should be reported from purge routine
	select {
	case err := <-errch:
		t.Errorf("unexpected purge error %v", err)
	case <-time.After(time.Millisecond):
	}
	close(stop)
}

func TestPurgeFileHoldingLock(t *testing.T) {
	dir, err := ioutil.TempDir("", "purgefile")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	for i := 0; i < 10; i++ {
		_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
		if err != nil {
			t.Fatal(err)
		}
	}

	// create a purge barrier at 5
	l, err := NewLock(path.Join(dir, fmt.Sprintf("%d.test", 5)))
	err = l.Lock()
	if err != nil {
		t.Fatal(err)
	}

	stop := make(chan struct{})
	errch := PurgeFile(dir, "test", 3, time.Millisecond, stop)

	var fnames []string
	for i := 0; i < 10; i++ {
		fnames, err = ReadDir(dir)
		if err != nil {
			t.Fatal(err)
		}
		if len(fnames) <= 5 {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	wnames := []string{"5.test", "6.test", "7.test", "8.test", "9.test"}
	if !reflect.DeepEqual(fnames, wnames) {
		t.Errorf("filenames = %v, want %v", fnames, wnames)
	}

	select {
	case err := <-errch:
		t.Errorf("unexpected purge error %v", err)
	case <-time.After(time.Millisecond):
	}

	// remove the purge barrier
	err = l.Unlock()
	if err != nil {
		t.Fatal(err)
	}
	err = l.Destroy()
	if err != nil {
		t.Fatal(err)
	}

	for i := 0; i < 10; i++ {
		fnames, err = ReadDir(dir)
		if err != nil {
			t.Fatal(err)
		}
		if len(fnames) <= 3 {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	wnames = []string{"7.test", "8.test", "9.test"}
	if !reflect.DeepEqual(fnames, wnames) {
		t.Errorf("filenames = %v, want %v", fnames, wnames)
	}

	select {
	case err := <-errch:
		t.Errorf("unexpected purge error %v", err)
	case <-time.After(time.Millisecond):
	}

	close(stop)
}
<|endoftext|>