<commit_before>package util

import (
	"reflect"
	"strconv"
	"strings"
)

func EncodeTag(tag string) (data map[string]string) {
	data = map[string]string{}
	if tag == "" {
		return
	}
	for _, item := range strings.Split(tag, " ") {
		if item == "" {
			continue
		}
		key := strings.Split(item, ":")[0]
		value := strings.Split(item, "\"")[1]
		data[key] = value
	}

	return
}

func MapListToObjList(obj interface{}, mappers []map[string]interface{}, useTag string) {

	pointer := reflect.Indirect(reflect.ValueOf(obj))
	typer := pointer.Type().Elem()

	for _, mapper := range mappers {
		item := reflect.New(typer)
		MapToObj(item.Interface(), mapper, useTag)
		pointer.Set(reflect.Append(pointer, reflect.Indirect(item)))
	}
}

func ObjListToMapList(obj interface{}, useTag string) (mappers []map[string]interface{}) {
	mappers = []map[string]interface{}{}

	value := reflect.ValueOf(obj)
	for i := 0; i < value.Len(); i = i + 1 {
		item := value.Index(i)
		mappers = append(mappers, ObjToMap(item.Interface(), useTag))
	}
	return
}

// 根据map的key=>value设置Obj的field=>fieldValue
// 如果传了useTag,那么就会根据obj的Tag的useTag的值获取mapValue并填充到field上,
// 返回设置成功的Fields列表字段
func MapToObj(obj interface{}, mapper map[string]interface{}, useTag string) (fields []string) {
	if mapper == nil || len(mapper) == 0 {
		return
	}
	pointer := reflect.Indirect(reflect.ValueOf(obj))
	var tag2field = map[string]string{}
	if useTag != "" {
		fieldTagMapper := GetTagMapperFromPool(obj)
		for k, v := range fieldTagMapper.GetFieldMapByTagName(useTag) {
			v = strings.Split(v, ",")[0]
			tag2field[v] = k
		}
	}

	fields = []string{}
	for fieldName, value := range mapper {
		if useTag != "" {
			fieldName = tag2field[fieldName]
		}
		field := pointer.FieldByName(fieldName)
		if field.IsValid() && field.CanInterface() {
			setFieldValue(field, value)
			fields = append(fields, fieldName)
		}
	}

	return
}

func MapStringToObj(obj interface{}, mapper map[string]string, useTag string) (fields []string) {
	mapper2 := map[string]interface{}{}
	for k, v := range mapper {
		mapper2[k] = v
	}
	return MapToObj(obj, mapper2, useTag)
}

func setFieldValue(field reflect.Value, value interface{}) {
	switch field.Interface().(type) {
	case bool:
		b, ok := Interface2Bool(value, false)
		if ok {
			field.SetBool(b)
		}
	case string:
		s, ok := Interface2String(value, false)
		if ok {
			field.SetString(s)
		}
	case int, int8, int16, int32, int64:
		i, ok := Interface2Int(value, false)
		if ok {
			field.SetInt(i)
		}
	case float32, float64:
		f, ok := Interface2Float(value, false)
		if ok {
			field.SetFloat(f)
		}
	default:
		println("not case type : " + field.Type().String())
		break
	}
}

func Interface2Int(value interface{}, strict bool) (v int64, ok bool) {
	switch value.(type) {
	case int:
		v, ok = int64(value.(int)), true
	case int8:
		v, ok = int64(value.(int8)), true
	case int32:
		v, ok = int64(value.(int32)), true
	case int64:
		v, ok = int64(value.(int64)), true
	}
	if ok {
		return
	}
	if strict {
		if !ok {
			return
		}
	}

	switch value.(type) {
	case string, []uint8:
		s, _ := Interface2String(value, true)
		i, err := strconv.ParseInt(s, 10, 64)
		if err == nil {
			v, ok = i, true
		}
	case float32, float64:
		f, _ := Interface2Float(value, true)
		v, ok = int64(f), true
	case bool:
		if value.(bool) {
			v = 1
		} else {
			v = 0
		}
		ok = true
	}
	return
}

func Interface2Bool(value interface{}, strict bool) (v bool, ok bool) {
	if strict {
		v, ok = value.(bool)
		return
	}
	switch value.(type) {
	case bool:
		v, ok = value.(bool), true
	case int8, int, int32, int64:
		i, _ := Interface2Int(value, true)
		v, ok = i == 1, true
	case float32, float64:
		i, _ := Interface2Float(value, true)
		v, ok = i == 1, true
	case string, []uint8:
		s, _ := Interface2String(value, true)
		s = strings.ToLower(s)
		v, ok = s == "1" || s == "true", true
	}

	return
}

func Interface2Float(value interface{}, strict bool) (v float64, ok bool) {
	switch value.(type) {
	case float32:
		v, ok = float64(value.(float32)), true
	case float64:
		v, ok = float64(value.(float64)), true
	}
	if ok {
		return
	}
	if strict {
		if !ok {
			return
		}
	}

	switch value.(type) {
	case int, int8, int32, int64:
		i, ok := Interface2Int(value, true)
		if ok {
			v, ok = float64(i), true
		}
	case string:
		f, err := strconv.ParseFloat(value.(string), 64)
		if err != nil {
			return 0, false
		}
		return f, true
	case []uint8:
		f, err := strconv.ParseFloat(string(value.([]uint8)), 64)
		if err != nil {
			return 0, false
		}
		return f, true
	}
	return
}

func Interface2String(value interface{}, strict bool) (v string, ok bool) {
	switch value.(type) {
	case string:
		v, ok = value.(string), true
	case []uint8:
		v, ok = string(value.([]uint8)), true
	}

	if ok {
		return
	}
	if strict {
		if !ok {
			return
		}
	}

	switch value.(type) {
	case int64, int8, int32, int:
		i, _ := Interface2Int(value, true)
		v, ok = strconv.FormatInt(i, 10), true
	case float64, float32:
		f, _ := Interface2Float(value, true)
		v, ok = strconv.FormatFloat(f, 'f', -1, 64), true
	case bool:
		v, ok = strconv.FormatBool(value.(bool)), true
	}
	return
}

func ObjToMap(obj interface{}, useTag string) map[string]interface{} {
	pointer := reflect.Indirect(reflect.ValueOf(obj))
	typer := pointer.Type()

	fieldNum := pointer.NumField()
	var fieldNameToTagName map[string]string
	if useTag != "" {
		fieldTagMapper := GetTagMapperFromPool(obj)
		fieldNameToTagName = fieldTagMapper.GetFieldMapByTagName(useTag)
	}

	data := map[string]interface{}{}

	for i := 0; i < fieldNum; i++ {
		field := pointer.Field(i)
		key := typer.Field(i).Name

		if useTag != "" {
			// 根据指定的tag的key重新映射
			key = fieldNameToTagName[key]
			// 如果有逗号 比如 json:"password,omitempty" 则只取逗号前面的第一个
			key = strings.Split(key, ",")[0]
			// 有值才填充
			if key != "" {
				data[key] = field.Interface()
			}
		} else {
			data[key] = field.Interface()
		}

	}

	return data
}

//判断一个array每一个原始是不是都在map的value里
func ArrayInMapValue(min []string, m map[string]string) (has bool, msg string) {
	if min == nil || len(min) == 0 {
		return true, ""
	}
	lenMin := len(min)
	for minI := 0; minI < lenMin; minI = minI + 1 {
		_, has = m[min[minI]]
		if !has {
			return false, min[minI]
		}
	}
	return true, ""
}

//获取map的keys
func GetMapKey(m map[string]string) (keys []string) {
	keys = []string{}
	for key := range m {
		keys = append(keys, key)
	}
	return keys
}

//判断一个array每一个元素是不是都在map的key里
func ArrayInMapKey(min []string, m map[string]string) (has bool, msg string) {
	if min == nil || len(min) == 0 {
		has = true
		return
	}
	if m == nil {
		has = false
		return
	}
	lenMin := len(min)
	for minI := 0; minI < lenMin; minI++ {
		_, has = m[min[minI]]
		if !has {
			msg = min[minI]
			return
		}
	}
	has = true
	return
}

func ArrayInArray(min []string, max []string) (has bool, msg string) {
	if min == nil || len(min) == 0 {
		return true, ""
	}

	lenMax := len(max)
	lenMin := len(min)
	for minI := 0; minI < lenMin; minI = minI + 1 {
		has := false
		for maxI := 0; maxI < lenMax; maxI = maxI + 1 {
			if max[maxI] == min[minI] {
				has = true
			}
		}
		if !has {
			return false, min[minI]
		}
	}
	return true, ""
}

// 判断item是否在数组里
// 如果数组为空则返回false
func ItemInArray(item string, max []string) (has bool) {
	return ArrayStringIndex(item, max) != -1
}

func ArrayStringIndex(item string, max []string) (index int) {
	index = -1
	if max == nil || len(max) == 0 {
		return
	}
	for i, l := 0, len(max); i < l; i++ {
		if max[i] == item {
			index = i
			return
		}
	}
	return
}

// 判断item是否在数组里
// 如果数组为空则返回false
func ItemInArrayInt(item int, max []int) (has bool) {

	if max == nil || len(max) == 0 {
		return false
	}

	lenMax := len(max)

	for maxI := 0; maxI < lenMax; maxI = maxI + 1 {
		if max[maxI] == item {
			return true
		}
	}

	return false
}

func IsEmptyValue(value interface{}) bool {
	v := reflect.ValueOf(value)
	switch v.Kind() {
	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
		return v.Len() == 0
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}
	return false
}

func EmptyObject(obj interface{}) {
	pointer := reflect.Indirect(reflect.ValueOf(obj))
	fieldNum := pointer.NumField()

	for i := 0; i < fieldNum; i++ {
		v := pointer.Field(i)
		switch v.Kind() {
		case reflect.String:
			v.SetString("")
			break
		case reflect.Bool:
			v.SetBool(false)
			break
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			v.SetInt(0)
			break
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
			v.SetUint(0)
			break
		case reflect.Float32, reflect.Float64:
			v.SetFloat(0)
			break
		case reflect.Interface, reflect.Ptr:
			break
		}
	}
}

func MapInterface2MapString(m map[string]interface{}) map[string]string {
	set := map[string]string{}

	for key, value := range m {
		v, ok := Interface2String(value, false)
		if ok {
			set[key] = v
		}
	}
	return set
}<commit_msg>fix not can interface bug<commit_after>package util

import (
	"reflect"
	"strconv"
	"strings"
)

func EncodeTag(tag string) (data map[string]string) {
	data = map[string]string{}
	if tag == "" {
		return
	}
	for _, item := range strings.Split(tag, " ") {
		if item == "" {
			continue
		}
		key := strings.Split(item, ":")[0]
		value := strings.Split(item, "\"")[1]
		data[key] = value
	}

	return
}

func MapListToObjList(obj interface{}, mappers []map[string]interface{}, useTag string) {

	pointer := reflect.Indirect(reflect.ValueOf(obj))
	typer := pointer.Type().Elem()

	for _, mapper := range mappers {
		item := reflect.New(typer)
		MapToObj(item.Interface(), mapper, useTag)
		pointer.Set(reflect.Append(pointer, reflect.Indirect(item)))
	}
}

func ObjListToMapList(obj interface{}, useTag string) (mappers []map[string]interface{}) {
	mappers = []map[string]interface{}{}

	value := reflect.ValueOf(obj)
	for i := 0; i < value.Len(); i = i + 1 {
		item := value.Index(i)
		mappers = append(mappers, ObjToMap(item.Interface(), useTag))
	}
	return
}

// 根据map的key=>value设置Obj的field=>fieldValue
// 如果传了useTag,那么就会根据obj的Tag的useTag的值获取mapValue并填充到field上,
// 返回设置成功的Fields列表字段
func MapToObj(obj interface{}, mapper map[string]interface{}, useTag string) (fields []string) {
	if mapper == nil || len(mapper) == 0 {
		return
	}
	pointer := reflect.Indirect(reflect.ValueOf(obj))
	var tag2field = map[string]string{}
	if useTag != "" {
		fieldTagMapper := GetTagMapperFromPool(obj)
		for k, v := range fieldTagMapper.GetFieldMapByTagName(useTag) {
			v = strings.Split(v, ",")[0]
			tag2field[v] = k
		}
	}

	fields = []string{}
	for fieldName, value := range mapper {
		if useTag != "" {
			fieldName = tag2field[fieldName]
		}
		field := pointer.FieldByName(fieldName)
		if field.IsValid() && field.CanInterface() {
			setFieldValue(field, value)
			fields = append(fields, fieldName)
		}
	}

	return
}

func MapStringToObj(obj interface{}, mapper map[string]string, useTag string) (fields []string) {
	mapper2 := map[string]interface{}{}
	for k, v := range mapper {
		mapper2[k] = v
	}
	return MapToObj(obj, mapper2, useTag)
}

func setFieldValue(field reflect.Value, value interface{}) {
	switch field.Interface().(type) {
	case bool:
		b, ok := Interface2Bool(value, false)
		if ok {
			field.SetBool(b)
		}
	case string:
		s, ok := Interface2String(value, false)
		if ok {
			field.SetString(s)
		}
	case int, int8, int16, int32, int64:
		i, ok := Interface2Int(value, false)
		if ok {
			field.SetInt(i)
		}
	case float32, float64:
		f, ok := Interface2Float(value, false)
		if ok {
			field.SetFloat(f)
		}
	default:
		println("not case type : " + field.Type().String())
		break
	}
}

func Interface2Int(value interface{}, strict bool) (v int64, ok bool) {
	switch value.(type) {
	case int:
		v, ok = int64(value.(int)), true
	case int8:
		v, ok = int64(value.(int8)), true
	case int32:
		v, ok = int64(value.(int32)), true
	case int64:
		v, ok = int64(value.(int64)), true
	}
	if ok {
		return
	}
	if strict {
		if !ok {
			return
		}
	}

	switch value.(type) {
	case string, []uint8:
		s, _ := Interface2String(value, true)
		i, err := strconv.ParseInt(s, 10, 64)
		if err == nil {
			v, ok = i, true
		}
	case float32, float64:
		f, _ := Interface2Float(value, true)
		v, ok = int64(f), true
	case bool:
		if value.(bool) {
			v = 1
		} else {
			v = 0
		}
		ok = true
	}
	return
}

func Interface2Bool(value interface{}, strict bool) (v bool, ok bool) {
	if strict {
		v, ok = value.(bool)
		return
	}
	switch value.(type) {
	case bool:
		v, ok = value.(bool), true
	case int8, int, int32, int64:
		i, _ := Interface2Int(value, true)
		v, ok = i == 1, true
	case float32, float64:
		i, _ := Interface2Float(value, true)
		v, ok = i == 1, true
	case string, []uint8:
		s, _ := Interface2String(value, true)
		s = strings.ToLower(s)
		v, ok = s == "1" || s == "true", true
	}

	return
}

func Interface2Float(value interface{}, strict bool) (v float64, ok bool) {
	switch value.(type) {
	case float32:
		v, ok = float64(value.(float32)), true
	case float64:
		v, ok = float64(value.(float64)), true
	}
	if ok {
		return
	}
	if strict {
		if !ok {
			return
		}
	}

	switch value.(type) {
	case int, int8, int32, int64:
		i, ok := Interface2Int(value, true)
		if ok {
			v, ok = float64(i), true
		}
	case string:
		f, err := strconv.ParseFloat(value.(string), 64)
		if err != nil {
			return 0, false
		}
		return f, true
	case []uint8:
		f, err := strconv.ParseFloat(string(value.([]uint8)), 64)
		if err != nil {
			return 0, false
		}
		return f, true
	}
	return
}

func Interface2String(value interface{}, strict bool) (v string, ok bool) {
	switch value.(type) {
	case string:
		v, ok = value.(string), true
	case []uint8:
		v, ok = string(value.([]uint8)), true
	}

	if ok {
		return
	}
	if strict {
		if !ok {
			return
		}
	}

	switch value.(type) {
	case int64, int8, int32, int:
		i, _ := Interface2Int(value, true)
		v, ok = strconv.FormatInt(i, 10), true
	case float64, float32:
		f, _ := Interface2Float(value, true)
		v, ok = strconv.FormatFloat(f, 'f', -1, 64), true
	case bool:
		v, ok = strconv.FormatBool(value.(bool)), true
	}
	return
}

func ObjToMap(obj interface{}, useTag string) map[string]interface{} {
	pointer := reflect.Indirect(reflect.ValueOf(obj))
	typer := pointer.Type()

	fieldNum := pointer.NumField()
	var fieldNameToTagName map[string]string
	if useTag != "" {
		fieldTagMapper := GetTagMapperFromPool(obj)
		fieldNameToTagName = fieldTagMapper.GetFieldMapByTagName(useTag)
	}

	data := map[string]interface{}{}

	for i := 0; i < fieldNum; i++ {
		field := pointer.Field(i)
		key := typer.Field(i).Name

		if !field.CanInterface(){
			continue
		}

		if useTag != "" {
			// 根据指定的tag的key重新映射
			key = fieldNameToTagName[key]
			// 如果有逗号 比如 json:"password,omitempty" 则只取逗号前面的第一个
			key = strings.Split(key, ",")[0]
			// 有值才填充
			if key != "" {
				data[key] = field.Interface()
			}
		} else {
			data[key] = field.Interface()
		}

	}

	return data
}

//判断一个array每一个原始是不是都在map的value里
func ArrayInMapValue(min []string, m map[string]string) (has bool, msg string) {
	if min == nil || len(min) == 0 {
		return true, ""
	}
	lenMin := len(min)
	for minI := 0; minI < lenMin; minI = minI + 1 {
		_, has = m[min[minI]]
		if !has {
			return false, min[minI]
		}
	}
	return true, ""
}

//获取map的keys
func GetMapKey(m map[string]string) (keys []string) {
	keys = []string{}
	for key := range m {
		keys = append(keys, key)
	}
	return keys
}

//判断一个array每一个元素是不是都在map的key里
func ArrayInMapKey(min []string, m map[string]string) (has bool, msg string) {
	if min == nil || len(min) == 0 {
		has = true
		return
	}
	if m == nil {
		has = false
		return
	}
	lenMin := len(min)
	for minI := 0; minI < lenMin; minI++ {
		_, has = m[min[minI]]
		if !has {
			msg = min[minI]
			return
		}
	}
	has = true
	return
}

func ArrayInArray(min []string, max []string) (has bool, msg string) {
	if min == nil || len(min) == 0 {
		return true, ""
	}

	lenMax := len(max)
	lenMin := len(min)
	for minI := 0; minI < lenMin; minI = minI + 1 {
		has := false
		for maxI := 0; maxI < lenMax; maxI = maxI + 1 {
			if max[maxI] == min[minI] {
				has = true
			}
		}
		if !has {
			return false, min[minI]
		}
	}
	return true, ""
}

// 判断item是否在数组里
// 如果数组为空则返回false
func ItemInArray(item string, max []string) (has bool) {
	return ArrayStringIndex(item, max) != -1
}

func ArrayStringIndex(item string, max []string) (index int) {
	index = -1
	if max == nil || len(max) == 0 {
		return
	}
	for i, l := 0, len(max); i < l; i++ {
		if max[i] == item {
			index = i
			return
		}
	}
	return
}

// 判断item是否在数组里
// 如果数组为空则返回false
func ItemInArrayInt(item int, max []int) (has bool) {

	if max == nil || len(max) == 0 {
		return false
	}

	lenMax := len(max)

	for maxI := 0; maxI < lenMax; maxI = maxI + 1 {
		if max[maxI] == item {
			return true
		}
	}

	return false
}

func IsEmptyValue(value interface{}) bool {
	v := reflect.ValueOf(value)
	switch v.Kind() {
	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
		return v.Len() == 0
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}
	return false
}

func EmptyObject(obj interface{}) {
	pointer := reflect.Indirect(reflect.ValueOf(obj))
	fieldNum := pointer.NumField()

	for i := 0; i < fieldNum; i++ {
		v := pointer.Field(i)
		switch v.Kind() {
		case reflect.String:
			v.SetString("")
			break
		case reflect.Bool:
			v.SetBool(false)
			break
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			v.SetInt(0)
			break
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
			v.SetUint(0)
			break
		case reflect.Float32, reflect.Float64:
			v.SetFloat(0)
			break
		case reflect.Interface, reflect.Ptr:
			break
		}
	}
}

func MapInterface2MapString(m map[string]interface{}) map[string]string {
	set := map[string]string{}

	for key, value := range m {
		v, ok := Interface2String(value, false)
		if ok {
			set[key] = v
		}
	}
	return set
}<|endoftext|>


<commit_before>package watch

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/jcelliott/lumber"
	"github.com/nanobox-io/nanobox-boxfile"

	"github.com/nanobox-io/nanobox/models"
	"github.com/nanobox-io/nanobox/util"
	"github.com/nanobox-io/nanobox/util/config"
)

var ignoreFile = []string{".git", ".hg", ".svn", ".bzr"}
var changeList = []string{}

// the watch package watches a folder and all its sub folders
// in doing so it may run into open file errors or things of that nature
// if it does, it will automatically fall back to a slower but still
// useful watching mechanism that looks at files and gets a hash of the content
// then when

type event struct {
	file  string
	error error
}

type Watcher interface {
	watch() error
	eventChan() chan event
	close() error
}

// watch a directory and report changes to nanobox
func Watch(container, path string) error {
	populateIgnore(path)

	lumber.Debug("watch: ignored dirs: %+v", ignoreFile)
	// try watching with the fast one
	watcher := newNotifyWatcher(path)
	err := watcher.watch()
	if err != nil {
		// if it fails display a message and try the slow one
		lumber.Info("Error occured in fast notify watcher: %s", err.Error())

		// print the warning
		fmt.Printf("\n-------------------------------------------------------\n\n")
		fmt.Printf("Uh oh, the live filesystem watcher has panic'ed.\n")
		fmt.Printf("We'll go ahead and rollover to a slower polling solution.\n")
		fmt.Printf("\n-------------------------------------------------------\n\n")

		watcher.close()
		watcher = newCrawlWatcher(path)
		err := watcher.watch()
		if err != nil {
			// neither of the watchers are working
			return err
		}
	}
	defer watcher.close()

	go batchPublish(container)

	// catch a kill signal
	for e := range watcher.eventChan() {
		efile := e.file
		if runtime.GOOS == "windows" {
			efile = strings.Replace(efile, "\\", "/", -1)
		}
		containerFile := filepath.ToSlash(filepath.Join("/app", strings.Replace(efile, config.LocalDir(), "", 1)))
		changeList = append(changeList, containerFile)
	}

	return nil
}

// publish in batches so to save clock cycles
func batchPublish(container string) {
	for {
		<-time.After(time.Second)
		if len(changeList) > 0 {
			lumber.Info("watcher: pushing: %+v", changeList)
			util.DockerExec(container, "root", "touch", append([]string{"-c"}, changeList...), nil)
			changeList = []string{}
		}
	}
}

// populate the ignore file from the nanoignore
func populateIgnore(path string) {
	// add pieces from the env
	env, err := models.FindEnvByID(config.EnvID())
	box := boxfile.New([]byte(env.BuiltBoxfile))
	for _, libDir := range box.Node("run.config").StringSliceValue("cache_dirs") {
		ignoreFile = append(ignoreFile, libDir)
	}

	// add parts from the nanoignore
	b, err := ioutil.ReadFile(filepath.ToSlash(filepath.Join(path, ".nanoignore")))
	if err != nil {
		return
	}

	stringFields := strings.Fields(string(b))
	for _, str := range stringFields {
		ignoreFile = append(ignoreFile, str)
	}

}
<commit_msg>refactor the wat we touch so mtime doesnt change<commit_after>package watch

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/jcelliott/lumber"
	"github.com/nanobox-io/nanobox-boxfile"

	"github.com/nanobox-io/nanobox/models"
	"github.com/nanobox-io/nanobox/util"
	"github.com/nanobox-io/nanobox/util/config"
)

var ignoreFile = []string{".git", ".hg", ".svn", ".bzr"}
var changeList = []string{}

// the watch package watches a folder and all its sub folders
// in doing so it may run into open file errors or things of that nature
// if it does, it will automatically fall back to a slower but still
// useful watching mechanism that looks at files and gets a hash of the content
// then when

type event struct {
	file  string
	error error
}

type Watcher interface {
	watch() error
	eventChan() chan event
	close() error
}

// watch a directory and report changes to nanobox
func Watch(container, path string) error {
	populateIgnore(path)

	lumber.Debug("watch: ignored dirs: %+v", ignoreFile)
	// try watching with the fast one
	watcher := newNotifyWatcher(path)
	err := watcher.watch()
	if err != nil {
		// if it fails display a message and try the slow one
		lumber.Info("Error occured in fast notify watcher: %s", err.Error())

		// print the warning
		fmt.Printf("\n-------------------------------------------------------\n\n")
		fmt.Printf("Uh oh, the live filesystem watcher has panic'ed.\n")
		fmt.Printf("We'll go ahead and rollover to a slower polling solution.\n")
		fmt.Printf("\n-------------------------------------------------------\n\n")

		watcher.close()
		watcher = newCrawlWatcher(path)
		err := watcher.watch()
		if err != nil {
			// neither of the watchers are working
			return err
		}
	}
	defer watcher.close()

	go batchPublish(container)

	// catch a kill signal
	for e := range watcher.eventChan() {
		efile := e.file
		if runtime.GOOS == "windows" {
			efile = strings.Replace(efile, "\\", "/", -1)
		}
		containerFile := filepath.ToSlash(filepath.Join("/app", strings.Replace(efile, config.LocalDir(), "", 1)))
		changeList = append(changeList, containerFile)
	}

	return nil
}

// publish in batches so to save clock cycles
func batchPublish(container string) {
	for {
		<-time.After(time.Second)
		if len(changeList) > 0 {
			lumber.Info("watcher: pushing: %+v", changeList)
			// util.DockerExec(container, "root", "touch", append([]string{"-c"}, changeList...), nil)
			touches := []string{}
			for _, change := range changeList {
				touches = append(touches, fmt.Sprintf("touch -c -r %s %s", change, change))
			}

			util.DockerExec(container, "root", "bash", append([]string{"-c"}, strings.Join(touches, " && ")), nil)
			changeList = []string{}
		}
	}
}

// populate the ignore file from the nanoignore
func populateIgnore(path string) {
	// add pieces from the env
	env, err := models.FindEnvByID(config.EnvID())
	box := boxfile.New([]byte(env.BuiltBoxfile))
	for _, libDir := range box.Node("run.config").StringSliceValue("cache_dirs") {
		ignoreFile = append(ignoreFile, libDir)
	}

	// add parts from the nanoignore
	b, err := ioutil.ReadFile(filepath.ToSlash(filepath.Join(path, ".nanoignore")))
	if err != nil {
		return
	}

	stringFields := strings.Fields(string(b))
	for _, str := range stringFields {
		ignoreFile = append(ignoreFile, str)
	}

}
<|endoftext|>


<commit_before>package utils

import (
    "gosync/config"
    "strings"
    "os"
    "io"
)

func GetBasePath(listener string) string {
    cfg := config.GetConfig()
    return cfg.Listeners[listener].BasePath
}

func GetBaseDir(listener string) string {
    cfg := config.GetConfig()
    return cfg.Listeners[listener].Directory
}

func GetRelativeBasePath(listener, local_path string) string {
    lPath := strings.TrimPrefix(local_path, GetBaseDir(listener))
    return GetBasePath(listener) + lPath
}

func GetRelativePath(listener, local_path string) string {
    return strings.TrimPrefix(local_path, GetBaseDir(listener))
}

func GetAbsPath(listener, db_path string) string {
    return GetBaseDir(listener) + db_path
}

func FileWrite(path string, content io.Reader, overwrite bool, uid, gid int, perms string) (int64,error ){
    if _, err := os.Stat(path); err == nil {
        // We wipe the file as we need to replace with a new one
        err := os.Remove(path)
        if err != nil {
            return 0, err
        }

    }

    file, err := os.Create(path)

    if err != nil {
        return 0, err
    }
    defer file.Close()

    size, err := io.Copy(file, content)

    file.Chown(uid, gid)
    //file.Chmod(perms)

    return size, err
}<commit_msg>Validating absolute path resolution<commit_after>package utils

import (
    "gosync/config"
    "strings"
    "os"
    "io"
    "log"
)

func GetBasePath(listener string) string {
    cfg := config.GetConfig()
    return cfg.Listeners[listener].BasePath
}

func GetBaseDir(listener string) string {
    cfg := config.GetConfig()
    return cfg.Listeners[listener].Directory
}

func GetRelativeBasePath(listener, local_path string) string {
    lPath := strings.TrimPrefix(local_path, GetBaseDir(listener))
    return GetBasePath(listener) + lPath
}

func GetRelativePath(listener, local_path string) string {
    return strings.TrimPrefix(local_path, GetBaseDir(listener))
}

func GetAbsPath(listener, db_path string) string {
    absPath := GetBaseDir(listener) + db_path
    log.Printf("=====> Absolute Path: %s <=====", absPath)
    return absPath
}

func FileWrite(path string, content io.Reader, overwrite bool, uid, gid int, perms string) (int64,error ){
    if _, err := os.Stat(path); err == nil {
        // We wipe the file as we need to replace with a new one
        err := os.Remove(path)
        if err != nil {
            return 0, err
        }

    }

    file, err := os.Create(path)

    if err != nil {
        return 0, err
    }
    defer file.Close()

    size, err := io.Copy(file, content)

    file.Chown(uid, gid)
    //file.Chmod(perms)

    return size, err
}<|endoftext|>


<commit_before>// Copyright (c) 2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"fmt"
	"path/filepath"
	"strings"

	humanize "github.com/dustin/go-humanize"
	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	"github.com/minio/madmin-go"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/pkg/console"
)

var adminBucketInfoFlags = []cli.Flag{}

type adminBucketInfoMessage struct {
	Status    string                 `json:"status"`
	URL       string                 `json:"url"`
	UsageInfo madmin.BucketUsageInfo `json:"usage"`
	Props     BucketInfo             `json:"props"`
}

func (bi adminBucketInfoMessage) String() string {
	var b strings.Builder

	fmt.Fprintf(&b, console.Colorize("Title", "Usage:\n"))

	fmt.Fprintf(&b, "%16s: %s\n", "Total size", console.Colorize("Count", humanize.IBytes(bi.UsageInfo.Size)))
	fmt.Fprintf(&b, "%16s: %s\n", "Objects count", console.Colorize("Count", humanize.Comma(int64(bi.UsageInfo.ObjectsCount))))
	fmt.Fprintf(&b, "%16s: %s\n", "Versions count", console.Colorize("Count", humanize.Comma(int64(bi.UsageInfo.VersionsCount))))
	fmt.Fprintf(&b, "\n")

	fmt.Fprintf(&b, console.Colorize("Title", "Properties:\n"))
	fmt.Fprintf(&b, prettyPrintBucketMetadata(bi.Props))
	return b.String()
}

func (bi adminBucketInfoMessage) JSON() string {
	jsonMessageBytes, e := json.MarshalIndent(bi, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(jsonMessageBytes)
}

var adminBucketInfoCmd = cli.Command{
	Name:         "info",
	Usage:        "display bucket information",
	Action:       mainAdminBucketInfo,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        append(adminBucketInfoFlags, globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Display the usage data and configuration of a bucket on MinIO.
     {{.Prompt}} {{.HelpName}} myminio/mybucket
`,
}

// checkAdminBucketInfoSyntax - validate all the passed arguments
func checkAdminBucketInfoSyntax(ctx *cli.Context) {
	if len(ctx.Args()) != 1 {
		cli.ShowCommandHelpAndExit(ctx, ctx.Command.Name, 1) // last argument is exit code
	}
}

// mainAdminBucketInfo is the handler for "mc admin bucket info" command.
func mainAdminBucketInfo(ctx *cli.Context) error {
	checkAdminBucketInfoSyntax(ctx)

	console.SetColor("Title", color.New(color.Bold, color.FgBlue))
	console.SetColor("Count", color.New(color.FgGreen))
	console.SetColor("Metadata", color.New(color.FgWhite))
	console.SetColor("Key", color.New(color.FgCyan))
	console.SetColor("Value", color.New(color.FgYellow))
	console.SetColor("Unset", color.New(color.FgRed))
	console.SetColor("Set", color.New(color.FgGreen))

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)

	// Create a new MinIO Admin Client
	adminClient, err := newAdminClient(aliasedURL)
	fatalIf(err, "Unable to initialize admin connection.")

	s3Client, err := newClient(aliasedURL)
	fatalIf(err, "Unable to initialize admin connection.")

	aliasedURL = filepath.ToSlash(aliasedURL)
	splits := splitStr(aliasedURL, "/", 3)
	bucket := splits[1]

	duinfo, e := adminClient.DataUsageInfo(globalContext)
	fatalIf(probe.NewError(e).Trace(args...), "Unable to get data usage")

	bi, err := s3Client.GetBucketInfo(globalContext)
	fatalIf(err.Trace(args...), "Unable to get bucket properties")

	bu, ok := duinfo.BucketsUsage[bucket]
	if !ok {
		fatalIf(errDummy().Trace(args...), "Unable to get bucket usage info. Bucket usage is not ready yet.")
	}

	printMsg(adminBucketInfoMessage{
		Status:    "success",
		URL:       aliasedURL,
		UsageInfo: bu,
		Props:     bi,
	})

	return nil
}
<commit_msg>Add objects sizes histogram to bucket info (#4236)<commit_after>// Copyright (c) 2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"fmt"
	"path/filepath"
	"sort"
	"strings"

	humanize "github.com/dustin/go-humanize"
	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	"github.com/minio/madmin-go"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/pkg/console"
)

var adminBucketInfoFlags = []cli.Flag{}

type adminBucketInfoMessage struct {
	Status    string                 `json:"status"`
	URL       string                 `json:"url"`
	UsageInfo madmin.BucketUsageInfo `json:"usage"`
	Props     BucketInfo             `json:"props"`
}

type histogramDef struct {
	start, end uint64
	text       string
}

var histogramTagsDesc = map[string]histogramDef{
	"LESS_THAN_1024_B":          {0, 1024, "less than 1024 bytes"},
	"BETWEEN_1024_B_AND_1_MB":   {1024, 1024 * 1024, "between 1024 bytes and 1 MB"},
	"BETWEEN_1_MB_AND_10_MB":    {1024 * 1024, 10 * 1024 * 1024, "between 1 MB and 10 MB"},
	"BETWEEN_10_MB_AND_64_MB":   {10 * 1024 * 1024, 64 * 1024 * 1024, "between 10 MB and 64 MB"},
	"BETWEEN_64_MB_AND_128_MB":  {64 * 1024 * 1024, 128 * 1024 * 1024, "between 64 MB and 128 MB"},
	"BETWEEN_128_MB_AND_512_MB": {128 * 1024 * 1024, 512 * 1024 * 1024, "between 128 MB and 512 MB"},
	"GREATER_THAN_512_MB":       {512 * 1024 * 1024, 0, "greater than 512 MB"},
}

// Return a sorted list of histograms
func sortHistogramTags() (orderedTags []string) {
	orderedTags = make([]string, 0, len(histogramTagsDesc))
	for tag := range histogramTagsDesc {
		orderedTags = append(orderedTags, tag)
	}
	sort.Slice(orderedTags, func(i, j int) bool {
		return histogramTagsDesc[orderedTags[i]].start < histogramTagsDesc[orderedTags[j]].start
	})
	return
}

func countDigits(num uint64) (count uint) {
	for num > 0 {
		num /= 10
		count++
	}
	return
}

func (bi adminBucketInfoMessage) String() string {
	var b strings.Builder

	fmt.Fprintf(&b, console.Colorize("Title", "Usage:\n"))

	fmt.Fprintf(&b, "%16s: %s\n", "Total size", console.Colorize("Count", humanize.IBytes(bi.UsageInfo.Size)))
	fmt.Fprintf(&b, "%16s: %s\n", "Objects count", console.Colorize("Count", humanize.Comma(int64(bi.UsageInfo.ObjectsCount))))
	fmt.Fprintf(&b, "%16s: %s\n", "Versions count", console.Colorize("Count", humanize.Comma(int64(bi.UsageInfo.VersionsCount))))
	fmt.Fprintf(&b, "\n")

	fmt.Fprintf(&b, console.Colorize("Title", "Properties:\n"))
	fmt.Fprintf(&b, prettyPrintBucketMetadata(bi.Props))

	fmt.Fprintf(&b, "\n")
	fmt.Fprintf(&b, console.Colorize("Title", "Object sizes histogram:\n"))

	var maxDigits uint
	for _, val := range bi.UsageInfo.ObjectSizesHistogram {
		if d := countDigits(val); d > maxDigits {
			maxDigits = d
		}
	}

	sortedTags := sortHistogramTags()
	for _, tagName := range sortedTags {
		val, ok := bi.UsageInfo.ObjectSizesHistogram[tagName]
		if ok {
			fmt.Fprintf(&b, "   %*d object(s) %s\n", maxDigits, val, histogramTagsDesc[tagName].text)
		}
	}

	return b.String()
}

func (bi adminBucketInfoMessage) JSON() string {
	jsonMessageBytes, e := json.MarshalIndent(bi, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(jsonMessageBytes)
}

var adminBucketInfoCmd = cli.Command{
	Name:         "info",
	Usage:        "display bucket information",
	Action:       mainAdminBucketInfo,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        append(adminBucketInfoFlags, globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Display the usage data and configuration of a bucket on MinIO.
     {{.Prompt}} {{.HelpName}} myminio/mybucket
`,
}

// checkAdminBucketInfoSyntax - validate all the passed arguments
func checkAdminBucketInfoSyntax(ctx *cli.Context) {
	if len(ctx.Args()) != 1 {
		cli.ShowCommandHelpAndExit(ctx, ctx.Command.Name, 1) // last argument is exit code
	}
}

// mainAdminBucketInfo is the handler for "mc admin bucket info" command.
func mainAdminBucketInfo(ctx *cli.Context) error {
	checkAdminBucketInfoSyntax(ctx)

	console.SetColor("Title", color.New(color.Bold, color.FgBlue))
	console.SetColor("Count", color.New(color.FgGreen))
	console.SetColor("Metadata", color.New(color.FgWhite))
	console.SetColor("Key", color.New(color.FgCyan))
	console.SetColor("Value", color.New(color.FgYellow))
	console.SetColor("Unset", color.New(color.FgRed))
	console.SetColor("Set", color.New(color.FgGreen))

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)

	// Create a new MinIO Admin Client
	adminClient, err := newAdminClient(aliasedURL)
	fatalIf(err, "Unable to initialize admin connection.")

	s3Client, err := newClient(aliasedURL)
	fatalIf(err, "Unable to initialize admin connection.")

	aliasedURL = filepath.ToSlash(aliasedURL)
	splits := splitStr(aliasedURL, "/", 3)
	bucket := splits[1]

	duinfo, e := adminClient.DataUsageInfo(globalContext)
	fatalIf(probe.NewError(e).Trace(args...), "Unable to get data usage")

	bi, err := s3Client.GetBucketInfo(globalContext)
	fatalIf(err.Trace(args...), "Unable to get bucket properties")

	bu, ok := duinfo.BucketsUsage[bucket]
	if !ok {
		fatalIf(errDummy().Trace(args...), "Unable to get bucket usage info. Bucket usage is not ready yet.")
	}

	printMsg(adminBucketInfoMessage{
		Status:    "success",
		URL:       aliasedURL,
		UsageInfo: bu,
		Props:     bi,
	})

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"runtime/pprof"
	"strconv"
	"sync"
	"time"

	"github.com/nabeken/delayd"
	"github.com/nabeken/delayd/testutil"
)

var (
	flagNumMsg    = flag.Int("n", 10, "Specify a number of messages to send")
	flagNumServer = flag.Int("s", 3, "Specify a number of servers")
	flagDuration  = flag.Duration("d", 1*time.Second, "Specify a delay. Default is 1s.")
	flagConfig    = flag.String("c", "/etc/delayd.toml", "Specify a config. Default is /etc/delayd.toml")
	flagProfile   = flag.String("p", "", "Specify a output file for cpu profiling.")
	flagNoServer  = flag.Bool("no-server", false, "Set true if you need only client")
)

const raftHost = "127.0.0.1"

// mps returns message per seconds.
func mps(n int, d time.Duration) float64 {
	return float64(n) / d.Seconds()
}

func generateMessages(n int, d time.Duration) []testutil.Message {
	msgs := make([]testutil.Message, n)
	for i := 0; i < n; i++ {
		msgs[i] = testutil.Message{
			Value: strconv.FormatInt(int64(i+1), 10),
			Delay: int64(d / time.Millisecond),
		}
	}
	return msgs
}

func main() {
	flag.Parse()

	config, err := delayd.LoadConfig(*flagConfig)
	if err != nil {
		log.Fatal(err)
	}

	// Use stdout instead of file
	config.LogDir = ""

	if *flagProfile != "" {
		f, err := os.Create(*flagProfile)
		if err != nil {
			delayd.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	client, err := testutil.AMQPClientFunc(config, ioutil.Discard)
	if err != nil {
		log.Fatal(err)
	}

	if !*flagNoServer {
		delayd.Info("launching servers and waiting for messages")
		raftServers, err := testutil.RaftServers(
			*flagNumServer,
			raftHost,
			testutil.AMQPServerFunc,
		)(config)
		if err != nil {
			log.Fatal(err)
		}

		for _, s := range raftServers {
			go s.Run()
			defer s.Stop()
		}
	} else {
		delayd.Info("-no-server is specified. Waiting for messages")
	}

	// Send messages to delayd exchange
	msgs := generateMessages(*flagNumMsg, *flagDuration)
	n := len(msgs)

	start := time.Now()
	if err := client.SendMessages(msgs); err != nil {
		log.Println(err)
		return
	}
	sent := time.Now()

	durationSent := sent.Sub(start)
	delayd.Infof("sent %d messages for %s, %f msg/s", n, durationSent, mps(n, durationSent))

	var wg sync.WaitGroup
	wg.Add(n)
	acked := 0
	go func() {
		for _ = range client.RecvLoop() {
			acked++
			if acked > n {
				continue
			}
			wg.Done()
		}
	}()

	// Wait for messages to be processed
	wg.Wait()
	done := time.Now()

	durationProcessed := done.Sub(sent)
	durationAll := done.Sub(start)

	delayd.Infof("processed %d messages for %s, %f msg/s", n, durationProcessed, mps(n, durationProcessed))
	delayd.Infof("total: processed %d messages for %s, %f msg/s", n, durationAll, mps(n, durationAll))

	// shutdown consumer
	client.Close()

	if acked > n {
		delayd.Infof("%d messages are duplicated", acked-n)
	}
}
<commit_msg>Add SQS support in bench<commit_after>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"runtime/pprof"
	"strconv"
	"sync"
	"time"

	"github.com/nabeken/delayd"
	"github.com/nabeken/delayd/testutil"
)

var (
	flagConfig    = flag.String("c", "/etc/delayd.toml", "Specify a config. Default is /etc/delayd.toml")
	flagProfile   = flag.String("p", "", "Specify a output file for cpu profiling.")
	flagBroker    = flag.String("b", "amqp", "Specify a broker. Default is amqp.")
	flagNumMsg    = flag.Int("n", 10, "Specify a number of messages to send")
	flagNumServer = flag.Int("s", 3, "Specify a number of servers")
	flagDuration  = flag.Duration("d", 1*time.Second, "Specify a delay. Default is 1s.")
	flagNoServer  = flag.Bool("no-server", false, "Set true if you need only client")
)

const raftHost = "127.0.0.1"

// mps returns message per seconds.
func mps(n int, d time.Duration) float64 {
	return float64(n) / d.Seconds()
}

func generateMessages(n int, d time.Duration) []testutil.Message {
	msgs := make([]testutil.Message, n)
	for i := 0; i < n; i++ {
		msgs[i] = testutil.Message{
			Value: strconv.FormatInt(int64(i+1), 10),
			Delay: int64(d / time.Millisecond),
		}
	}
	return msgs
}

func getBroker(broker string, config delayd.Config) (testutil.Client, testutil.ServerFunc) {
	switch broker {
	case "amqp":
		client, err := testutil.AMQPClientFunc(config, ioutil.Discard)
		if err != nil {
			log.Fatal(err)
		}
		return client, testutil.AMQPServerFunc
	case "sqs":
		client, err := testutil.SQSClientFunc(config, ioutil.Discard)
		if err != nil {
			log.Fatal(err)
		}
		return client, testutil.SQSServerFunc
	}
	panic("unknown broker")
}

func main() {
	flag.Parse()

	config, err := delayd.LoadConfig(*flagConfig)
	if err != nil {
		log.Fatal(err)
	}

	// Use stdout instead of file
	config.LogDir = ""

	if *flagProfile != "" {
		f, err := os.Create(*flagProfile)
		if err != nil {
			delayd.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	client, sf := getBroker(*flagBroker, config)
	if !*flagNoServer {
		delayd.Info("launching servers and waiting for messages")
		raftServers, err := testutil.RaftServers(
			*flagNumServer,
			raftHost,
			sf,
		)(config)
		if err != nil {
			log.Fatal(err)
		}

		for _, s := range raftServers {
			go s.Run()
			defer s.Stop()
		}
	} else {
		delayd.Info("-no-server is specified. Waiting for messages")
	}

	// Send messages to delayd exchange
	msgs := generateMessages(*flagNumMsg, *flagDuration)
	n := len(msgs)

	start := time.Now()
	if err := client.SendMessages(msgs); err != nil {
		log.Println(err)
		return
	}
	sent := time.Now()

	durationSent := sent.Sub(start)
	delayd.Infof("sent %d messages for %s, %f msg/s", n, durationSent, mps(n, durationSent))

	var wg sync.WaitGroup
	wg.Add(n)
	acked := 0
	go func() {
		for _ = range client.RecvLoop() {
			acked++
			if acked > n {
				continue
			}
			wg.Done()
		}
	}()

	// Wait for messages to be processed
	wg.Wait()
	done := time.Now()

	durationProcessed := done.Sub(sent)
	durationAll := done.Sub(start)

	delayd.Infof("processed %d messages for %s, %f msg/s", n, durationProcessed, mps(n, durationProcessed))
	delayd.Infof("total: processed %d messages for %s, %f msg/s", n, durationAll, mps(n, durationAll))

	// shutdown consumer
	client.Close()

	if acked > n {
		delayd.Infof("%d messages are duplicated", acked-n)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rsa"
	"fmt"
	"time"

	"github.com/datawire/ambassador/pkg/k8s"
	"github.com/dgrijalva/jwt-go"
	"github.com/pkg/browser"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	k8sTypesMetaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	k8sClientCoreV1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/rest"
)

const SecretName = "ambassador-internal"

type LoginClaimsV1 struct {
	LoginTokenVersion  string `json:"login_token_version"`
	jwt.StandardClaims `json:",inline"`
}

func aesLogin(cmd *cobra.Command, args []string) error {
	fmt.Println("Connecting to the Ambassador Edge Stack admin UI in this cluster...")

	// Grab options
	context, _ := cmd.Flags().GetString("context")
	namespace, _ := cmd.Flags().GetString("namespace")
	justShowURL, _ := cmd.Flags().GetBool("url")
	showToken, _ := cmd.Flags().GetBool("token")

	// Prepare to talk to the cluster
	kubeinfo := k8s.NewKubeInfo("", context, namespace) // Default namespace here
	restconfig, err := kubeinfo.GetRestConfig()
	if err != nil {
		return errors.Wrap(err, "Failed to connect to cluster (rest)")
	}

	// Obtain signing key
	// -> kubectl -n $namespace get secret $SecretName -o json
	privateKey, err := getSigningKey(restconfig, namespace)
	if err != nil {
		fmt.Println()
		fmt.Println("Failed to obtain expected information from the cluster.")
		fmt.Println("Is kubectl configured to connect to the correct cluster?")
		fmt.Printf("Is %s the namespace where Ambassador is installed?\n\n", namespace)
		fmt.Printf("    kubectl -n %s get svc,deploy\n\n", namespace)
		return err
	}

	// Figure out the correct hostname
	hostname := args[0]
	// FIXME: validate that hostname by querying
	// https://{{hostname}}/edge_stack/admin/api/ambassador_cluster_id and
	// verifying that it returns the same UUID via direct access and via
	// port-forward/teleproxy. This avoids leaking login credentials to the
	// operator of a different website.

	// Construct claims
	now := time.Now()
	duration := 30 * time.Minute
	claims := &LoginClaimsV1{
		"v1",
		jwt.StandardClaims{
			IssuedAt:  now.Unix(),
			NotBefore: now.Unix(),
			ExpiresAt: (now.Add(duration)).Unix(),
		},
	}

	// Generate JWT
	token := jwt.NewWithClaims(jwt.GetSigningMethod("PS512"), claims)
	tokenString, err := token.SignedString(privateKey)
	if err != nil {
		return errors.Wrap(err, "Unexpected error generating JWT")
	}

	// Output
	url := fmt.Sprintf("https://%s/edge_stack/admin/#%s", hostname, tokenString)

	if !justShowURL {
		err = browser.OpenURL(url)
		if err != nil {
			fmt.Println("Unexpected error while trying to open your browser.")
			err = errors.Wrap(err, "browse")
		}
	}

	if justShowURL || err != nil {
		fmt.Println("Visit the following URL to access the Ambassador Edge Stack admin UI:")
	} else {
		fmt.Println("Ambassador Edge Stack admin UI has been opened in your browser.")
	}
	fmt.Println("    ", url)

	if showToken {
		fmt.Println()
		fmt.Println("The login token is")
		fmt.Println("    ", tokenString)
	}

	return err
}

func getSigningKey(restconfig *rest.Config, namespace string) (*rsa.PrivateKey, error) {
	coreClient, err := k8sClientCoreV1.NewForConfig(restconfig)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to connect to cluster (core)")
	}
	secretInterface := coreClient.Secrets(namespace)
	secret, err := secretInterface.Get(SecretName, k8sTypesMetaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	// Parse out the private key from the secret
	privatePEM, ok := secret.Data["rsa.key"]
	if !ok {
		return nil, errors.Errorf("secret name=%q namespace=%q exists but does not contain an %q %s field",
			SecretName, namespace, "rsa.key", "private-key")
	}
	return jwt.ParseRSAPrivateKeyFromPEM(privatePEM)
}
<commit_msg>Refactor login to allow calling from install<commit_after>package main

import (
	"crypto/rsa"
	"fmt"
	"time"

	"github.com/datawire/ambassador/pkg/k8s"
	"github.com/dgrijalva/jwt-go"
	"github.com/pkg/browser"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	k8sTypesMetaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	k8sClientCoreV1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/rest"
)

const SecretName = "ambassador-internal"

type LoginClaimsV1 struct {
	LoginTokenVersion  string `json:"login_token_version"`
	jwt.StandardClaims `json:",inline"`
}

func aesLogin(cmd *cobra.Command, args []string) error {
	fmt.Println("Connecting to the Ambassador Edge Stack admin UI in this cluster...")

	// Grab options
	context, _ := cmd.Flags().GetString("context")
	namespace, _ := cmd.Flags().GetString("namespace")
	justShowURL, _ := cmd.Flags().GetBool("url")
	showToken, _ := cmd.Flags().GetBool("token")

	// Figure out the correct hostname
	hostname := args[0]

	// Prepare to talk to the cluster
	kubeinfo := k8s.NewKubeInfo("", context, namespace) // Default namespace is "ambassador"

	return do_login(kubeinfo, context, namespace, hostname, justShowURL, showToken)
}

func do_login(kubeinfo *k8s.KubeInfo, context, namespace, hostname string, justShowURL, showToken bool) error {
	restconfig, err := kubeinfo.GetRestConfig()
	if err != nil {
		return errors.Wrap(err, "Failed to connect to cluster (rest)")
	}

	// Obtain signing key
	// -> kubectl -n $namespace get secret $SecretName -o json
	privateKey, err := getSigningKey(restconfig, namespace)
	if err != nil {
		fmt.Println()
		fmt.Println("Failed to obtain expected information from the cluster.")
		fmt.Println("Is kubectl configured to connect to the correct cluster?")
		fmt.Printf("Is %s the namespace where Ambassador is installed?\n\n", namespace)
		fmt.Printf("    kubectl -n %s get svc,deploy\n\n", namespace)
		return err
	}

	// FIXME: validate that hostname by querying
	// https://{{hostname}}/edge_stack/admin/api/ambassador_cluster_id and
	// verifying that it returns the same UUID via direct access and via
	// port-forward/teleproxy. This avoids leaking login credentials to the
	// operator of a different website.

	// Construct claims
	now := time.Now()
	duration := 30 * time.Minute
	claims := &LoginClaimsV1{
		"v1",
		jwt.StandardClaims{
			IssuedAt:  now.Unix(),
			NotBefore: now.Unix(),
			ExpiresAt: (now.Add(duration)).Unix(),
		},
	}

	// Generate JWT
	token := jwt.NewWithClaims(jwt.GetSigningMethod("PS512"), claims)
	tokenString, err := token.SignedString(privateKey)
	if err != nil {
		return errors.Wrap(err, "Unexpected error generating JWT")
	}

	// Output
	url := fmt.Sprintf("https://%s/edge_stack/admin/#%s", hostname, tokenString)

	if !justShowURL {
		err = browser.OpenURL(url)
		if err != nil {
			fmt.Println("Unexpected error while trying to open your browser.")
			err = errors.Wrap(err, "browse")
		}
	}

	if justShowURL || err != nil {
		fmt.Println("Visit the following URL to access the Ambassador Edge Stack admin UI:")
	} else {
		fmt.Println("Ambassador Edge Stack admin UI has been opened in your browser.")
	}
	fmt.Println("    ", url)

	if showToken {
		fmt.Println()
		fmt.Println("The login token is")
		fmt.Println("    ", tokenString)
	}

	return err
}

// getSigningKey retrieves the designated secret from the cluster and returns
// the private key extracted from the secret data
func getSigningKey(restconfig *rest.Config, namespace string) (*rsa.PrivateKey, error) {
	coreClient, err := k8sClientCoreV1.NewForConfig(restconfig)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to connect to cluster (core)")
	}
	secretInterface := coreClient.Secrets(namespace)
	secret, err := secretInterface.Get(SecretName, k8sTypesMetaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	// Parse out the private key from the secret
	privatePEM, ok := secret.Data["rsa.key"]
	if !ok {
		return nil, errors.Errorf("secret name=%q namespace=%q exists but does not contain an %q %s field",
			SecretName, namespace, "rsa.key", "private-key")
	}
	return jwt.ParseRSAPrivateKeyFromPEM(privatePEM)
}
<|endoftext|>


<commit_before>// epoxy_client is a command line utility for requesting nextboot configurations
// from the ePoxy server and executing them.
//
// epoxy_client should be embedded in initram images served by ePoxy. Once the
// network is initialized, epoxy_client can complete actions for the current
// boot stage. i.e. download config from epoxy, download kernel for stage3,
// kexec kernel.
package main

import (
	"flag"
	"io/ioutil"
	"log"
	"net/url"
	"time"

	"github.com/m-lab/epoxy/nextboot"
	"github.com/m-lab/go/rtx"
)

const timeout = 6 * time.Hour

var (
	flagCmdline = flag.String("cmdline", "/proc/cmdline",
		"Read kernel cmdline parameters from the contents of this file.")
	flagAction = flag.String("action", "epoxy.stage2",
		"Execute the config loaded from the URL in this kernel parameter.")
	flagAddKargs = flag.Bool("add-kargs", false,
		"Combine the local kargs with those returned from the action url. "+
			"Existing kargs are never replaced. Only useful for stage1.")
	flagReport = flag.String("report", "epoxy.report",
		"Report success or errors with the URL in this kernel parameter.")
	flagDryrun = flag.Bool("dryrun", false,
		"Request all configs but do not run commands. May change state in the ePoxy server.")
	flagNoRetry = flag.Bool("no-retry", false, "Do not retry in case of failure")
)

func main() {
	var result string
	var runErr error

	flag.Parse()
	c := &nextboot.Config{}

	b, err := ioutil.ReadFile(*flagCmdline)
	if err != nil {
		log.Fatal(err)
	}
	// Read and parse parameters from *flagCmdline.
	c.ParseCmdline(string(b))

	deadline := time.Now().Add(timeout)

	for {
		// Run the config loaded from the action URL.
		runErr := c.Run(*flagAction, *flagAddKargs, *flagDryrun)
		if runErr != nil {
			// Define a successful result.
			result = "error: " + runErr.Error()
		} else {
			result = "success"
		}
		log.Println("Result:", result)

		// Report a message to the ePoxy server after running.
		values := url.Values{}
		// TODO: report additional host information.
		// TODO: log the evaluate state of c.V1 -- helpful especially for errors.
		values.Set("message", result)

		err = c.Report(*flagReport, values, *flagDryrun)
		if err != nil {
			log.Print(err)
		}

		// Stop the retry loop if the -no-retry flag has been provided,
		// the last execution succeeded or enough time has passed.
		if *flagNoRetry || runErr == nil || time.Now().After(deadline) {
			break
		}

		log.Println("Waiting 1 minute before retrying...")
		time.Sleep(1 * time.Minute)
	}

	// If the run step failed, reboot the machine
	if runErr != nil {
		reboot()
	}
}

func reboot() {
	err := ioutil.WriteFile("/proc/sys/kernel/sysrq", []byte{'1'}, 0644)
	rtx.Must(err, "Error while writing sysrq")

	err = ioutil.WriteFile("/proc/sysrq-trigger", []byte{'b'}, 0644)
	rtx.Must(err, "Error while sending sysrq")
}
<commit_msg>Change -no-retry to -retry and add comment.<commit_after>// epoxy_client is a command line utility for requesting nextboot configurations
// from the ePoxy server and executing them.
//
// epoxy_client should be embedded in initram images served by ePoxy. Once the
// network is initialized, epoxy_client can complete actions for the current
// boot stage. i.e. download config from epoxy, download kernel for stage3,
// kexec kernel.
package main

import (
	"flag"
	"io/ioutil"
	"log"
	"net/url"
	"time"

	"github.com/m-lab/epoxy/nextboot"
	"github.com/m-lab/go/rtx"
)

const timeout = 6 * time.Hour

var (
	flagCmdline = flag.String("cmdline", "/proc/cmdline",
		"Read kernel cmdline parameters from the contents of this file.")
	flagAction = flag.String("action", "epoxy.stage2",
		"Execute the config loaded from the URL in this kernel parameter.")
	flagAddKargs = flag.Bool("add-kargs", false,
		"Combine the local kargs with those returned from the action url. "+
			"Existing kargs are never replaced. Only useful for stage1.")
	flagReport = flag.String("report", "epoxy.report",
		"Report success or errors with the URL in this kernel parameter.")
	flagDryrun = flag.Bool("dryrun", false,
		"Request all configs but do not run commands. May change state in the ePoxy server.")
	flagRetry = flag.Bool("retry", true, "Retry in case of failure.")
)

func main() {
	var result string
	var runErr error

	flag.Parse()
	c := &nextboot.Config{}

	b, err := ioutil.ReadFile(*flagCmdline)
	if err != nil {
		log.Fatal(err)
	}
	// Read and parse parameters from *flagCmdline.
	c.ParseCmdline(string(b))

	deadline := time.Now().Add(timeout)

	for {
		// Run the config loaded from the action URL.
		runErr := c.Run(*flagAction, *flagAddKargs, *flagDryrun)
		if runErr != nil {
			// Define a successful result.
			result = "error: " + runErr.Error()
		} else {
			result = "success"
		}
		log.Println("Result:", result)

		// Report a message to the ePoxy server after running.
		values := url.Values{}
		// TODO: report additional host information.
		// TODO: log the evaluate state of c.V1 -- helpful especially for errors.
		values.Set("message", result)

		err = c.Report(*flagReport, values, *flagDryrun)
		if err != nil {
			log.Print(err)
		}

		// Stop the retry loop if the -no-retry flag has been provided,
		// the last execution succeeded or enough time has passed.
		if !*flagRetry || runErr == nil || time.Now().After(deadline) {
			break
		}

		log.Println("Waiting 1 minute before retrying...")
		time.Sleep(1 * time.Minute)
	}

	// If the run step failed, reboot the machine
	if runErr != nil {
		reboot()
	}
}

func reboot() {
	err := ioutil.WriteFile("/proc/sys/kernel/sysrq", []byte{'1'}, 0644)
	rtx.Must(err, "Error while writing sysrq")

	// 'b' will immediately reboot the system without syncing or unmounting
	// your disks.
	err = ioutil.WriteFile("/proc/sysrq-trigger", []byte{'b'}, 0644)
	rtx.Must(err, "Error while sending sysrq")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/api"
)

var (
	addr  string
	n     int
	appid string
	group string
	topic string
	step  int
)

func init() {
	flag.StringVar(&addr, "addr", "http://localhost:9192", "sub kateway addr")
	flag.StringVar(&group, "g", "mygroup1", "consumer group name")
	flag.StringVar(&appid, "appid", "app1", "consume whose topic")
	flag.IntVar(&step, "step", 1, "display progress step")
	flag.StringVar(&topic, "t", "foobar", "topic to sub")
	flag.IntVar(&n, "n", 1000000, "run sub how many times")
	flag.Parse()
}

func main() {
	c := api.NewClient("app2", nil)
	c.Connect(addr)
	i := 0
	t0 := time.Now()
	err := c.Subscribe(appid, topic, "v1", group, func(statusCode int, msg []byte) error {
		i++
		if n > 0 && i >= n {
			return api.ErrSubStop
		}

		if i%step == 0 {
			log.Println(string(msg))
		}

		return nil
	})
	if err != nil {
		fmt.Println(err)
	}

	elapsed := time.Since(t0)
	fmt.Printf("%d msgs in %s, tps: %.2f\n", n, elapsed, float64(n)/elapsed.Seconds())
}
<commit_msg>sub print http status code<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/api"
)

var (
	addr  string
	n     int
	appid string
	group string
	topic string
	step  int
)

func init() {
	flag.StringVar(&addr, "addr", "http://localhost:9192", "sub kateway addr")
	flag.StringVar(&group, "g", "mygroup1", "consumer group name")
	flag.StringVar(&appid, "appid", "app1", "consume whose topic")
	flag.IntVar(&step, "step", 1, "display progress step")
	flag.StringVar(&topic, "t", "foobar", "topic to sub")
	flag.IntVar(&n, "n", 1000000, "run sub how many times")
	flag.Parse()
}

func main() {
	c := api.NewClient("app2", nil)
	c.Connect(addr)
	i := 0
	t0 := time.Now()
	err := c.Subscribe(appid, topic, "v1", group, func(statusCode int, msg []byte) error {
		i++
		if n > 0 && i >= n {
			return api.ErrSubStop
		}

		if i%step == 0 {
			log.Println(statusCode, string(msg))
		}

		return nil
	})
	if err != nil {
		fmt.Println(err)
	}

	elapsed := time.Since(t0)
	fmt.Printf("%d msgs in %s, tps: %.2f\n", n, elapsed, float64(n)/elapsed.Seconds())
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os"

	"github.com/GoogleCloudPlatform/skaffold/cmd/skaffold/app"
)

func main() {
	if err := app.Run(); err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		os.Exit(1)
	}
	os.Exit(0)
}
<commit_msg>Remove duplicate error<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"os"

	"github.com/GoogleCloudPlatform/skaffold/cmd/skaffold/app"
)

func main() {
	if err := app.Run(); err != nil {
		os.Exit(1)
	}
	os.Exit(0)
}
<|endoftext|>


<commit_before>package sloth

import (
  "fmt"
  "bytes"
  "net/http"
  "net/url"
  "database/sql"
  // "crypto/hmac"
  _ "github.com/go-sql-driver/mysql"
)

// Hooks

var _ RestfulHook = (*RestHook)(nil)

type RestfulHook interface {
  Mesg(interface{}) (*http.Response, error)
  Kill() (sql.Result, error)
}

type RestHook struct {
  subscriberUrl    string
  subscriberMethod string
}

func (hook *RestHook) Url() string {
  return hook.subscriberUrl
}

func (hook *RestHook) Method() string {
  return hook.subscriberMethod
}

func (hook *RestHook) Mesg(data interface{}) (*http.Response, error) {
  hookCaller   := &http.Client{}
  dataBytes, _ := AsBytes(data)

  req, err := http.NewRequest(hook.subscriberMethod, hook.subscriberUrl, bytes.NewBuffer(dataBytes))

  if err != nil {
    panic(err)
  }

  req.Header.Add("X-Sloth-Hook-Id",        `W/"something"`)
  req.Header.Add("X-Sloth-Hook-Signature", `W/"abc123"`) // TODO - HMAC

  resp, err := hookCaller.Do(req)

  defer resp.Body.Close()

  return resp, err
}

func (hook *RestHook) Kill() (sql.Result, error) {
  return new(HookRepo).Db().Exec("delete from hooks where subscriber_url = ? and subscriber_method = ?", hook.subscriberUrl, hook.subscriberMethod)
}

// Hook resource

var _ HookableResource = (*HookResource)(nil)

type HookableResource interface {
  RestfulResource

  Hooks() []RestHook
  Subscribe(subUrl string, subMethod string)
  Broadcast(data interface{})
}

type HookResource struct {
  UrlSlug string
  RestResource
}

func (resource *HookResource) Hooks() []RestHook {
  return nil // TODO - call sql, get all hooks for resource
}

func (resource *HookResource) Slug() string {
  return resource.UrlSlug
}

func (resource *HookResource) Put(values url.Values) (int, interface{}) {
  resource.Subscribe(values["subscriber_url"][0], values["subscriber_method"][0])
  return 200, "FIXME"
}

func (resource *HookResource) Subscribe(subUrl string, subMethod string) {
  newHook := RestHook {
    subscriberUrl    : subUrl,
    subscriberMethod : subMethod,
  }

  new(HookRepo).Add(&newHook)

  fmt.Println("Successful subscription!", subUrl, subMethod)
}

func (resource *HookResource) Broadcast(data interface{}) {
  for _, hook := range resource.Hooks() {
    go func() { // WARN - this can get a bit crazy if we have thousands of subscribers. need to make this scale.
      hook.Mesg(data)
    }()
  }
}

// Hook repository

type HookRepo struct { }

func (repo *HookRepo) Db() *sql.DB {
  db, err := sql.Open("mysql", "user:password@/hooks") // FIXME - integrate with config

  if err != nil {
    fmt.Println("improve this error")
  }

  return db
}

func (repo *HookRepo) All() (*sql.Rows, error) {
  return repo.Db().Query("select id, subscriber_url, subscriber_method from hooks")
}

func (repo *HookRepo) Add(hook *RestHook) (sql.Result, error) {
  return repo.Db().Exec("insert into hooks (subscriber_url, subscriber_method) values (?, ?)", hook.subscriberUrl, hook.subscriberMethod)
}

func (repo *HookRepo) Delete(hook *RestHook) (sql.Result, error) {
  return repo.Db().Exec("delete from hooks where subscriber_url = ? and subscriber_method = ?", hook.subscriberUrl, hook.subscriberMethod)
}

func (repo *HookRepo) Close() {
  repo.Db().Close()
}
<commit_msg>hook resource, fetch all hook subscriptions<commit_after>package sloth

import (
  "fmt"
  "bytes"
  "net/http"
  "net/url"
  "database/sql"
  // "crypto/hmac"
  _ "github.com/go-sql-driver/mysql"
)

// Hooks

var _ RestfulHook = (*RestHook)(nil)

type RestfulHook interface {
  Mesg(interface{}) (*http.Response, error)
  Kill() (sql.Result, error)
}

type RestHook struct {
  id int
  resourceSlug     string
  subscriberUrl    string
  subscriberMethod string
}

func (hook *RestHook) Url() string {
  return hook.subscriberUrl
}

func (hook *RestHook) Method() string {
  return hook.subscriberMethod
}

func (hook *RestHook) Mesg(data interface{}) (*http.Response, error) {
  hookCaller   := &http.Client{}
  dataBytes, _ := AsBytes(data)

  req, err := http.NewRequest(hook.subscriberMethod, hook.subscriberUrl, bytes.NewBuffer(dataBytes))

  if err != nil {
    panic(err)
  }

  req.Header.Add("X-Sloth-Hook-Id",        `W/"something"`)
  req.Header.Add("X-Sloth-Hook-Signature", `W/"abc123"`) // TODO - HMAC

  resp, err := hookCaller.Do(req)

  defer resp.Body.Close()

  return resp, err
}

func (hook *RestHook) Kill() (sql.Result, error) {
  return new(HookRepo).Db().Exec("delete from hooks where subscriber_url = ? and subscriber_method = ?", hook.subscriberUrl, hook.subscriberMethod)
}

// Hook resource

var _ HookableResource = (*HookResource)(nil)

type HookableResource interface {
  RestfulResource

  Hooks() []RestHook
  Subscribe(subUrl string, subMethod string)
  Broadcast(data interface{})
}

type HookResource struct {
  UrlSlug string
  RestResource
}

func (resource *HookResource) Hooks() []RestHook {
  var parsedHooks []RestHook

  storedHooks, err := new(HookRepo).ForResource(resource)

  defer storedHooks.Close()

  for storedHooks.Next() {
    var id  int
    var subscriberUrl    string
    var subscriberMethod string

    err := storedHooks.Scan(&id, &subscriberUrl, &subscriberMethod)

    if err != nil {
      panic("Failed to parse RestHook from repository")
    }

    newHook := RestHook{subscriberUrl: subscriberUrl, subscriberMethod: subscriberMethod}

    parsedHooks = append(parsedHooks, newHook)
  }

  err = storedHooks.Err()

  if err != nil {
    panic("Failed to acquire RestHooks from repository")
  }

  return parsedHooks
}

func (resource *HookResource) Slug() string {
  return resource.UrlSlug
}

func (resource *HookResource) Put(values url.Values) (int, interface{}) {
  resource.Subscribe(values["subscriber_url"][0], values["subscriber_method"][0])
  return 200, "FIXME"
}

func (resource *HookResource) Subscribe(subUrl string, subMethod string) {
  newHook := RestHook {
    subscriberUrl    : subUrl,
    subscriberMethod : subMethod,
  }

  new(HookRepo).Add(&newHook)

  fmt.Println("Successful subscription!", subUrl, subMethod)
}

func (resource *HookResource) Broadcast(data interface{}) {
  for _, hook := range resource.Hooks() {
    go func() { // WARN - this can get a bit crazy if we have thousands of subscribers. need to make this scale properly
      hook.Mesg(data)
    }()
  }
}

// Hook repository

type HookRepo struct { }

func (repo *HookRepo) Db() *sql.DB {
  db, err := sql.Open("mysql", "user:password@/hooks") // FIXME - integrate with config

  if err != nil {
    fmt.Println("improve this error")
  }

  return db
}

func (repo *HookRepo) All() (*sql.Rows, error) {
  return repo.Db().Query("select id, subscriber_url, subscriber_method from hooks")
}

func (repo *HookRepo) ForResource(resource *HookResource) (*sql.Rows, error) {
  return repo.Db().Query("select id, subscriber_url, subscriber_method from hooks where resource_slug = ?", resource.Slug())
}

func (repo *HookRepo) Add(hook *RestHook) (sql.Result, error) {
  return repo.Db().Exec("insert into hooks (resource_slug, subscriber_url, subscriber_method) values (?, ?)", hook.resourceSlug, hook.subscriberUrl, hook.subscriberMethod)
}

func (repo *HookRepo) Delete(hook *RestHook) (sql.Result, error) {
  return repo.Db().Exec("delete from hooks where subscriber_url = ? and subscriber_method = ?", hook.resourceSlug, hook.subscriberUrl, hook.subscriberMethod)
}

func (repo *HookRepo) Close() {
  repo.Db().Close()
}
<|endoftext|>


<commit_before>package buffer

import (
	"container/list"
	"fmt"
	"sync"
	"time"
)

type StringItem string

func (s StringItem) Size() int64 {
	return int64(len(s))
}

type BytesItem []byte

func (s BytesItem) Size() int64 {
	return int64(len(s))
}

type Sizer interface {
	Size() int64
}

type Chunk interface {
	Push(Sizer)
}

type Handler interface {
	Write([]Sizer) (int, error)
}

type ChunkHandler func(c Chunk) error

type MemoryChunk struct {
	Size  int64
	Items []Sizer
}

func (m *MemoryChunk) Push(s Sizer) {
	m.Size += s.Size()
	m.Items = append(m.Items, s)
}

type Memory struct {
	chunks        *list.List
	maxChunkSize  int64
	maxQueueSize  int64
	flushInterval time.Duration
	handler       Handler
	eventCh       chan bool
	closed        bool
	m             sync.Mutex
}

func NewMemory(opts *Options, h Handler) *Memory {
	m := &Memory{
		chunks:        list.New(),
		maxChunkSize:  int64(opts.MaxChunkSize),
		maxQueueSize:  int64(opts.MaxQueueSize),
		flushInterval: time.Duration(opts.FlushInterval),
		handler:       h,
		eventCh:       make(chan bool),
	}
	go m.pop()
	return m
}

func (m *Memory) Push(s Sizer) error {
	n := s.Size()
	if n > m.maxChunkSize {
		return fmt.Errorf("Too large item: %d, max: %d", n, m.maxChunkSize)
	}
	m.m.Lock()
	defer m.m.Unlock()

	e := m.chunks.Front()
	if e == nil || e.Value.(*MemoryChunk).Size+n > m.maxChunkSize {
		if e != nil {
			m.eventCh <- true
		}

		e = m.chunks.PushFront(&MemoryChunk{})
		if int64(m.chunks.Len()) > m.maxQueueSize {
			m.chunks.Remove(m.chunks.Back())
		}
	}

	e.Value.(*MemoryChunk).Push(s)
	return nil
}

func (m *Memory) Close() {
	m.closed = true
	close(m.eventCh)
	m.m.Lock()
	defer m.m.Unlock()
	for e := m.chunks.Front(); e != nil; e = e.Next() {
		m.handler.Write(e.Value.(*MemoryChunk).Items)
	}
	m.chunks.Init()
}

func (m *Memory) pop() {
	var tick <-chan time.Time
	tick = time.Tick(m.flushInterval)
	for {
		select {
		case <-m.eventCh:
			if m.closed {
				return
			}
		case <-tick:
		}

		m.m.Lock()
		e := m.chunks.Back()
		if e == nil {
			m.m.Unlock()
			continue
		}

		chunk := e.Value.(*MemoryChunk)
		n, err := m.handler.Write(chunk.Items)
		if err != nil {
			copy(chunk.Items, chunk.Items[n:])
		} else {
			m.chunks.Remove(e)
		}
		m.m.Unlock()
	}
}
<commit_msg>Make flush_interval of buffer options can be 0<commit_after>package buffer

import (
	"container/list"
	"fmt"
	"sync"
	"time"
)

type StringItem string

func (s StringItem) Size() int64 {
	return int64(len(s))
}

type BytesItem []byte

func (s BytesItem) Size() int64 {
	return int64(len(s))
}

type Sizer interface {
	Size() int64
}

type Chunk interface {
	Push(Sizer)
}

type Handler interface {
	Write([]Sizer) (int, error)
}

type ChunkHandler func(c Chunk) error

type MemoryChunk struct {
	Size  int64
	Items []Sizer
}

func (m *MemoryChunk) Push(s Sizer) {
	m.Size += s.Size()
	m.Items = append(m.Items, s)
}

type Memory struct {
	chunks        *list.List
	maxChunkSize  int64
	maxQueueSize  int64
	flushInterval time.Duration
	handler       Handler
	eventCh       chan bool
	closed        bool
	m             sync.Mutex
}

func NewMemory(opts *Options, h Handler) *Memory {
	m := &Memory{
		chunks:        list.New(),
		maxChunkSize:  int64(opts.MaxChunkSize),
		maxQueueSize:  int64(opts.MaxQueueSize),
		flushInterval: time.Duration(opts.FlushInterval),
		handler:       h,
		eventCh:       make(chan bool),
	}
	go m.pop()
	return m
}

func (m *Memory) Push(s Sizer) error {
	n := s.Size()
	if n > m.maxChunkSize {
		return fmt.Errorf("Too large item: %d, max: %d", n, m.maxChunkSize)
	}
	m.m.Lock()
	defer m.m.Unlock()

	e := m.chunks.Front()
	if e == nil || e.Value.(*MemoryChunk).Size+n > m.maxChunkSize {
		if e != nil {
			m.notify()
		}

		e = m.chunks.PushFront(&MemoryChunk{})
		if int64(m.chunks.Len()) > m.maxQueueSize {
			m.chunks.Remove(m.chunks.Back())
		}
	}

	e.Value.(*MemoryChunk).Push(s)
	if m.flushInterval == 0 {
		m.notify()
	}
	return nil
}

func (m *Memory) Close() {
	m.closed = true
	close(m.eventCh)
	m.m.Lock()
	defer m.m.Unlock()
	for e := m.chunks.Front(); e != nil; e = e.Next() {
		m.handler.Write(e.Value.(*MemoryChunk).Items)
	}
	m.chunks.Init()
}

func (m *Memory) notify() {
	select {
	case m.eventCh <- true:
	default:
	}
}

func (m *Memory) pop() {
	var tick <-chan time.Time
	if m.flushInterval > 0 {
		tick = time.Tick(m.flushInterval)
	}
	for {
		select {
		case <-m.eventCh:
			if m.closed {
				return
			}
		case <-tick:
		}

		m.m.Lock()
		e := m.chunks.Back()
		if e == nil {
			m.m.Unlock()
			continue
		}

		chunk := e.Value.(*MemoryChunk)
		n, err := m.handler.Write(chunk.Items)
		if err != nil {
			copy(chunk.Items, chunk.Items[n:])
		} else {
			m.chunks.Remove(e)
		}
		m.m.Unlock()
	}
}
<|endoftext|>


<commit_before>package promgrpc

import (
	"context"
	"fmt"
	"strconv"

	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/grpc/stats"
)

// StatsHandlerCollector is a simple wrapper for stats Handler and prometheus Collector interfaces.
type StatsHandlerCollector interface {
	stats.Handler
	prometheus.Collector
}

var _ StatsHandlerCollector = &StatsHandler{}

// StatsHandler wraps set of stats handlers and coordinate their execution.
// Additionally, it tags RPC requests with a common set of labels.
// That way it reduces context manipulation overhead and improves overall performance.
type StatsHandler struct {
	handlers []StatsHandlerCollector
}

// NewStatsHandler allocates a new coordinator.
// It allows passing a various number of handlers that later it will iterate through them.
func NewStatsHandler(handlers ...StatsHandlerCollector) *StatsHandler {
	return &StatsHandler{
		handlers: handlers,
	}
}

// ClientStatsHandler instantiates a default client-side coordinator together with every metric specific stats handler provided by this package.
func ClientStatsHandler(opts ...ShareableOption) *StatsHandler {
	return defaultStatsHandler(Client, opts...)
}

// ClientStatsHandler instantiates a default server-side coordinator together with every metric specific stats handler provided by this package.
func ServerStatsHandler(opts ...ShareableOption) *StatsHandler {
	return defaultStatsHandler(Server, opts...)
}

func defaultStatsHandler(sub Subsystem, opts ...ShareableOption) *StatsHandler {
	var (
		collectorOpts    []CollectorOption
		statsHandlerOpts []StatsHandlerOption
	)

	for _, opt := range opts {
		switch val := opt.(type) {
		case StatsHandlerOption:
			statsHandlerOpts = append(statsHandlerOpts, val)
		case CollectorOption:
			collectorOpts = append(collectorOpts, val)
		default:
			panic(fmt.Sprintf("shareable option does not implement any known type: %T", opt))
		}
	}

	return NewStatsHandler(
		NewConnectionsStatsHandler(sub, NewConnectionsGaugeVec(sub, collectorOpts...)),
		NewRequestsTotalStatsHandler(sub, NewRequestsTotalCounterVec(sub, collectorOpts...), statsHandlerOpts...),
		NewRequestsInFlightStatsHandler(sub, NewRequestsInFlightGaugeVec(sub, collectorOpts...), statsHandlerOpts...),
		NewRequestDurationStatsHandler(sub, NewRequestDurationHistogramVec(sub, collectorOpts...), statsHandlerOpts...),
		NewResponsesTotalStatsHandler(sub, NewResponsesTotalCounterVec(sub, collectorOpts...), statsHandlerOpts...),
		NewMessagesReceivedTotalStatsHandler(sub, NewMessagesReceivedTotalCounterVec(sub, collectorOpts...), statsHandlerOpts...),
		NewMessagesSentTotalStatsHandler(sub, NewMessagesSentTotalCounterVec(sub, collectorOpts...), statsHandlerOpts...),
		NewMessageSentSizeStatsHandler(sub, NewMessageSentSizeHistogramVec(sub, collectorOpts...), statsHandlerOpts...),
		NewMessageReceivedSizeStatsHandler(sub, NewMessageReceivedSizeHistogramVec(sub, collectorOpts...), statsHandlerOpts...),
	)
}

// TagRPC implements stats Handler interface.
func (h *StatsHandler) TagRPC(ctx context.Context, inf *stats.RPCTagInfo) context.Context {
	service, method := split(inf.FullMethodName)

	ctx = context.WithValue(ctx, tagRPCKey, rpcTag{
		isFailFast: strconv.FormatBool(inf.FailFast),
		service:    service,
		method:     method,
	})

	for _, c := range h.handlers {
		ctx = c.TagRPC(ctx, inf)
	}
	return ctx
}

// HandleRPC implements stats Handler interface.
func (h *StatsHandler) HandleRPC(ctx context.Context, sts stats.RPCStats) {
	for _, c := range h.handlers {
		c.HandleRPC(ctx, sts)
	}
}

// TagConn implements stats Handler interface.
func (h *StatsHandler) TagConn(ctx context.Context, inf *stats.ConnTagInfo) context.Context {
	for _, c := range h.handlers {
		ctx = c.TagConn(ctx, inf)
	}
	return ctx
}

// HandleConn implements stats Handler interface.
func (h *StatsHandler) HandleConn(ctx context.Context, sts stats.ConnStats) {
	for _, c := range h.handlers {
		c.HandleConn(ctx, sts)
	}
}

// Describe implements prometheus Collector interface.
func (h *StatsHandler) Describe(in chan<- *prometheus.Desc) {
	for _, c := range h.handlers {
		c.Describe(in)
	}
}

// Collect implements prometheus Collector interface.
func (h *StatsHandler) Collect(in chan<- prometheus.Metric) {
	for _, c := range h.handlers {
		c.Collect(in)
	}
}

type baseStatsHandler struct {
	subsystem Subsystem
	collector prometheus.Collector
	options   statsHandlerOptions
}

// TagRPC implements stats Handler interface.
func (h *baseStatsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
	return ctx
}

// TagConn implements stats Handler interface.
func (h *baseStatsHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context {
	return context.WithValue(ctx, tagConnKey, prometheus.Labels{
		labelRemoteAddr:      info.RemoteAddr.String(),
		labelLocalAddr:       info.LocalAddr.String(),
		labelClientUserAgent: userAgent(ctx),
	})
}

// HandleConn implements stats Handler interface.
func (h *baseStatsHandler) HandleConn(ctx context.Context, stat stats.ConnStats) {
}

// HandleRPC implements stats Handler interface.
func (h *baseStatsHandler) HandleRPC(ctx context.Context, stat stats.RPCStats) {
}

// Describe implements prometheus Collector interface.
func (h *baseStatsHandler) Describe(in chan<- *prometheus.Desc) {
	h.collector.Describe(in)
}

// Collect implements prometheus Collector interface.
func (h *baseStatsHandler) Collect(in chan<- prometheus.Metric) {
	h.collector.Collect(in)
}
<commit_msg>v4 - improved StatsHandler  documentation #2<commit_after>package promgrpc

import (
	"context"
	"fmt"
	"strconv"

	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/grpc/stats"
)

// StatsHandlerCollector is a simple wrapper for stats Handler and prometheus Collector interfaces.
type StatsHandlerCollector interface {
	stats.Handler
	prometheus.Collector
}

var _ StatsHandlerCollector = &StatsHandler{}

// StatsHandler wraps set of stats handlers and coordinate their execution.
// Additionally, it tags RPC requests with a common set of labels.
// That way it reduces context manipulation overhead and improves overall performance.
type StatsHandler struct {
	handlers []StatsHandlerCollector
}

// NewStatsHandler allocates a new coordinator.
// It allows passing a various number of handlers that later it will iterate through.
func NewStatsHandler(handlers ...StatsHandlerCollector) *StatsHandler {
	return &StatsHandler{
		handlers: handlers,
	}
}

// ClientStatsHandler instantiates a default client-side coordinator together with every metric specific stats handler provided by this package.
func ClientStatsHandler(opts ...ShareableOption) *StatsHandler {
	return defaultStatsHandler(Client, opts...)
}

// ClientStatsHandler instantiates a default server-side coordinator together with every metric specific stats handler provided by this package.
func ServerStatsHandler(opts ...ShareableOption) *StatsHandler {
	return defaultStatsHandler(Server, opts...)
}

func defaultStatsHandler(sub Subsystem, opts ...ShareableOption) *StatsHandler {
	var (
		collectorOpts    []CollectorOption
		statsHandlerOpts []StatsHandlerOption
	)

	for _, opt := range opts {
		switch val := opt.(type) {
		case StatsHandlerOption:
			statsHandlerOpts = append(statsHandlerOpts, val)
		case CollectorOption:
			collectorOpts = append(collectorOpts, val)
		default:
			panic(fmt.Sprintf("shareable option does not implement any known type: %T", opt))
		}
	}

	return NewStatsHandler(
		NewConnectionsStatsHandler(sub, NewConnectionsGaugeVec(sub, collectorOpts...)),
		NewRequestsTotalStatsHandler(sub, NewRequestsTotalCounterVec(sub, collectorOpts...), statsHandlerOpts...),
		NewRequestsInFlightStatsHandler(sub, NewRequestsInFlightGaugeVec(sub, collectorOpts...), statsHandlerOpts...),
		NewRequestDurationStatsHandler(sub, NewRequestDurationHistogramVec(sub, collectorOpts...), statsHandlerOpts...),
		NewResponsesTotalStatsHandler(sub, NewResponsesTotalCounterVec(sub, collectorOpts...), statsHandlerOpts...),
		NewMessagesReceivedTotalStatsHandler(sub, NewMessagesReceivedTotalCounterVec(sub, collectorOpts...), statsHandlerOpts...),
		NewMessagesSentTotalStatsHandler(sub, NewMessagesSentTotalCounterVec(sub, collectorOpts...), statsHandlerOpts...),
		NewMessageSentSizeStatsHandler(sub, NewMessageSentSizeHistogramVec(sub, collectorOpts...), statsHandlerOpts...),
		NewMessageReceivedSizeStatsHandler(sub, NewMessageReceivedSizeHistogramVec(sub, collectorOpts...), statsHandlerOpts...),
	)
}

// TagRPC implements stats Handler interface.
func (h *StatsHandler) TagRPC(ctx context.Context, inf *stats.RPCTagInfo) context.Context {
	service, method := split(inf.FullMethodName)

	ctx = context.WithValue(ctx, tagRPCKey, rpcTag{
		isFailFast: strconv.FormatBool(inf.FailFast),
		service:    service,
		method:     method,
	})

	for _, c := range h.handlers {
		ctx = c.TagRPC(ctx, inf)
	}
	return ctx
}

// HandleRPC implements stats Handler interface.
func (h *StatsHandler) HandleRPC(ctx context.Context, sts stats.RPCStats) {
	for _, c := range h.handlers {
		c.HandleRPC(ctx, sts)
	}
}

// TagConn implements stats Handler interface.
func (h *StatsHandler) TagConn(ctx context.Context, inf *stats.ConnTagInfo) context.Context {
	for _, c := range h.handlers {
		ctx = c.TagConn(ctx, inf)
	}
	return ctx
}

// HandleConn implements stats Handler interface.
func (h *StatsHandler) HandleConn(ctx context.Context, sts stats.ConnStats) {
	for _, c := range h.handlers {
		c.HandleConn(ctx, sts)
	}
}

// Describe implements prometheus Collector interface.
func (h *StatsHandler) Describe(in chan<- *prometheus.Desc) {
	for _, c := range h.handlers {
		c.Describe(in)
	}
}

// Collect implements prometheus Collector interface.
func (h *StatsHandler) Collect(in chan<- prometheus.Metric) {
	for _, c := range h.handlers {
		c.Collect(in)
	}
}

type baseStatsHandler struct {
	subsystem Subsystem
	collector prometheus.Collector
	options   statsHandlerOptions
}

// TagRPC implements stats Handler interface.
func (h *baseStatsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
	return ctx
}

// TagConn implements stats Handler interface.
func (h *baseStatsHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context {
	return context.WithValue(ctx, tagConnKey, prometheus.Labels{
		labelRemoteAddr:      info.RemoteAddr.String(),
		labelLocalAddr:       info.LocalAddr.String(),
		labelClientUserAgent: userAgent(ctx),
	})
}

// HandleConn implements stats Handler interface.
func (h *baseStatsHandler) HandleConn(ctx context.Context, stat stats.ConnStats) {
}

// HandleRPC implements stats Handler interface.
func (h *baseStatsHandler) HandleRPC(ctx context.Context, stat stats.RPCStats) {
}

// Describe implements prometheus Collector interface.
func (h *baseStatsHandler) Describe(in chan<- *prometheus.Desc) {
	h.collector.Describe(in)
}

// Collect implements prometheus Collector interface.
func (h *baseStatsHandler) Collect(in chan<- prometheus.Metric) {
	h.collector.Collect(in)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"). You
 * may not use this file except in compliance with the License. A copy of
 * the License is located at
 *
 * 	http://aws.amazon.com/apache2.0/
 *
 * or in the "license" file accompanying this file. This file is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 * ANY KIND, either express or implied. See the License for the specific
 * language governing permissions and limitations under the License.
 */
package ecr

import (
	"bytes"
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ecr"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/remotes"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"golang.org/x/net/context/ctxhttp"
)

// ecrFetcher implements the containerd remotes.Fetcher interface and can be
// used to pull images from Amazon ECR.
type ecrFetcher struct {
	ecrBase
}

var _ remotes.Fetcher = (*ecrFetcher)(nil)

func (f *ecrFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
	ctx = log.WithLogger(ctx, log.G(ctx).WithField("desc", desc))
	log.G(ctx).Debug("ecr.fetch")
	// need to do different things based on the media type
	switch desc.MediaType {
	case
		ocispec.MediaTypeImageManifest,
		images.MediaTypeDockerSchema2Manifest,
		images.MediaTypeDockerSchema1Manifest:
		return f.fetchManifest(ctx, desc)
	case
		images.MediaTypeDockerSchema2Layer,
		images.MediaTypeDockerSchema2LayerGzip,
		images.MediaTypeDockerSchema2Config,
		ocispec.MediaTypeImageLayerGzip,
		ocispec.MediaTypeImageLayer,
		ocispec.MediaTypeImageConfig:
		return f.fetchLayer(ctx, desc)
	default:
		log.G(ctx).
			WithField("media type", desc.MediaType).
			Error("ecr.fetcher: unimplemented media type")
		return nil, unimplemented
	}
	return nil, unimplemented
}

func (f *ecrFetcher) fetchManifest(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
	image, err := f.getManifest(ctx)
	if err != nil {
		return nil, err
	}
	if image == nil {
		return nil, errors.New("fetchManifest: nil image")
	}
	return ioutil.NopCloser(bytes.NewReader([]byte(aws.StringValue(image.ImageManifest)))), nil
}

func (f *ecrFetcher) fetchLayer(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
	log.G(ctx).Debug("ecr.fetcher.layer")
	getDownloadUrlForLayerInput := &ecr.GetDownloadUrlForLayerInput{
		RegistryId:     aws.String(f.ecrSpec.Registry()),
		RepositoryName: aws.String(f.ecrSpec.Repository),
		LayerDigest:    aws.String(desc.Digest.String()),
	}
	output, err := f.client.GetDownloadUrlForLayer(getDownloadUrlForLayerInput)
	if err != nil {
		return nil, err
	}

	downloadURL := aws.StringValue(output.DownloadUrl)
	req, err := http.NewRequest(http.MethodGet, downloadURL, nil)
	if err != nil {
		log.G(ctx).
			WithError(err).
			WithField("desc", desc).
			WithField("url", downloadURL).
			Error("ecr.fetcher.layer: failed to create HTTP request")
		return nil, err
	}
	log.G(ctx).WithField("url", downloadURL).Debug("ecr.fetcher.layer")

	req.Header.Set("Accept", strings.Join([]string{desc.MediaType, `*`}, ", "))
	resp, err := f.doRequest(ctx, req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode > 299 {
		resp.Body.Close()
		return nil, errors.Errorf("ecr.fetcher.layer: unexpected status code %v: %v", downloadURL, resp.Status)
	}
	log.G(ctx).WithField("desc", desc).Debug("ecr.fetcher.layer: returning body")
	return resp.Body, nil
}

func (f *ecrFetcher) doRequest(ctx context.Context, req *http.Request) (*http.Response, error) {
	client := http.DefaultClient // TODO
	resp, err := ctxhttp.Do(ctx, client, req)
	if err != nil {
		return nil, errors.Wrap(err, "failed to do request")
	}
	return resp, nil
}
<commit_msg>fetcher: add support for foreign layers<commit_after>/*
 * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"). You
 * may not use this file except in compliance with the License. A copy of
 * the License is located at
 *
 * 	http://aws.amazon.com/apache2.0/
 *
 * or in the "license" file accompanying this file. This file is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 * ANY KIND, either express or implied. See the License for the specific
 * language governing permissions and limitations under the License.
 */
package ecr

import (
	"bytes"
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ecr"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/remotes"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"golang.org/x/net/context/ctxhttp"
)

// ecrFetcher implements the containerd remotes.Fetcher interface and can be
// used to pull images from Amazon ECR.
type ecrFetcher struct {
	ecrBase
}

var _ remotes.Fetcher = (*ecrFetcher)(nil)

func (f *ecrFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
	ctx = log.WithLogger(ctx, log.G(ctx).WithField("desc", desc))
	log.G(ctx).Debug("ecr.fetch")
	// need to do different things based on the media type
	switch desc.MediaType {
	case
		ocispec.MediaTypeImageManifest,
		images.MediaTypeDockerSchema2Manifest,
		images.MediaTypeDockerSchema1Manifest:
		return f.fetchManifest(ctx, desc)
	case
		images.MediaTypeDockerSchema2Layer,
		images.MediaTypeDockerSchema2LayerGzip,
		images.MediaTypeDockerSchema2Config,
		ocispec.MediaTypeImageLayerGzip,
		ocispec.MediaTypeImageLayer,
		ocispec.MediaTypeImageConfig:
		return f.fetchLayer(ctx, desc)
	case images.MediaTypeDockerSchema2LayerForeignGzip:
		return f.fetchForeignLayer(ctx, desc)
	default:
		log.G(ctx).
			WithField("media type", desc.MediaType).
			Error("ecr.fetcher: unimplemented media type")
		return nil, unimplemented
	}
	return nil, unimplemented
}

func (f *ecrFetcher) fetchManifest(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
	image, err := f.getManifest(ctx)
	if err != nil {
		return nil, err
	}
	if image == nil {
		return nil, errors.New("fetchManifest: nil image")
	}
	return ioutil.NopCloser(bytes.NewReader([]byte(aws.StringValue(image.ImageManifest)))), nil
}

func (f *ecrFetcher) fetchLayer(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
	log.G(ctx).Debug("ecr.fetcher.layer")
	getDownloadUrlForLayerInput := &ecr.GetDownloadUrlForLayerInput{
		RegistryId:     aws.String(f.ecrSpec.Registry()),
		RepositoryName: aws.String(f.ecrSpec.Repository),
		LayerDigest:    aws.String(desc.Digest.String()),
	}
	output, err := f.client.GetDownloadUrlForLayer(getDownloadUrlForLayerInput)
	if err != nil {
		return nil, err
	}

	downloadURL := aws.StringValue(output.DownloadUrl)
	return f.fetchLayerURL(ctx, desc, downloadURL)
}

func (f *ecrFetcher) fetchForeignLayer(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
	log.G(ctx).Debug("ecr.fetcher.layer.foreign")
	if len(desc.URLs) < 1 {
		log.G(ctx).Error("cannot pull foreign layer without URL")
	}
	// TODO try more than one URL
	return f.fetchLayerURL(ctx, desc, desc.URLs[0])
}

func (f *ecrFetcher) fetchLayerURL(ctx context.Context, desc ocispec.Descriptor, downloadURL string) (io.ReadCloser, error) {
	req, err := http.NewRequest(http.MethodGet, downloadURL, nil)
	if err != nil {
		log.G(ctx).
			WithError(err).
			WithField("url", downloadURL).
			Error("ecr.fetcher.layer.url: failed to create HTTP request")
		return nil, err
	}
	log.G(ctx).WithField("url", downloadURL).Debug("ecr.fetcher.layer.url")

	req.Header.Set("Accept", strings.Join([]string{desc.MediaType, `*`}, ", "))
	resp, err := f.doRequest(ctx, req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode > 299 {
		resp.Body.Close()
		if resp.StatusCode == http.StatusNotFound {
			return nil, errors.Wrapf(errdefs.ErrNotFound, "content at %v not found", downloadURL)
		}
		return nil, errors.Errorf("ecr.fetcher.layer.url: unexpected status code %v: %v", downloadURL, resp.Status)
	}
	log.G(ctx).WithField("desc", desc).Debug("ecr.fetcher.layer.url: returning body")
	return resp.Body, nil
}

func (f *ecrFetcher) doRequest(ctx context.Context, req *http.Request) (*http.Response, error) {
	client := http.DefaultClient // TODO
	resp, err := ctxhttp.Do(ctx, client, req)
	if err != nil {
		return nil, errors.Wrap(err, "failed to do request")
	}
	return resp, nil
}
<|endoftext|>