<commit_before>package QesyDb

import (
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"reflect"
	"strconv"
	"strings"
	"time"
)

var Db *sql.DB

type Model struct {
	Cond   interface{}
	Insert map[string]string
	Update map[string]string
	Field  string
	Table  string
	Index  string
	Limit  interface{}
	Sort   string
	IsDeug int
	Tx     *sql.Tx
}

// Connect  is a method with a sql.
func Connect(connStr string) {
	sqlDb, err := sql.Open("mysql", connStr)
	//defer sqlDb.Close()
	sqlDb.SetConnMaxLifetime(1800)
	sqlDb.SetMaxIdleConns(0)
	sqlDb.SetMaxOpenConns(600)
	if err != nil {
		log.Fatal("mysql connect error")
	}
	err = sqlDb.Ping()
	if err != nil {
		log.Fatal("mysql ping error")
	}
	fmt.Println("mysql connect sueccss")
	Db = sqlDb
}

func Begin() (*sql.Tx, error) {
	return Db.Begin()
}

func Rollback(tx *sql.Tx) error {
	return tx.Rollback()
}

func Commit(tx *sql.Tx) error {
	return tx.Commit()
}

// ExecSelectIndex  is a method with a sql.
func (m *Model) ExecSelectIndex() (map[string]map[string]string, error) {
	resultsSlice, _ := m.execSelect()
	retArr := map[string]map[string]string{}
	for _, v := range resultsSlice {
		if v[m.Index] == "" {
			continue
		}
		retArr[v[m.Index]] = v
	}
	m.Clean()
	return retArr, nil
}

func (m *Model) ExecSelect() ([]map[string]string, error) {
	ret, err := m.execSelect()
	m.Clean()
	return ret, err
}

// ExecSelect is a method with a sql.
func (m *Model) execSelect() ([]map[string]string, error) {
	cond := m.getSQLCond()
	field := m.getSQLField()
	sort := m.getSort()
	limit := m.getSQLLimite()
	sqlStr := "SELECT " + field + " FROM " + m.Table + cond + sort + limit + ";"
	m.Debug(sqlStr)
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	//defer stmt.Close()
	resultsSlice := []map[string]string{}
	if err != nil {
		return resultsSlice, err
	}
	rows, err := stmt.Query()
	if err != nil {
		fmt.Println("DBERR:", rows, err, sqlStr)
		return resultsSlice, err
	}
	defer rows.Close()
	fields, err := rows.Columns()
	if err != nil {
		return resultsSlice, err
	}

	for rows.Next() {
		result := make(map[string]string)
		var scanResultContainers []interface{}
		for i := 0; i < len(fields); i++ {
			var scanResultContainer interface{}
			scanResultContainers = append(scanResultContainers, &scanResultContainer)
		}
		if err := rows.Scan(scanResultContainers...); err != nil {
			return resultsSlice, err
		}
		for k, v := range fields {
			rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[k]))
			if rawValue.Interface() == nil {
				continue
			}
			rawType := reflect.TypeOf(rawValue.Interface())
			rawVal := reflect.ValueOf(rawValue.Interface())
			var str string
			switch rawType.Kind() {
			case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				str = strconv.FormatInt(rawVal.Int(), 10)
				result[v] = str
			case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
				str = strconv.FormatUint(rawVal.Uint(), 10)
				result[v] = str
			case reflect.Float32, reflect.Float64:
				str = strconv.FormatFloat(rawVal.Float(), 'f', -1, 64)
				result[v] = str
			case reflect.Slice:
				if rawType.Elem().Kind() == reflect.Uint8 {
					result[v] = string(rawVal.Interface().([]byte))
					break
				}
			case reflect.String:
				str = rawVal.String()
				result[v] = str
			case reflect.Struct:
				str = rawVal.Interface().(time.Time).Format("2006-01-02 15:04:05.000 -0700")
				result[v] = str
			case reflect.Bool:
				if rawVal.Bool() {
					result[v] = "1"
				} else {
					result[v] = "0"
				}
			}
		}
		resultsSlice = append(resultsSlice, result)
	}
	return resultsSlice, nil
}

func (m *Model) ExecSelectOne() (map[string]string, error) {
	resultsSlice, err := m.ExecSelect()
	if len(resultsSlice) == 0 {
		return map[string]string{}, err
	}
	return resultsSlice[0], nil
}

func (m *Model) ExecUpdate() (sql.Result, error) {
	updateStr := m.getSQLUpdate()
	condStr := m.getSQLCond()
	sqlStr := "UPDATE " + m.Table + " SET " + updateStr + condStr + ";"
	m.Debug(sqlStr)
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	if err != nil {
		fmt.Println("DBERR:", stmt, err, sqlStr)
		return nil, err
	}
	result, err := stmt.Exec()
	m.Clean()
	return result, err
}

func (m *Model) ExecInsert() (sql.Result, error) {
	insert := m.getSQLInsert()
	sqlStr := "INSERT INTO " + m.Table + " " + insert + ";"
	m.Debug(sqlStr)
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	if err != nil {
		fmt.Println("DBERR:", stmt, err, sqlStr)
		return nil, err
	}
	result, err := stmt.Exec()
	m.Clean()
	return result, err
}

func (m *Model) ExecReplace() (sql.Result, error) {
	insert := m.getSQLInsert()
	sqlStr := "REPLACE INTO " + m.Table + " " + insert + ";"
	m.Debug(sqlStr)
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	if err != nil {
		fmt.Println("DBERR:", stmt, err, sqlStr)
		return nil, err
	}
	result, err := stmt.Exec()
	m.Clean()
	return result, err
}

func (m *Model) ExecDelete() (sql.Result, error) {
	condStr := m.getSQLCond()
	sqlStr := "DELETE FROM " + m.Table + condStr + ";"
	m.Debug(sqlStr)
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	if err != nil {
		fmt.Println("DBERR:", stmt, err, sqlStr)
		return nil, err
	}
	result, err := stmt.Exec()
	m.Clean()
	return result, err
}

func GetLastInsertId(result sql.Result) (int64, error) {
	return result.LastInsertId()
}

func GetRowsAffected(result sql.Result) (int64, error) {
	return result.RowsAffected()
}

func (m *Model) getSQLCond() string {
	if str, ok := m.Cond.(string); ok {
		return str
	}
	var strArr []string
	if arr, ok := m.Cond.(map[string]string); ok {
		var strArr []string
		for k, v := range arr {
			strArr = append(strArr, k+"='"+v+"'")
		}
		return " WHERE " + strings.Join(strArr, " && ")
	}
	if arr, ok := m.Cond.(map[string]interface{}); ok {
		for k, v := range arr {
			if isStr, ok := v.(string); ok {
				strArr = append(strArr, k+"='"+isStr+"'")
			}
			if isStrArr, ok := v.([]string); ok {
				for k, v := range isStrArr {
					isStrArr[k] = "'" + v + "'"
				}
				strArr = append(strArr, k+" in ("+strings.Join(isStrArr, ",")+")")
			}
		}
		return " WHERE " + strings.Join(strArr, " && ")
	}
	return ""
}

func (m *Model) getSQLField() string {
	if m.Field != "" {
		return m.Field
	}
	return "*"
}

func (m *Model) getSort() string {
	if m.Sort != "" {
		return " ORDER BY " + m.Sort + " "
	}
	return ""
}

func (m *Model) getSQLUpdate() string {
	var strArr []string
	for k, v := range m.Update {
		strArr = append(strArr, k+"='"+v+"'")
	}
	return strings.Join(strArr, ",")
}

func (m *Model) getSQLInsert() string {
	var fieldArr, valueArr []string
	for k, v := range m.Insert {
		fieldArr = append(fieldArr, k)
		valueArr = append(valueArr, "'"+v+"'")
	}
	return "(" + strings.Join(fieldArr, ",") + ") values (" + strings.Join(valueArr, ",") + ")"
}

func (m *Model) getSQLLimite() string {
	if strArr, ok := m.Limit.([2]int); ok {
		return " LIMIT " + fmt.Sprintf("%d", strArr[0]) + ", " + fmt.Sprintf("%d", strArr[1])
	}
	return ""
}

func (m *Model) Clean() {
	m.Cond = nil
	m.Insert = nil
	m.Update = nil
	m.Field = ""
	m.Table = ""
	m.Index = ""
	m.Limit = nil
	m.Sort = ""
	m.IsDeug = 0
}

func (m *Model) Debug(sql string) {
	if m.IsDeug == 1 {
		fmt.Println(sql)
	}
}
<commit_msg>add exec<commit_after>package QesyDb

import (
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"reflect"
	"strconv"
	"strings"
	"time"
)

var Db *sql.DB

type Model struct {
	Cond   interface{}
	Insert map[string]string
	Update map[string]string
	Field  string
	Table  string
	Index  string
	Limit  interface{}
	Sort   string
	IsDeug int
	Tx     *sql.Tx
}

// Connect  is a method with a sql.
func Connect(connStr string) {
	sqlDb, err := sql.Open("mysql", connStr)
	//defer sqlDb.Close()
	sqlDb.SetConnMaxLifetime(1800)
	sqlDb.SetMaxIdleConns(0)
	sqlDb.SetMaxOpenConns(600)
	if err != nil {
		log.Fatal("mysql connect error")
	}
	err = sqlDb.Ping()
	if err != nil {
		log.Fatal("mysql ping error")
	}
	fmt.Println("mysql connect sueccss")
	Db = sqlDb
}

func Begin() (*sql.Tx, error) {
	return Db.Begin()
}

func Rollback(tx *sql.Tx) error {
	return tx.Rollback()
}

func Commit(tx *sql.Tx) error {
	return tx.Commit()
}

// ExecSelectIndex  is a method with a sql.
func (m *Model) ExecSelectIndex() (map[string]map[string]string, error) {
	resultsSlice, _ := m.execSelect()
	retArr := map[string]map[string]string{}
	for _, v := range resultsSlice {
		if v[m.Index] == "" {
			continue
		}
		retArr[v[m.Index]] = v
	}
	m.Clean()
	return retArr, nil
}

func (m *Model) ExecSelect() ([]map[string]string, error) {
	ret, err := m.execSelect()
	m.Clean()
	return ret, err
}

// ExecSelect is a method with a sql.
func (m *Model) execSelect() ([]map[string]string, error) {
	cond := m.getSQLCond()
	field := m.getSQLField()
	sort := m.getSort()
	limit := m.getSQLLimite()
	sqlStr := "SELECT " + field + " FROM " + m.Table + cond + sort + limit + ";"
	m.Debug(sqlStr)
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	//defer stmt.Close()
	resultsSlice := []map[string]string{}
	if err != nil {
		return resultsSlice, err
	}
	rows, err := stmt.Query()
	if err != nil {
		fmt.Println("DBERR:", rows, err, sqlStr)
		return resultsSlice, err
	}
	defer rows.Close()
	fields, err := rows.Columns()
	if err != nil {
		return resultsSlice, err
	}

	for rows.Next() {
		result := make(map[string]string)
		var scanResultContainers []interface{}
		for i := 0; i < len(fields); i++ {
			var scanResultContainer interface{}
			scanResultContainers = append(scanResultContainers, &scanResultContainer)
		}
		if err := rows.Scan(scanResultContainers...); err != nil {
			return resultsSlice, err
		}
		for k, v := range fields {
			rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[k]))
			if rawValue.Interface() == nil {
				continue
			}
			rawType := reflect.TypeOf(rawValue.Interface())
			rawVal := reflect.ValueOf(rawValue.Interface())
			var str string
			switch rawType.Kind() {
			case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				str = strconv.FormatInt(rawVal.Int(), 10)
				result[v] = str
			case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
				str = strconv.FormatUint(rawVal.Uint(), 10)
				result[v] = str
			case reflect.Float32, reflect.Float64:
				str = strconv.FormatFloat(rawVal.Float(), 'f', -1, 64)
				result[v] = str
			case reflect.Slice:
				if rawType.Elem().Kind() == reflect.Uint8 {
					result[v] = string(rawVal.Interface().([]byte))
					break
				}
			case reflect.String:
				str = rawVal.String()
				result[v] = str
			case reflect.Struct:
				str = rawVal.Interface().(time.Time).Format("2006-01-02 15:04:05.000 -0700")
				result[v] = str
			case reflect.Bool:
				if rawVal.Bool() {
					result[v] = "1"
				} else {
					result[v] = "0"
				}
			}
		}
		resultsSlice = append(resultsSlice, result)
	}
	return resultsSlice, nil
}

func (m *Model) ExecSelectOne() (map[string]string, error) {
	resultsSlice, err := m.ExecSelect()
	if len(resultsSlice) == 0 {
		return map[string]string{}, err
	}
	return resultsSlice[0], nil
}

func (m *Model) ExecUpdate() (sql.Result, error) {
	updateStr := m.getSQLUpdate()
	condStr := m.getSQLCond()
	sqlStr := "UPDATE " + m.Table + " SET " + updateStr + condStr + ";"
	m.Debug(sqlStr)
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	if err != nil {
		fmt.Println("DBERR:", stmt, err, sqlStr)
		return nil, err
	}
	result, err := stmt.Exec()
	m.Clean()
	return result, err
}

func (m *Model) ExecInsert() (sql.Result, error) {
	insert := m.getSQLInsert()
	sqlStr := "INSERT INTO " + m.Table + " " + insert + ";"
	m.Debug(sqlStr)
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	if err != nil {
		fmt.Println("DBERR:", stmt, err, sqlStr)
		return nil, err
	}
	result, err := stmt.Exec()
	m.Clean()
	return result, err
}

func (m *Model) ExecReplace() (sql.Result, error) {
	insert := m.getSQLInsert()
	sqlStr := "REPLACE INTO " + m.Table + " " + insert + ";"
	m.Debug(sqlStr)
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	if err != nil {
		fmt.Println("DBERR:", stmt, err, sqlStr)
		return nil, err
	}
	result, err := stmt.Exec()
	m.Clean()
	return result, err
}

func (m *Model) ExecDelete() (sql.Result, error) {
	condStr := m.getSQLCond()
	sqlStr := "DELETE FROM " + m.Table + condStr + ";"
	m.Debug(sqlStr)
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	if err != nil {
		fmt.Println("DBERR:", stmt, err, sqlStr)
		return nil, err
	}
	result, err := stmt.Exec()
	m.Clean()
	return result, err
}

func (m *Model) Exec(sqlStr string) (sql.Result, error) {
	var err error
	var stmt *sql.Stmt
	if m.Tx == nil {
		stmt, err = Db.Prepare(sqlStr)
		defer stmt.Close()
	} else {
		stmt, err = m.Tx.Prepare(sqlStr)
	}
	if err != nil {
		fmt.Println("DBERR:", stmt, err, sqlStr)
		return nil, err
	}
	result, err := stmt.Exec()
	m.Clean()
	return result, err
}

func GetLastInsertId(result sql.Result) (int64, error) {
	return result.LastInsertId()
}

func GetRowsAffected(result sql.Result) (int64, error) {
	return result.RowsAffected()
}

func (m *Model) getSQLCond() string {
	if str, ok := m.Cond.(string); ok {
		return str
	}
	var strArr []string
	if arr, ok := m.Cond.(map[string]string); ok {
		var strArr []string
		for k, v := range arr {
			strArr = append(strArr, k+"='"+v+"'")
		}
		return " WHERE " + strings.Join(strArr, " && ")
	}
	if arr, ok := m.Cond.(map[string]interface{}); ok {
		for k, v := range arr {
			if isStr, ok := v.(string); ok {
				strArr = append(strArr, k+"='"+isStr+"'")
			}
			if isStrArr, ok := v.([]string); ok {
				for k, v := range isStrArr {
					isStrArr[k] = "'" + v + "'"
				}
				strArr = append(strArr, k+" in ("+strings.Join(isStrArr, ",")+")")
			}
		}
		return " WHERE " + strings.Join(strArr, " && ")
	}
	return ""
}

func (m *Model) getSQLField() string {
	if m.Field != "" {
		return m.Field
	}
	return "*"
}

func (m *Model) getSort() string {
	if m.Sort != "" {
		return " ORDER BY " + m.Sort + " "
	}
	return ""
}

func (m *Model) getSQLUpdate() string {
	var strArr []string
	for k, v := range m.Update {
		strArr = append(strArr, k+"='"+v+"'")
	}
	return strings.Join(strArr, ",")
}

func (m *Model) getSQLInsert() string {
	var fieldArr, valueArr []string
	for k, v := range m.Insert {
		fieldArr = append(fieldArr, k)
		valueArr = append(valueArr, "'"+v+"'")
	}
	return "(" + strings.Join(fieldArr, ",") + ") values (" + strings.Join(valueArr, ",") + ")"
}

func (m *Model) getSQLLimite() string {
	if strArr, ok := m.Limit.([2]int); ok {
		return " LIMIT " + fmt.Sprintf("%d", strArr[0]) + ", " + fmt.Sprintf("%d", strArr[1])
	}
	return ""
}

func (m *Model) Clean() {
	m.Cond = nil
	m.Insert = nil
	m.Update = nil
	m.Field = ""
	m.Table = ""
	m.Index = ""
	m.Limit = nil
	m.Sort = ""
	m.IsDeug = 0
}

func (m *Model) Debug(sql string) {
	if m.IsDeug == 1 {
		fmt.Println(sql)
	}
}
<|endoftext|>


<commit_before>package schema

import (
	"bytes"
	"sort"
	"strconv"
)

func SerializeValueForHash(buf *bytes.Buffer, val interface{}, schema *Schema) {
	if val == nil {
		buf.WriteRune(';')
		return
	}

	switch schema.Type {
	case TypeBool:
		if val.(bool) {
			buf.WriteRune('1')
		} else {
			buf.WriteRune('0')
		}
	case TypeInt:
		buf.WriteString(strconv.Itoa(val.(int)))
	case TypeFloat:
		buf.WriteString(strconv.FormatFloat(val.(float64), 'g', -1, 64))
	case TypeString:
		buf.WriteString(val.(string))
	case TypeList:
		buf.WriteRune('(')
		l := val.([]interface{})
		for _, innerVal := range l {
			serializeCollectionMemberForHash(buf, innerVal, schema.Elem)
		}
		buf.WriteRune(')')
	case TypeMap:
		m := val.(map[string]interface{})
		var keys []string
		for k := range m {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		buf.WriteRune('[')
		for _, k := range keys {
			innerVal := m[k]
			buf.WriteString(k)
			buf.WriteRune(':')
			serializeCollectionMemberForHash(buf, innerVal, schema.Elem)
		}
		buf.WriteRune(']')
	case TypeSet:
		buf.WriteRune('{')
		s := val.(*Set)
		for _, innerVal := range s.List() {
			serializeCollectionMemberForHash(buf, innerVal, schema.Elem)
		}
		buf.WriteRune('}')
	default:
		panic("unknown schema type to serialize")
	}
	buf.WriteRune(';')
}

// SerializeValueForHash appends a serialization of the given resource config
// to the given buffer, guaranteeing deterministic results given the same value
// and schema.
//
// Its primary purpose is as input into a hashing function in order
// to hash complex substructures when used in sets, and so the serialization
// is not reversible.
func SerializeResourceForHash(buf *bytes.Buffer, val interface{}, resource *Resource) {
	sm := resource.Schema
	m := val.(map[string]interface{})
	var keys []string
	for k := range sm {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		innerSchema := sm[k]
		// Skip attributes that are not user-provided. Computed attributes
		// do not contribute to the hash since their ultimate value cannot
		// be known at plan/diff time.
		if !(innerSchema.Required || innerSchema.Optional) {
			continue
		}

		buf.WriteString(k)
		buf.WriteRune(':')
		innerVal := m[k]
		SerializeValueForHash(buf, innerVal, innerSchema)
	}
}

func serializeCollectionMemberForHash(buf *bytes.Buffer, val interface{}, elem interface{}) {
	switch tElem := elem.(type) {
	case *Schema:
		SerializeValueForHash(buf, val, tElem)
	case *Resource:
		buf.WriteRune('<')
		SerializeResourceForHash(buf, val, tElem)
		buf.WriteString(">;")
	default:
		panic("invalid element type")
	}
}
<commit_msg>Serialization for hash panics on TypeMap<commit_after>package schema

import (
	"bytes"
	"fmt"
	"sort"
	"strconv"
)

func SerializeValueForHash(buf *bytes.Buffer, val interface{}, schema *Schema) {
	if val == nil {
		buf.WriteRune(';')
		return
	}

	switch schema.Type {
	case TypeBool:
		if val.(bool) {
			buf.WriteRune('1')
		} else {
			buf.WriteRune('0')
		}
	case TypeInt:
		buf.WriteString(strconv.Itoa(val.(int)))
	case TypeFloat:
		buf.WriteString(strconv.FormatFloat(val.(float64), 'g', -1, 64))
	case TypeString:
		buf.WriteString(val.(string))
	case TypeList:
		buf.WriteRune('(')
		l := val.([]interface{})
		for _, innerVal := range l {
			serializeCollectionMemberForHash(buf, innerVal, schema.Elem)
		}
		buf.WriteRune(')')
	case TypeMap:

		m := val.(map[string]interface{})
		var keys []string
		for k := range m {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		buf.WriteRune('[')
		for _, k := range keys {
			innerVal := m[k]
			if innerVal == nil {
				continue
			}
			buf.WriteString(k)
			buf.WriteRune(':')

			switch innerVal := innerVal.(type) {
			case int:
				buf.WriteString(strconv.Itoa(innerVal))
			case float64:
				buf.WriteString(strconv.FormatFloat(innerVal, 'g', -1, 64))
			case string:
				buf.WriteString(innerVal)
			default:
				panic(fmt.Sprintf("unknown value type in TypeMap %T", innerVal))
			}

			buf.WriteRune(';')
		}
		buf.WriteRune(']')
	case TypeSet:
		buf.WriteRune('{')
		s := val.(*Set)
		for _, innerVal := range s.List() {
			serializeCollectionMemberForHash(buf, innerVal, schema.Elem)
		}
		buf.WriteRune('}')
	default:
		panic("unknown schema type to serialize")
	}
	buf.WriteRune(';')
}

// SerializeValueForHash appends a serialization of the given resource config
// to the given buffer, guaranteeing deterministic results given the same value
// and schema.
//
// Its primary purpose is as input into a hashing function in order
// to hash complex substructures when used in sets, and so the serialization
// is not reversible.
func SerializeResourceForHash(buf *bytes.Buffer, val interface{}, resource *Resource) {
	sm := resource.Schema
	m := val.(map[string]interface{})
	var keys []string
	for k := range sm {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		innerSchema := sm[k]
		// Skip attributes that are not user-provided. Computed attributes
		// do not contribute to the hash since their ultimate value cannot
		// be known at plan/diff time.
		if !(innerSchema.Required || innerSchema.Optional) {
			continue
		}

		buf.WriteString(k)
		buf.WriteRune(':')
		innerVal := m[k]
		SerializeValueForHash(buf, innerVal, innerSchema)
	}
}

func serializeCollectionMemberForHash(buf *bytes.Buffer, val interface{}, elem interface{}) {
	switch tElem := elem.(type) {
	case *Schema:
		SerializeValueForHash(buf, val, tElem)
	case *Resource:
		buf.WriteRune('<')
		SerializeResourceForHash(buf, val, tElem)
		buf.WriteString(">;")
	default:
		panic(fmt.Sprintf("invalid element type: %T", tElem))
	}
}
<|endoftext|>


<commit_before>package torrent

import (
	"context"
	"errors"
	"io"
	"log"
	"sync"

	"github.com/anacrolix/missinggo"
	"github.com/anacrolix/torrent/peer_protocol"
)

type Reader interface {
	io.Reader
	io.Seeker
	io.Closer
	missinggo.ReadContexter
	SetReadahead(int64)
	SetResponsive()
}

// Piece range by piece index, [begin, end).
type pieceRange struct {
	begin, end pieceIndex
}

// Accesses Torrent data via a Client. Reads block until the data is
// available. Seeks and readahead also drive Client behaviour.
type reader struct {
	t          *Torrent
	responsive bool
	// Adjust the read/seek window to handle Readers locked to File extents
	// and the like.
	offset, length int64
	// Ensure operations that change the position are exclusive, like Read()
	// and Seek().
	opMu sync.Mutex

	// Required when modifying pos and readahead, or reading them without
	// opMu.
	mu        sync.Locker
	pos       int64
	readahead int64
	// The cached piece range this reader wants downloaded. The zero value
	// corresponds to nothing. We cache this so that changes can be detected,
	// and bubbled up to the Torrent only as required.
	pieces pieceRange
}

var _ io.ReadCloser = &reader{}

// Don't wait for pieces to complete and be verified. Read calls return as
// soon as they can when the underlying chunks become available.
func (r *reader) SetResponsive() {
	r.responsive = true
	r.t.cl.event.Broadcast()
}

// Disable responsive mode. TODO: Remove?
func (r *reader) SetNonResponsive() {
	r.responsive = false
	r.t.cl.event.Broadcast()
}

// Configure the number of bytes ahead of a read that should also be
// prioritized in preparation for further reads.
func (r *reader) SetReadahead(readahead int64) {
	r.mu.Lock()
	r.readahead = readahead
	r.mu.Unlock()
	r.t.cl.lock()
	defer r.t.cl.unlock()
	r.posChanged()
}

func (r *reader) readable(off int64) (ret bool) {
	if r.t.closed.IsSet() {
		return true
	}
	req, ok := r.t.offsetRequest(r.torrentOffset(off))
	if !ok {
		panic(off)
	}
	if r.responsive {
		return r.t.haveChunk(req)
	}
	return r.t.pieceComplete(pieceIndex(req.Index))
}

// How many bytes are available to read. Max is the most we could require.
func (r *reader) available(off, max int64) (ret int64) {
	off += r.offset
	for max > 0 {
		req, ok := r.t.offsetRequest(off)
		if !ok {
			break
		}
		if !r.t.haveChunk(req) {
			break
		}
		len1 := int64(req.Length) - (off - r.t.requestOffset(req))
		max -= len1
		ret += len1
		off += len1
	}
	// Ensure that ret hasn't exceeded our original max.
	if max < 0 {
		ret += max
	}
	return
}

func (r *reader) waitReadable(off int64) {
	// We may have been sent back here because we were told we could read but
	// it failed.
	r.t.cl.event.Wait()
}

// Calculates the pieces this reader wants downloaded, ignoring the cached
// value at r.pieces.
func (r *reader) piecesUncached() (ret pieceRange) {
	ra := r.readahead
	if ra < 1 {
		// Needs to be at least 1, because [x, x) means we don't want
		// anything.
		ra = 1
	}
	if ra > r.length-r.pos {
		ra = r.length - r.pos
	}
	ret.begin, ret.end = r.t.byteRegionPieces(r.torrentOffset(r.pos), ra)
	return
}

func (r *reader) Read(b []byte) (n int, err error) {
	return r.ReadContext(context.Background(), b)
}

func (r *reader) ReadContext(ctx context.Context, b []byte) (n int, err error) {
	// This is set under the Client lock if the Context is canceled.
	var ctxErr error
	if ctx.Done() != nil {
		ctx, cancel := context.WithCancel(ctx)
		// Abort the goroutine when the function returns.
		defer cancel()
		go func() {
			<-ctx.Done()
			r.t.cl.lock()
			ctxErr = ctx.Err()
			r.t.tickleReaders()
			r.t.cl.unlock()
		}()
	}
	// Hmmm, if a Read gets stuck, this means you can't change position for
	// other purposes. That seems reasonable, but unusual.
	r.opMu.Lock()
	defer r.opMu.Unlock()
	for len(b) != 0 {
		var n1 int
		n1, err = r.readOnceAt(b, r.pos, &ctxErr)
		if n1 == 0 {
			if err == nil {
				panic("expected error")
			}
			break
		}
		b = b[n1:]
		n += n1
		r.mu.Lock()
		r.pos += int64(n1)
		r.posChanged()
		r.mu.Unlock()
	}
	if r.pos >= r.length {
		err = io.EOF
	} else if err == io.EOF {
		err = io.ErrUnexpectedEOF
	}
	return
}

// Wait until some data should be available to read. Tickles the client if it
// isn't. Returns how much should be readable without blocking.
func (r *reader) waitAvailable(pos, wanted int64, ctxErr *error) (avail int64) {
	r.t.cl.lock()
	defer r.t.cl.unlock()
	for !r.readable(pos) && *ctxErr == nil {
		r.waitReadable(pos)
	}
	return r.available(pos, wanted)
}

func (r *reader) torrentOffset(readerPos int64) int64 {
	return r.offset + readerPos
}

// Performs at most one successful read to torrent storage.
func (r *reader) readOnceAt(b []byte, pos int64, ctxErr *error) (n int, err error) {
	if pos >= r.length {
		err = io.EOF
		return
	}
	for {
		avail := r.waitAvailable(pos, int64(len(b)), ctxErr)
		if avail == 0 {
			if r.t.closed.IsSet() {
				err = errors.New("torrent closed")
				return
			}
			if *ctxErr != nil {
				err = *ctxErr
				return
			}
		}
		pi := peer_protocol.Integer(r.torrentOffset(pos) / r.t.info.PieceLength)
		ip := r.t.info.Piece(int(pi))
		po := r.torrentOffset(pos) % r.t.info.PieceLength
		b1 := missinggo.LimitLen(b, ip.Length()-po, avail)
		n, err = r.t.readAt(b1, r.torrentOffset(pos))
		if n != 0 {
			err = nil
			return
		}
		r.t.cl.lock()
		// TODO: Just reset pieces in the readahead window. This might help
		// prevent thrashing with small caches and file and piece priorities.
		log.Printf("error reading torrent %q piece %d offset %d, %d bytes: %s", r.t, pi, po, len(b1), err)
		r.t.updateAllPieceCompletions()
		r.t.updateAllPiecePriorities()
		r.t.cl.unlock()
	}
}

func (r *reader) Close() error {
	r.t.cl.lock()
	defer r.t.cl.unlock()
	r.t.deleteReader(r)
	return nil
}

func (r *reader) posChanged() {
	to := r.piecesUncached()
	from := r.pieces
	if to == from {
		return
	}
	r.pieces = to
	// log.Printf("reader pos changed %v->%v", from, to)
	r.t.readerPosChanged(from, to)
}

func (r *reader) Seek(off int64, whence int) (ret int64, err error) {
	r.opMu.Lock()
	defer r.opMu.Unlock()

	r.mu.Lock()
	defer r.mu.Unlock()
	switch whence {
	case io.SeekStart:
		r.pos = off
	case io.SeekCurrent:
		r.pos += off
	case io.SeekEnd:
		r.pos = r.length + off
	default:
		err = errors.New("bad whence")
	}
	ret = r.pos

	r.posChanged()
	return
}
<commit_msg>Fix double quoting in a log statement<commit_after>package torrent

import (
	"context"
	"errors"
	"io"
	"log"
	"sync"

	"github.com/anacrolix/missinggo"
	"github.com/anacrolix/torrent/peer_protocol"
)

type Reader interface {
	io.Reader
	io.Seeker
	io.Closer
	missinggo.ReadContexter
	SetReadahead(int64)
	SetResponsive()
}

// Piece range by piece index, [begin, end).
type pieceRange struct {
	begin, end pieceIndex
}

// Accesses Torrent data via a Client. Reads block until the data is
// available. Seeks and readahead also drive Client behaviour.
type reader struct {
	t          *Torrent
	responsive bool
	// Adjust the read/seek window to handle Readers locked to File extents
	// and the like.
	offset, length int64
	// Ensure operations that change the position are exclusive, like Read()
	// and Seek().
	opMu sync.Mutex

	// Required when modifying pos and readahead, or reading them without
	// opMu.
	mu        sync.Locker
	pos       int64
	readahead int64
	// The cached piece range this reader wants downloaded. The zero value
	// corresponds to nothing. We cache this so that changes can be detected,
	// and bubbled up to the Torrent only as required.
	pieces pieceRange
}

var _ io.ReadCloser = &reader{}

// Don't wait for pieces to complete and be verified. Read calls return as
// soon as they can when the underlying chunks become available.
func (r *reader) SetResponsive() {
	r.responsive = true
	r.t.cl.event.Broadcast()
}

// Disable responsive mode. TODO: Remove?
func (r *reader) SetNonResponsive() {
	r.responsive = false
	r.t.cl.event.Broadcast()
}

// Configure the number of bytes ahead of a read that should also be
// prioritized in preparation for further reads.
func (r *reader) SetReadahead(readahead int64) {
	r.mu.Lock()
	r.readahead = readahead
	r.mu.Unlock()
	r.t.cl.lock()
	defer r.t.cl.unlock()
	r.posChanged()
}

func (r *reader) readable(off int64) (ret bool) {
	if r.t.closed.IsSet() {
		return true
	}
	req, ok := r.t.offsetRequest(r.torrentOffset(off))
	if !ok {
		panic(off)
	}
	if r.responsive {
		return r.t.haveChunk(req)
	}
	return r.t.pieceComplete(pieceIndex(req.Index))
}

// How many bytes are available to read. Max is the most we could require.
func (r *reader) available(off, max int64) (ret int64) {
	off += r.offset
	for max > 0 {
		req, ok := r.t.offsetRequest(off)
		if !ok {
			break
		}
		if !r.t.haveChunk(req) {
			break
		}
		len1 := int64(req.Length) - (off - r.t.requestOffset(req))
		max -= len1
		ret += len1
		off += len1
	}
	// Ensure that ret hasn't exceeded our original max.
	if max < 0 {
		ret += max
	}
	return
}

func (r *reader) waitReadable(off int64) {
	// We may have been sent back here because we were told we could read but
	// it failed.
	r.t.cl.event.Wait()
}

// Calculates the pieces this reader wants downloaded, ignoring the cached
// value at r.pieces.
func (r *reader) piecesUncached() (ret pieceRange) {
	ra := r.readahead
	if ra < 1 {
		// Needs to be at least 1, because [x, x) means we don't want
		// anything.
		ra = 1
	}
	if ra > r.length-r.pos {
		ra = r.length - r.pos
	}
	ret.begin, ret.end = r.t.byteRegionPieces(r.torrentOffset(r.pos), ra)
	return
}

func (r *reader) Read(b []byte) (n int, err error) {
	return r.ReadContext(context.Background(), b)
}

func (r *reader) ReadContext(ctx context.Context, b []byte) (n int, err error) {
	// This is set under the Client lock if the Context is canceled.
	var ctxErr error
	if ctx.Done() != nil {
		ctx, cancel := context.WithCancel(ctx)
		// Abort the goroutine when the function returns.
		defer cancel()
		go func() {
			<-ctx.Done()
			r.t.cl.lock()
			ctxErr = ctx.Err()
			r.t.tickleReaders()
			r.t.cl.unlock()
		}()
	}
	// Hmmm, if a Read gets stuck, this means you can't change position for
	// other purposes. That seems reasonable, but unusual.
	r.opMu.Lock()
	defer r.opMu.Unlock()
	for len(b) != 0 {
		var n1 int
		n1, err = r.readOnceAt(b, r.pos, &ctxErr)
		if n1 == 0 {
			if err == nil {
				panic("expected error")
			}
			break
		}
		b = b[n1:]
		n += n1
		r.mu.Lock()
		r.pos += int64(n1)
		r.posChanged()
		r.mu.Unlock()
	}
	if r.pos >= r.length {
		err = io.EOF
	} else if err == io.EOF {
		err = io.ErrUnexpectedEOF
	}
	return
}

// Wait until some data should be available to read. Tickles the client if it
// isn't. Returns how much should be readable without blocking.
func (r *reader) waitAvailable(pos, wanted int64, ctxErr *error) (avail int64) {
	r.t.cl.lock()
	defer r.t.cl.unlock()
	for !r.readable(pos) && *ctxErr == nil {
		r.waitReadable(pos)
	}
	return r.available(pos, wanted)
}

func (r *reader) torrentOffset(readerPos int64) int64 {
	return r.offset + readerPos
}

// Performs at most one successful read to torrent storage.
func (r *reader) readOnceAt(b []byte, pos int64, ctxErr *error) (n int, err error) {
	if pos >= r.length {
		err = io.EOF
		return
	}
	for {
		avail := r.waitAvailable(pos, int64(len(b)), ctxErr)
		if avail == 0 {
			if r.t.closed.IsSet() {
				err = errors.New("torrent closed")
				return
			}
			if *ctxErr != nil {
				err = *ctxErr
				return
			}
		}
		pi := peer_protocol.Integer(r.torrentOffset(pos) / r.t.info.PieceLength)
		ip := r.t.info.Piece(int(pi))
		po := r.torrentOffset(pos) % r.t.info.PieceLength
		b1 := missinggo.LimitLen(b, ip.Length()-po, avail)
		n, err = r.t.readAt(b1, r.torrentOffset(pos))
		if n != 0 {
			err = nil
			return
		}
		r.t.cl.lock()
		// TODO: Just reset pieces in the readahead window. This might help
		// prevent thrashing with small caches and file and piece priorities.
		log.Printf("error reading torrent %s piece %d offset %d, %d bytes: %s",
			r.t.infoHash.HexString(), pi, po, len(b1), err)
		r.t.updateAllPieceCompletions()
		r.t.updateAllPiecePriorities()
		r.t.cl.unlock()
	}
}

func (r *reader) Close() error {
	r.t.cl.lock()
	defer r.t.cl.unlock()
	r.t.deleteReader(r)
	return nil
}

func (r *reader) posChanged() {
	to := r.piecesUncached()
	from := r.pieces
	if to == from {
		return
	}
	r.pieces = to
	// log.Printf("reader pos changed %v->%v", from, to)
	r.t.readerPosChanged(from, to)
}

func (r *reader) Seek(off int64, whence int) (ret int64, err error) {
	r.opMu.Lock()
	defer r.opMu.Unlock()

	r.mu.Lock()
	defer r.mu.Unlock()
	switch whence {
	case io.SeekStart:
		r.pos = off
	case io.SeekCurrent:
		r.pos += off
	case io.SeekEnd:
		r.pos = r.length + off
	default:
		err = errors.New("bad whence")
	}
	ret = r.pos

	r.posChanged()
	return
}
<|endoftext|>


<commit_before>package main_test

import (
	. "github.com/alphagov/cloudflare-configure"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"

	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"testing"
)

var _ = Describe("CloudFlare", func() {
	var (
		server     *ghttp.Server
		query      *CloudFlareQuery
		cloudFlare *CloudFlare
	)

	BeforeEach(func() {
		server = ghttp.NewServer()
		query = &CloudFlareQuery{RootURL: server.URL()}
		cloudFlare = NewCloudFlare(query)
	})

	AfterEach(func() {
		server.Close()
	})

	Describe("Zones()", func() {
		BeforeEach(func() {
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/zones"),
					ghttp.RespondWith(http.StatusOK, `{
						"errors": [],
						"messages": [],
						"result": [
							{"id": "123", "name": "foo"},
							{"id": "456", "name": "bar"}
						],
						"success": true
					}`),
				),
			)
		})

		It("should return two CloudFlareZoneItems", func() {
			zones, err := cloudFlare.Zones()

			Expect(err).To(BeNil())
			Expect(zones).To(Equal([]CloudFlareZoneItem{
				CloudFlareZoneItem{
					ID:   "123",
					Name: "foo",
				},
				CloudFlareZoneItem{
					ID:   "456",
					Name: "bar",
				},
			}))
		})
	})

	Describe("MakeRequest()", func() {
		var req *http.Request

		BeforeEach(func() {
			req, _ = query.NewRequest("GET", "/something")
		})

		Context("200, success: false, errors: []", func() {
			BeforeEach(func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", "/something"),
						ghttp.RespondWith(http.StatusOK, `{
							"errors": [],
							"messages": [],
							"result": [],
							"success": false
						}`),
					),
				)
			})

			It("should return error", func() {
				resp, err := cloudFlare.MakeRequest(req)

				Expect(resp).To(BeNil())
				Expect(err).ToNot(BeNil())
			})
		})

		Context("200, success: true, errors: [something bad]", func() {
			BeforeEach(func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", "/something"),
						ghttp.RespondWith(http.StatusOK, `{
							"errors": ["something bad"],
							"messages": [],
							"result": [],
							"success": true
						}`),
					),
				)
			})

			It("should return error", func() {
				resp, err := cloudFlare.MakeRequest(req)

				Expect(resp).To(BeNil())
				Expect(err).ToNot(BeNil())
			})
		})

		Context("500, empty body", func() {
			BeforeEach(func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", "/something"),
						ghttp.RespondWith(http.StatusServiceUnavailable, ""),
					),
				)
			})

			It("should return error", func() {
				resp, err := cloudFlare.MakeRequest(req)

				Expect(resp).To(BeNil())
				Expect(err).ToNot(BeNil())
			})
		})
	})
})

func testCloudFlareServer(status int, body string) *httptest.Server {
	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(status)
		fmt.Fprintf(w, body)
	}))
}

func TestGettingSettings(t *testing.T) {
	const zoneID = "123"

	expectedSettings := []CloudFlareConfigItem{
		CloudFlareConfigItem{
			ID:         "always_online",
			Value:      "off",
			ModifiedOn: "2014-07-09T11:50:56.595672Z",
			Editable:   true,
		},
		CloudFlareConfigItem{
			ID:         "browser_cache_ttl",
			Value:      float64(14400),
			ModifiedOn: "2014-07-09T11:50:56.595672Z",
			Editable:   true,
		},
	}

	testServer := testCloudFlareServer(200, `{
		"errors": [],
		"messages": [],
		"result": [
			{"id": "always_online", "value": "off", "modified_on": "2014-07-09T11:50:56.595672Z", "editable": true},
			{"id": "browser_cache_ttl", "value": 14400, "modified_on": "2014-07-09T11:50:56.595672Z", "editable": true}
		],
		"success": true
	}`)
	defer testServer.Close()

	query := &CloudFlareQuery{RootURL: testServer.URL}
	cloudFlare := NewCloudFlare(query)

	settings, err := cloudFlare.Settings(zoneID)
	if err != nil {
		t.Fatalf("Expected to get settings with no errors", err.Error())
	}
	if len(settings) != 2 {
		t.Fatalf("Expected 2 settings items, got %d", len(settings))
	}
	if !reflect.DeepEqual(settings, expectedSettings) {
		t.Fatal("Settings response doesn't match", settings)
	}
}

func TestChangeSetting(t *testing.T) {
	const zoneID = "123"
	const settingID = "always_online"
	const settingVal = "off"

	receivedRequest := false

	testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		receivedRequest = true

		if method := r.Method; method != "PATCH" {
			t.Fatal("Incorrect request method", method)
		}

		expectedURL := fmt.Sprintf("/zones/%s/settings/%s", zoneID, settingID)
		if !strings.HasSuffix(r.URL.String(), expectedURL) {
			t.Fatal("Request URL was incorrect")
		}

		body, err := ioutil.ReadAll(r.Body)
		if err != nil {
			t.Fatal("Unable to read request body", err)
		}

		var setting CloudFlareRequestItem
		err = json.Unmarshal(body, &setting)
		if err != nil {
			t.Fatal("Unable to parse request body", err)
		}

		expectedSetting := &CloudFlareRequestItem{
			Value: settingVal,
		}
		if !reflect.DeepEqual(setting, *expectedSetting) {
			t.Fatal("Request was incorrect", setting, expectedSetting)
		}

		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, `{
			"errors": [],
			"messages": [],
			"result": {
				"id": "always_online",
				"value": "off",
				"modified_on": "2014-07-09T11:50:56.595672Z",
				"editable": true
			},
			"success": true
		}`)
	}))
	defer testServer.Close()

	query := &CloudFlareQuery{
		RootURL:   testServer.URL,
		AuthEmail: "user@example.com",
		AuthKey:   "abc123",
	}
	cloudFlare := NewCloudFlare(query)

	err := cloudFlare.Set(zoneID, settingID, settingVal)
	if err != nil {
		t.Fatal("Unable to set setting")
	}

	if !receivedRequest {
		t.Fatal("Expected test server to receive request")
	}
}
<commit_msg>Convert Set() and Settings() to ginkgo<commit_after>package main_test

import (
	. "github.com/alphagov/cloudflare-configure"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"

	"fmt"
	"net/http"
)

var _ = Describe("CloudFlare", func() {
	var (
		server     *ghttp.Server
		query      *CloudFlareQuery
		cloudFlare *CloudFlare
	)

	BeforeEach(func() {
		server = ghttp.NewServer()
		query = &CloudFlareQuery{RootURL: server.URL()}
		cloudFlare = NewCloudFlare(query)
	})

	AfterEach(func() {
		server.Close()
	})

	Describe("Zones()", func() {
		BeforeEach(func() {
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/zones"),
					ghttp.RespondWith(http.StatusOK, `{
						"errors": [],
						"messages": [],
						"result": [
							{"id": "123", "name": "foo"},
							{"id": "456", "name": "bar"}
						],
						"success": true
					}`),
				),
			)
		})

		It("should return two CloudFlareZoneItems", func() {
			zones, err := cloudFlare.Zones()

			Expect(zones).To(Equal([]CloudFlareZoneItem{
				CloudFlareZoneItem{
					ID:   "123",
					Name: "foo",
				},
				CloudFlareZoneItem{
					ID:   "456",
					Name: "bar",
				},
			}))
			Expect(err).To(BeNil())
		})
	})

	Describe("MakeRequest()", func() {
		var req *http.Request

		BeforeEach(func() {
			req, _ = query.NewRequest("GET", "/something")
		})

		Context("200, success: false, errors: []", func() {
			BeforeEach(func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", "/something"),
						ghttp.RespondWith(http.StatusOK, `{
							"errors": [],
							"messages": [],
							"result": [],
							"success": false
						}`),
					),
				)
			})

			It("should return error", func() {
				resp, err := cloudFlare.MakeRequest(req)

				Expect(resp).To(BeNil())
				Expect(err).ToNot(BeNil())
			})
		})

		Context("200, success: true, errors: [something bad]", func() {
			BeforeEach(func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", "/something"),
						ghttp.RespondWith(http.StatusOK, `{
							"errors": ["something bad"],
							"messages": [],
							"result": [],
							"success": true
						}`),
					),
				)
			})

			It("should return error", func() {
				resp, err := cloudFlare.MakeRequest(req)

				Expect(resp).To(BeNil())
				Expect(err).ToNot(BeNil())
			})
		})

		Context("500, empty body", func() {
			BeforeEach(func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", "/something"),
						ghttp.RespondWith(http.StatusServiceUnavailable, ""),
					),
				)
			})

			It("should return error", func() {
				resp, err := cloudFlare.MakeRequest(req)

				Expect(resp).To(BeNil())
				Expect(err).ToNot(BeNil())
			})
		})
	})

	Describe("Settings()", func() {
		var zoneID = "123"

		BeforeEach(func() {
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET",
						fmt.Sprintf("/zones/%s/settings", zoneID),
					),
					ghttp.RespondWith(http.StatusOK, `{
						"errors": [],
						"messages": [],
						"result": [
							{
								"id": "always_online",
								"value": "off",
								"modified_on": "2014-07-09T11:50:56.595672Z",
								"editable": true
							},
							{
								"id": "browser_cache_ttl",
								"value": 14400,
								"modified_on": "2014-07-09T11:50:56.595672Z",
								"editable": true
							}
						],
						"success": true
					}`),
				),
			)
		})

		It("should return two CloudFlareConfigItems", func() {
			settings, err := cloudFlare.Settings(zoneID)

			Expect(settings).To(Equal([]CloudFlareConfigItem{
				CloudFlareConfigItem{
					ID:         "always_online",
					Value:      "off",
					ModifiedOn: "2014-07-09T11:50:56.595672Z",
					Editable:   true,
				},
				CloudFlareConfigItem{
					ID:         "browser_cache_ttl",
					Value:      float64(14400),
					ModifiedOn: "2014-07-09T11:50:56.595672Z",
					Editable:   true,
				},
			}))
			Expect(err).To(BeNil())
		})
	})

	Describe("Set()", func() {
		var zoneID = "123"
		var settingKey = "always_online"
		var settingVal = "off"

		BeforeEach(func() {
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PATCH",
						fmt.Sprintf("/zones/%s/settings/%s", zoneID, settingKey),
					),
					ghttp.VerifyJSON(fmt.Sprintf(`{"value": "%s"}`, settingVal)),

					ghttp.RespondWith(http.StatusOK, `{
						"errors": [],
						"messages": [],
						"result": {
							"id": "always_online",
							"value": "off",
							"modified_on": "2014-07-09T11:50:56.595672Z",
							"editable": true
						},
						"success": true
					}`),
				),
			)
		})

		It("should return two CloudFlareConfigItems", func() {
			err := cloudFlare.Set(zoneID, settingKey, settingVal)

			Expect(err).To(BeNil())
		})
	})
})
<|endoftext|>


<commit_before>package cioutil

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/Sirupsen/logrus"
	"github.com/garyburd/go-oauth/oauth"
)

// ClientRequest defines information that can be used to make a request
type ClientRequest struct {
	Method      string
	Path        string
	FormValues  interface{}
	QueryValues interface{}
}

// DoFormRequest makes the actual request
func (cio Cio) DoFormRequest(request ClientRequest, result interface{}) error {

	// Construct the url
	cioURL := cio.Host + request.Path + QueryString(request.QueryValues)

	// Construct the body
	var bodyReader io.Reader
	bodyValues := FormValues(request.FormValues)
	bodyString := bodyValues.Encode()
	if len(bodyString) > 0 {
		bodyReader = bytes.NewReader([]byte(bodyString))
	}
	logRequest(cio.Log, cioURL, bodyValues)

	// Construct the request
	httpReq, err := cio.createRequest(request, cioURL, bodyReader, bodyValues)
	if err != nil {
		return err
	}

	// Send the request
	return cio.sendRequest(httpReq, result, cioURL)
}

// createRequest creates the *http.Request object
func (cio Cio) createRequest(request ClientRequest, cioURL string, bodyReader io.Reader, bodyValues url.Values) (*http.Request, error) {
	// Construct the request
	httpReq, err := http.NewRequest(request.Method, cioURL, bodyReader)
	if err != nil {
		return httpReq, fmt.Errorf("Could not create request: %s", err)
	}

	// oAuth signature
	var client oauth.Client
	client.Credentials = oauth.Credentials{Token: cio.apiKey, Secret: cio.apiSecret}

	// Add headers
	httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	httpReq.Header.Set("Accept", "application/json")
	httpReq.Header.Set("Accept-Charset", "utf-8")
	httpReq.Header.Set("User-Agent", "Golang CIO Library")
	httpReq.Header.Set("Authorization", client.AuthorizationHeader(nil, request.Method, httpReq.URL, bodyValues))

	return httpReq, nil
}

// sendRequest sends the *http.Request
func (cio Cio) sendRequest(httpReq *http.Request, result interface{}, cioURL string) error {
	// Create the HTTP client
	httpClient := &http.Client{
		Transport: http.DefaultTransport,
		Timeout:   cio.RequestTimeout,
	}

	// Make the request
	res, err := httpClient.Do(httpReq)
	if err != nil {
		return fmt.Errorf("Failed to make request: %s", err)
	}

	// Parse the response
	defer func() {
		if closeErr := res.Body.Close(); closeErr != nil {
			logBodyCloseError(cio.Log, closeErr)
		}
	}()

	resBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return fmt.Errorf("Could not read response: %s", err)
	}
	resBodyString := string(resBody)

	// Unmarshal result
	err = json.Unmarshal(resBody, &result)

	// Log the response
	logResponse(cio.Log, cioURL, res.StatusCode, resBodyString, err)

	// Return special error if Status Code >= 400
	if res.StatusCode >= 400 {
		return fmt.Errorf("%d Status Code with Payload %s", res.StatusCode, resBodyString)
	}

	// Return Unmarshal error (if any) if Status Code is < 400
	return err
}

// logRequest logs the request about to be made to CIO, redacting sensitive information in the body
func logRequest(log Logger, cioURL string, bodyValues url.Values) {
	if log != nil {

		// Copy url.Values
		redactedValues := url.Values{}
		for k, v := range bodyValues {
			redactedValues[k] = v
		}

		// Redact sensitive information
		if val := redactedValues.Get("password"); len(val) > 0 {
			redactedValues.Set("password", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}
		if val := redactedValues.Get("provider_refresh_token"); len(val) > 0 {
			redactedValues.Set("provider_refresh_token", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}
		if val := redactedValues.Get("provider_consumer_key"); len(val) > 0 {
			redactedValues.Set("provider_consumer_key", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}
		if val := redactedValues.Get("provider_consumer_secret"); len(val) > 0 {
			redactedValues.Set("provider_consumer_secret", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}

		// Actually log
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithFields(logrus.Fields{"url": cioURL, "payload": redactedValues.Encode()}).Debug("Creating new request to CIO")
		} else {
			// Else just log with Println
			log.Println("Creating new request to: " + cioURL + " with payload: " + redactedValues.Encode())
		}
	}
}

// logBodyCloseError logs any error that happens when trying to close the *http.Response.Body
func logBodyCloseError(log Logger, closeError error) {
	if log != nil {
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithError(closeError).Warn("Unable to close response body from CIO")
		} else {
			// Else just log with Println
			log.Println("Unable to close response body from CIO, with error: " + closeError.Error())
		}
	}
}

// logResponse logs the response from CIO, if any logger is set
func logResponse(log Logger, cioURL string, statusCode int, responseBody string, unmarshalError error) {
	if log != nil {

		// TODO: redact access_token and access_token_secret before logging (only occurs with 3-legged oauth [rare])

		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logEntry := logrusLogger.WithFields(logrus.Fields{
				"url":        cioURL,
				"statusCode": fmt.Sprintf("%d", statusCode),
				"payload":    responseBody})
			if unmarshalError != nil || statusCode >= 400 {
				logEntry.Warn("Received response from CIO")
			} else {
				logEntry.Debug("Received response from CIO")
			}

		} else {
			// Else just log with Println
			log.Println("Received response from: " + cioURL +
				" with status code: " + fmt.Sprintf("%d", statusCode) +
				" and payload: " + responseBody)
		}
	}
}
<commit_msg>log request method (GET/POST/etc)<commit_after>package cioutil

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/Sirupsen/logrus"
	"github.com/garyburd/go-oauth/oauth"
)

// ClientRequest defines information that can be used to make a request
type ClientRequest struct {
	Method      string
	Path        string
	FormValues  interface{}
	QueryValues interface{}
}

// DoFormRequest makes the actual request
func (cio Cio) DoFormRequest(request ClientRequest, result interface{}) error {

	// Construct the url
	cioURL := cio.Host + request.Path + QueryString(request.QueryValues)

	// Construct the body
	var bodyReader io.Reader
	bodyValues := FormValues(request.FormValues)
	bodyString := bodyValues.Encode()
	if len(bodyString) > 0 {
		bodyReader = bytes.NewReader([]byte(bodyString))
	}
	logRequest(cio.Log, request.Method, cioURL, bodyValues)

	// Construct the request
	httpReq, err := cio.createRequest(request, cioURL, bodyReader, bodyValues)
	if err != nil {
		return err
	}

	// Send the request
	return cio.sendRequest(httpReq, result, cioURL)
}

// createRequest creates the *http.Request object
func (cio Cio) createRequest(request ClientRequest, cioURL string, bodyReader io.Reader, bodyValues url.Values) (*http.Request, error) {
	// Construct the request
	httpReq, err := http.NewRequest(request.Method, cioURL, bodyReader)
	if err != nil {
		return httpReq, fmt.Errorf("Could not create request: %s", err)
	}

	// oAuth signature
	var client oauth.Client
	client.Credentials = oauth.Credentials{Token: cio.apiKey, Secret: cio.apiSecret}

	// Add headers
	httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	httpReq.Header.Set("Accept", "application/json")
	httpReq.Header.Set("Accept-Charset", "utf-8")
	httpReq.Header.Set("User-Agent", "Golang CIO Library")
	httpReq.Header.Set("Authorization", client.AuthorizationHeader(nil, request.Method, httpReq.URL, bodyValues))

	return httpReq, nil
}

// sendRequest sends the *http.Request
func (cio Cio) sendRequest(httpReq *http.Request, result interface{}, cioURL string) error {
	// Create the HTTP client
	httpClient := &http.Client{
		Transport: http.DefaultTransport,
		Timeout:   cio.RequestTimeout,
	}

	// Make the request
	res, err := httpClient.Do(httpReq)
	if err != nil {
		return fmt.Errorf("Failed to make request: %s", err)
	}

	// Parse the response
	defer func() {
		if closeErr := res.Body.Close(); closeErr != nil {
			logBodyCloseError(cio.Log, closeErr)
		}
	}()

	resBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return fmt.Errorf("Could not read response: %s", err)
	}
	resBodyString := string(resBody)

	// Unmarshal result
	err = json.Unmarshal(resBody, &result)

	// Log the response
	logResponse(cio.Log, httpReq.Method, cioURL, res.StatusCode, resBodyString, err)

	// Return special error if Status Code >= 400
	if res.StatusCode >= 400 {
		return fmt.Errorf("%d Status Code with Payload %s", res.StatusCode, resBodyString)
	}

	// Return Unmarshal error (if any) if Status Code is < 400
	return err
}

// logRequest logs the request about to be made to CIO, redacting sensitive information in the body
func logRequest(log Logger, method string, cioURL string, bodyValues url.Values) {
	if log != nil {

		// Copy url.Values
		redactedValues := url.Values{}
		for k, v := range bodyValues {
			redactedValues[k] = v
		}

		// Redact sensitive information
		if val := redactedValues.Get("password"); len(val) > 0 {
			redactedValues.Set("password", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}
		if val := redactedValues.Get("provider_refresh_token"); len(val) > 0 {
			redactedValues.Set("provider_refresh_token", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}
		if val := redactedValues.Get("provider_consumer_key"); len(val) > 0 {
			redactedValues.Set("provider_consumer_key", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}
		if val := redactedValues.Get("provider_consumer_secret"); len(val) > 0 {
			redactedValues.Set("provider_consumer_secret", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}

		// Actually log
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithFields(logrus.Fields{"httpMethod": method, "url": cioURL, "payload": redactedValues.Encode()}).Debug("Creating new request to CIO")
		} else {
			// Else just log with Println
			log.Println("Creating new " + method + " request to: " + cioURL + " with payload: " + redactedValues.Encode())
		}
	}
}

// logBodyCloseError logs any error that happens when trying to close the *http.Response.Body
func logBodyCloseError(log Logger, closeError error) {
	if log != nil {
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithError(closeError).Warn("Unable to close response body from CIO")
		} else {
			// Else just log with Println
			log.Println("Unable to close response body from CIO, with error: " + closeError.Error())
		}
	}
}

// logResponse logs the response from CIO, if any logger is set
func logResponse(log Logger, method string, cioURL string, statusCode int, responseBody string, unmarshalError error) {
	if log != nil {

		// TODO: redact access_token and access_token_secret before logging (only occurs with 3-legged oauth [rare])

		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logEntry := logrusLogger.WithFields(logrus.Fields{
				"httpMethod": method,
				"url":        cioURL,
				"statusCode": fmt.Sprintf("%d", statusCode),
				"payload":    responseBody})
			if unmarshalError != nil || statusCode >= 400 {
				logEntry.Warn("Received response from CIO")
			} else {
				logEntry.Debug("Received response from CIO")
			}

		} else {
			// Else just log with Println
			log.Println("Received response from " + method + " to: " + cioURL +
				" with status code: " + fmt.Sprintf("%d", statusCode) +
				" and payload: " + responseBody)
		}
	}
}
<|endoftext|>


<commit_before>package cloudwatch

import (
	"fmt"
	"log"
	"regexp"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/lucagrulla/cw/timeutil"
)

func params(logGroupName string, streamNames []*string, epochStartTime int64, epochEndTime int64, grep *string, follow *bool) *cloudwatchlogs.FilterLogEventsInput {
	startTimeInt64 := epochStartTime * secondInMillis
	endTimeInt64 := epochEndTime * secondInMillis
	params := &cloudwatchlogs.FilterLogEventsInput{
		LogGroupName: &logGroupName,
		Interleaved:  aws.Bool(true),
		StartTime:    &startTimeInt64}

	if *grep != "" {
		params.FilterPattern = grep
	}

	if streamNames != nil {
		params.LogStreamNames = streamNames
	}

	if !*follow && endTimeInt64 != 0 {
		params.EndTime = &endTimeInt64
	}
	return params
}

type eventCache struct {
	seen map[string]bool
	sync.RWMutex
}

func (c *eventCache) Has(eventID string) bool {
	c.RLock()
	defer c.RUnlock()
	return c.seen[eventID]
}

func (c *eventCache) Add(eventID string) {
	c.Lock()
	defer c.Unlock()
	c.seen[eventID] = true
}

func (c *eventCache) Size() int {
	c.RLock()
	defer c.RUnlock()
	return len(c.seen)
}

func (c *eventCache) Reset() {
	c.Lock()
	defer c.Unlock()
	c.seen = make(map[string]bool)
}

type logStreams struct {
	groupStreams []*string
	sync.RWMutex
}

func (s *logStreams) reset(groupStreams []*string) {
	s.Lock()
	defer s.Unlock()
	s.groupStreams = groupStreams
}

func (s *logStreams) get() []*string {
	s.Lock()
	defer s.Unlock()
	return s.groupStreams
}

//Tail tails the given stream names in the specified log group name
//To tail all the available streams logStreamName has to be '*'
//It returns a channel where logs line are published
//Unless the follow flag is true the channel is closed once there are no more events available
func Tail(logGroupName *string, logStreamName *string, follow *bool, startTime *time.Time, endTime *time.Time, grep *string, grepv *string) <-chan *cloudwatchlogs.FilteredLogEvent {
	cwl := cwClient()

	startTimeEpoch := timeutil.ParseTime(startTime.Format(timeutil.TimeFormat)).Unix()
	lastSeenTimestamp := startTimeEpoch

	var endTimeEpoch int64
	if !endTime.IsZero() {
		endTimeEpoch = timeutil.ParseTime(endTime.Format(timeutil.TimeFormat)).Unix()
	}

	ch := make(chan *cloudwatchlogs.FilteredLogEvent)
	timer := time.NewTimer(time.Millisecond * 250)

	cache := &eventCache{seen: make(map[string]bool)}
	logStreams := &logStreams{}

	if *logStreamName != "*" {
		getStreams := func(logGroupName *string, logStreamName *string) []*string {
			var streams []*string
			for stream := range LsStreams(logGroupName, logStreamName, lastSeenTimestamp*secondInMillis, endTimeEpoch*secondInMillis) {
				streams = append(streams, stream)
			}
			if len(streams) == 0 {
				fmt.Println("No such log stream(s).")
				close(ch)
			}
			if len(streams) >= 100 { //FilterLogEventPages won't take more than 100 stream names
				streams = streams[0:100]
			}
			return streams
		}
		logStreams.reset(getStreams(logGroupName, logStreamName))

		go func() {
			ticker := time.NewTicker(time.Second * 5)
			for range ticker.C {
				logStreams.reset(getStreams(logGroupName, logStreamName))
			}
		}()
	}

	re := regexp.MustCompile(*grepv)
	pageHandler := func(res *cloudwatchlogs.FilterLogEventsOutput, lastPage bool) bool {
		for _, event := range res.Events {
			if *grepv == "" || !re.MatchString(*event.Message) {
				eventTimestamp := *event.Timestamp / secondInMillis
				if eventTimestamp != lastSeenTimestamp {
					lastSeenTimestamp = eventTimestamp
					if cache.Size() >= 1000 {
						cache.Reset()
					}
				}

				if !cache.Has(*event.EventId) {
					cache.Add(*event.EventId)
					ch <- event
				} else {
					//fmt.Printf("%s already seen\n", *event.EventId)
				}
			}
		}

		if lastPage {
			if !*follow {
				close(ch)
			} else {
				//fmt.Println("LAST PAGE")
				//AWS API accepts 5 reqs/sec
				timer.Reset(time.Millisecond * 205)
			}
		}
		return !lastPage
	}
	if *follow || lastSeenTimestamp == startTimeEpoch {
		go func() {
			for range timer.C {
				//FilterLogEventPages won't take more than 100 stream names
				logParam := params(*logGroupName, logStreams.get(), lastSeenTimestamp, endTimeEpoch, grep, follow)
				error := cwl.FilterLogEventsPages(logParam, pageHandler)
				if error != nil {
					if awsErr, ok := error.(awserr.Error); ok {
						log.Fatalf(awsErr.Message())
					}
				}
			}
		}()
	}
	return ch
}
<commit_msg>increase cache size<commit_after>package cloudwatch

import (
	"fmt"
	"log"
	"regexp"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/lucagrulla/cw/timeutil"
)

func params(logGroupName string, streamNames []*string, epochStartTime int64, epochEndTime int64, grep *string, follow *bool) *cloudwatchlogs.FilterLogEventsInput {
	startTimeInt64 := epochStartTime * secondInMillis
	endTimeInt64 := epochEndTime * secondInMillis
	params := &cloudwatchlogs.FilterLogEventsInput{
		LogGroupName: &logGroupName,
		Interleaved:  aws.Bool(true),
		StartTime:    &startTimeInt64}

	if *grep != "" {
		params.FilterPattern = grep
	}

	if streamNames != nil {
		params.LogStreamNames = streamNames
	}

	if !*follow && endTimeInt64 != 0 {
		params.EndTime = &endTimeInt64
	}
	return params
}

type eventCache struct {
	seen map[string]bool
	sync.RWMutex
}

func (c *eventCache) Has(eventID string) bool {
	c.RLock()
	defer c.RUnlock()
	return c.seen[eventID]
}

func (c *eventCache) Add(eventID string) {
	c.Lock()
	defer c.Unlock()
	c.seen[eventID] = true
}

func (c *eventCache) Size() int {
	c.RLock()
	defer c.RUnlock()
	return len(c.seen)
}

func (c *eventCache) Reset() {
	c.Lock()
	defer c.Unlock()
	c.seen = make(map[string]bool)
}

type logStreams struct {
	groupStreams []*string
	sync.RWMutex
}

func (s *logStreams) reset(groupStreams []*string) {
	s.Lock()
	defer s.Unlock()
	s.groupStreams = groupStreams
}

func (s *logStreams) get() []*string {
	s.Lock()
	defer s.Unlock()
	return s.groupStreams
}

//Tail tails the given stream names in the specified log group name
//To tail all the available streams logStreamName has to be '*'
//It returns a channel where logs line are published
//Unless the follow flag is true the channel is closed once there are no more events available
func Tail(logGroupName *string, logStreamName *string, follow *bool, startTime *time.Time, endTime *time.Time, grep *string, grepv *string) <-chan *cloudwatchlogs.FilteredLogEvent {
	cwl := cwClient()

	startTimeEpoch := timeutil.ParseTime(startTime.Format(timeutil.TimeFormat)).Unix()
	lastSeenTimestamp := startTimeEpoch

	var endTimeEpoch int64
	if !endTime.IsZero() {
		endTimeEpoch = timeutil.ParseTime(endTime.Format(timeutil.TimeFormat)).Unix()
	}

	ch := make(chan *cloudwatchlogs.FilteredLogEvent)
	timer := time.NewTimer(time.Millisecond * 250)

	cache := &eventCache{seen: make(map[string]bool)}
	go func() { //check cache size every 250ms and eventually purge
		cacheTicker := time.NewTicker(250 * time.Millisecond)
		for range cacheTicker.C {
			size := cache.Size()
			if size >= 5000 {
				// fmt.Printf(">>>cache reset:%d,\n ", size)
				cache.Reset()
			}
		}
	}()
	logStreams := &logStreams{}

	if *logStreamName != "*" {
		getStreams := func(logGroupName *string, logStreamName *string) []*string {
			var streams []*string
			for stream := range LsStreams(logGroupName, logStreamName, lastSeenTimestamp*secondInMillis, endTimeEpoch*secondInMillis) {
				streams = append(streams, stream)
			}
			if len(streams) == 0 {
				fmt.Println("No such log stream(s).")
				close(ch)
			}
			if len(streams) >= 100 { //FilterLogEventPages won't take more than 100 stream names
				streams = streams[0:100]
			}
			return streams
		}
		logStreams.reset(getStreams(logGroupName, logStreamName))

		go func() {
			ticker := time.NewTicker(time.Second * 5)
			for range ticker.C {
				logStreams.reset(getStreams(logGroupName, logStreamName))
			}
		}()
	}

	re := regexp.MustCompile(*grepv)
	pageHandler := func(res *cloudwatchlogs.FilterLogEventsOutput, lastPage bool) bool {
		for _, event := range res.Events {
			if *grepv == "" || !re.MatchString(*event.Message) {

				if !cache.Has(*event.EventId) {
					eventTimestamp := *event.Timestamp / secondInMillis

					if eventTimestamp != lastSeenTimestamp {
						if eventTimestamp < lastSeenTimestamp {
							// fmt.Printf("OLD EVENT:%s, evTS:%d, lTS:%d, cache size:%d \n", event, eventTimestamp, lastSeenTimestamp, cache.Size())
						}
						lastSeenTimestamp = eventTimestamp
					}
					cache.Add(*event.EventId)
					ch <- event
				} else {
					//fmt.Printf("%s already seen\n", *event.EventId)
				}
			}
		}

		if lastPage {
			if !*follow {
				close(ch)
			} else {
				//fmt.Println("LAST PAGE")
				//AWS API accepts 5 reqs/sec
				timer.Reset(time.Millisecond * 205)
			}
		}
		return !lastPage
	}
	if *follow || lastSeenTimestamp == startTimeEpoch {
		go func() {
			for range timer.C {
				//FilterLogEventPages won't take more than 100 stream names
				logParam := params(*logGroupName, logStreams.get(), lastSeenTimestamp, endTimeEpoch, grep, follow)
				error := cwl.FilterLogEventsPages(logParam, pageHandler)
				if error != nil {
					if awsErr, ok := error.(awserr.Error); ok {
						log.Fatalf(awsErr.Message())
					}
				}
			}
		}()
	}
	return ch
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/rubyist/circuitbreaker"
	"time"
	log "github.com/Sirupsen/logrus"
)

type ExtendedCircuitBreakerMeta struct {
	CircuitBreakerMeta
	CB *circuit.Breaker
}

func NewCircuitBreaker(apiSpec APIDefinition) ExtendedCircuitBreakerMeta {
	breakerMeta := ExtendedCircuitBreakerMeta{CircuitBreakerMeta: apiSpec.CircuitBreakerMeta}
	breakerMeta.CB = circuit.NewRateBreaker(apiSpec.CircuitBreaker.ThresholdPercent, apiSpec.CircuitBreaker.Samples)

	events := breakerMeta.CB.Subscribe()

	go func() {
		path := apiSpec.Proxy.ListenPath
		timerActive := false
		for {
			e := <-events
			switch e {
			case circuit.BreakerTripped:
				log.Warning("[PROXY] [CIRCUIT BREKER] Breaker tripped for path: ", path)
				log.Debug("Breaker tripped: ", e)

				// Start a timer function
				if !timerActive {
					go func(timeout int, breaker *circuit.Breaker) {
						log.Debug("-- Sleeping for (s): ", timeout)
						time.Sleep(time.Duration(timeout) * time.Second)
						log.Debug("-- Resetting breaker")
						breaker.Reset()
						timerActive = false
					}(apiSpec.CircuitBreaker.ReturnToServiceAfter, breakerMeta.CB)
					timerActive = true
				}
			}
		}
	}()

	return breakerMeta
}<commit_msg>Created circuit breaker<commit_after>package main

import (
	"github.com/rubyist/circuitbreaker"
	"time"
	log "github.com/Sirupsen/logrus"
)

type ExtendedCircuitBreakerMeta struct {
	CircuitBreakerMeta
	CB *circuit.Breaker
}

func NewCircuitBreaker(apiSpec *APIDefinition) ExtendedCircuitBreakerMeta {
	breakerMeta := ExtendedCircuitBreakerMeta{CircuitBreakerMeta: apiSpec.CircuitBreaker}
	breakerMeta.CB = circuit.NewRateBreaker(apiSpec.CircuitBreaker.ThresholdPercent, apiSpec.CircuitBreaker.Samples)

	events := breakerMeta.CB.Subscribe()

	go func() {
		path := apiSpec.Proxy.ListenPath
		timerActive := false
		for {
			e := <-events
			switch e {
			case circuit.BreakerTripped:
				log.Warning("[PROXY] [CIRCUIT BREKER] Breaker tripped for path: ", path)
				log.Debug("Breaker tripped: ", e)

				// Start a timer function
				if !timerActive {
					go func(timeout int, breaker *circuit.Breaker) {
						log.Debug("-- Sleeping for (s): ", timeout)
						time.Sleep(time.Duration(timeout) * time.Second)
						log.Debug("-- Resetting breaker")
						breaker.Reset()
						timerActive = false
					}(apiSpec.CircuitBreaker.ReturnToServiceAfter, breakerMeta.CB)
					timerActive = true
				}
			}
		}
	}()

	return breakerMeta
}
<|endoftext|>


<commit_before>// Package releases has information about the Mattermost platform version archive. If you ask
// politely, it will give you a list of currently supported Team and Enterprise versions.
//
// This is the most fragile part of mattercheck, because it relies heavily on the structure
// of an external HTML document.
package releases

import (
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/blang/semver"
	"github.com/dmke/mattercheck/version"
	"gopkg.in/xmlpath.v2"
)

// TODO: a JSON feed would be nice (https://github.com/mattermost/docs/issues/1190#issuecomment-302162095)
const releasesURL = "https://docs.mattermost.com/administration/version-archive.html"

var (
	absEnt  = xmlpath.MustCompile(`//div[@id='mattermost-enterprise-edition']/dl/dt`)
	absTeam = xmlpath.MustCompile(`//div[@id='mattermost-team-edition-server-archive']/dl/dt`)

	relChangeLog = xmlpath.MustCompile(`./a[1]/@href`)
	relDownload  = xmlpath.MustCompile(`./a[2]/@href`)
	relChecksum  = xmlpath.MustCompile(`./following-sibling::dd/ul/li[2]/code/span[@class="pre"]`)
)

var baseURL = func() *url.URL {
	base, err := url.Parse(releasesURL)
	if err != nil {
		log.Fatalf("cannot parse release URL (%s): %v", releasesURL, err)
	}
	return base
}()

// Archive allows you to compare a given version with all supported versions.
type Archive struct {
	ent, team *Release
}

// UpdateCandidate returns the newest known version (compared to the given version). It returns
// nil, if there is no newer version found.
func (a *Archive) UpdateCandidate(v *version.Version) *Release {
	var ref *Release
	if v.Enterprise {
		ref = a.ent
	} else {
		ref = a.team
	}

	if ref != nil && ref.Version.GT(*v.Version) {
		return ref
	}
	return nil
}

// LatestReleases return the latest enterprise and team version from the
// archive.
func (a *Archive) LatestReleases() (ent, team *Release) {
	return a.ent, a.team
}

// A Release contains information about a specific release entry found on
// https://docs.mattermost.com/administration/version-archive.html
type Release struct {
	Version   *version.Version
	ChangeLog string // URL to change log
	Download  string // download URL for Linux 64bit tar.gz
	Checksum  string // SHA256 checksum hash
}

// FetchSupported extract version information from the Mattermost version archive. Only supported
// versions (i.e. supported by Mattermost, Inc.) will be taken into the result set.
func FetchSupported() (*Archive, error) {
	doc, err := get()
	if err != nil {
		return nil, err
	}

	return &Archive{
		ent:  findLatestRelease(absEnt, doc),
		team: findLatestRelease(absTeam, doc),
	}, nil
}

func findLatestRelease(path *xmlpath.Path, root *xmlpath.Node) (release *Release) {
	max := &semver.Version{}

	iter := path.Iter(root)
	for iter.Next() {
		node := iter.Node()
		v, err := version.ExtractFromBytes(node.Bytes(), path == absEnt)
		if err != nil {
			// TODO: return error? verbose log?
			continue
		}

		if v.LTE(*max) {
			continue
		}
		max = v.Version

		r := &Release{
			Version:   v,
			Download:  "-",
			ChangeLog: "-",
			Checksum:  "-",
		}
		if s, ok := relDownload.String(node); ok {
			if u, err := absoluteURL(s); err == nil {
				r.Download = u
			}
		}
		if s, ok := relChangeLog.String(node); ok {
			u, err := url.Parse(s)
			if err != nil {
				r.ChangeLog = s
			} else {
				r.ChangeLog = baseURL.ResolveReference(u).String()
			}
		}
		if s, ok := relChecksum.String(node); ok {
			r.Checksum = "sha256:" + s
		}
		release = r
	}
	return release
}

// get can be replaced in tests
var get = func() (*xmlpath.Node, error) {
	req, err := http.NewRequest(http.MethodGet, releasesURL, nil)
	if err != nil {
		return nil, err
	}

	cli := http.Client{Timeout: 5 * time.Second}
	res, err := cli.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return xmlpath.ParseHTML(res.Body)
}

func absoluteURL(path string) (string, error) {
	u, err := url.Parse(path)
	if err != nil {
		return "", err
	}
	return baseURL.ResolveReference(u).String(), nil
}
<commit_msg>fix checksum retrieval<commit_after>// Package releases has information about the Mattermost platform version archive. If you ask
// politely, it will give you a list of currently supported Team and Enterprise versions.
//
// This is the most fragile part of mattercheck, because it relies heavily on the structure
// of an external HTML document.
package releases

import (
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/blang/semver"
	"github.com/dmke/mattercheck/version"
	"gopkg.in/xmlpath.v2"
)

// TODO: a JSON feed would be nice (https://github.com/mattermost/docs/issues/1190#issuecomment-302162095)
const releasesURL = "https://docs.mattermost.com/administration/version-archive.html"

var (
	absEnt  = xmlpath.MustCompile(`//div[@id='mattermost-enterprise-edition']/dl/dt`)
	absTeam = xmlpath.MustCompile(`//div[@id='mattermost-team-edition-server-archive']/dl/dt`)

	relChangeLog = xmlpath.MustCompile(`./a[1]/@href`)
	relDownload  = xmlpath.MustCompile(`./a[2]/@href`)
	relChecksum  = xmlpath.MustCompile(`./following-sibling::dd/ul/li[2]/p/code/span[@class="pre"]`)
)

var baseURL = func() *url.URL {
	base, err := url.Parse(releasesURL)
	if err != nil {
		log.Fatalf("cannot parse release URL (%s): %v", releasesURL, err)
	}
	return base
}()

// Archive allows you to compare a given version with all supported versions.
type Archive struct {
	ent, team *Release
}

// UpdateCandidate returns the newest known version (compared to the given version). It returns
// nil, if there is no newer version found.
func (a *Archive) UpdateCandidate(v *version.Version) *Release {
	var ref *Release
	if v.Enterprise {
		ref = a.ent
	} else {
		ref = a.team
	}

	if ref != nil && ref.Version.GT(*v.Version) {
		return ref
	}
	return nil
}

// LatestReleases return the latest enterprise and team version from the
// archive.
func (a *Archive) LatestReleases() (ent, team *Release) {
	return a.ent, a.team
}

// A Release contains information about a specific release entry found on
// https://docs.mattermost.com/administration/version-archive.html
type Release struct {
	Version   *version.Version
	ChangeLog string // URL to change log
	Download  string // download URL for Linux 64bit tar.gz
	Checksum  string // SHA256 checksum hash
}

// FetchSupported extract version information from the Mattermost version archive. Only supported
// versions (i.e. supported by Mattermost, Inc.) will be taken into the result set.
func FetchSupported() (*Archive, error) {
	doc, err := get()
	if err != nil {
		return nil, err
	}

	return &Archive{
		ent:  findLatestRelease(absEnt, doc),
		team: findLatestRelease(absTeam, doc),
	}, nil
}

func findLatestRelease(path *xmlpath.Path, root *xmlpath.Node) (release *Release) {
	max := &semver.Version{}

	iter := path.Iter(root)
	for iter.Next() {
		node := iter.Node()
		v, err := version.ExtractFromBytes(node.Bytes(), path == absEnt)
		if err != nil {
			// TODO: return error? verbose log?
			continue
		}

		if v.LTE(*max) {
			continue
		}
		max = v.Version

		r := &Release{
			Version:   v,
			Download:  "-",
			ChangeLog: "-",
			Checksum:  "-",
		}
		if s, ok := relDownload.String(node); ok {
			if u, err := absoluteURL(s); err == nil {
				r.Download = u
			}
		}
		if s, ok := relChangeLog.String(node); ok {
			u, err := url.Parse(s)
			if err != nil {
				r.ChangeLog = s
			} else {
				r.ChangeLog = baseURL.ResolveReference(u).String()
			}
		}
		if s, ok := relChecksum.String(node); ok {
			r.Checksum = "sha256:" + s
		}
		release = r
	}
	return release
}

// get can be replaced in tests
var get = func() (*xmlpath.Node, error) {
	req, err := http.NewRequest(http.MethodGet, releasesURL, nil)
	if err != nil {
		return nil, err
	}

	cli := http.Client{Timeout: 5 * time.Second}
	res, err := cli.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return xmlpath.ParseHTML(res.Body)
}

func absoluteURL(path string) (string, error) {
	u, err := url.Parse(path)
	if err != nil {
		return "", err
	}
	return baseURL.ResolveReference(u).String(), nil
}
<|endoftext|>


<commit_before>package containerd

import (
	"context"
	"io"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/images/archive"
	"github.com/containerd/containerd/platforms"
	"github.com/docker/distribution/reference"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// ExportImage exports a list of images to the given output stream. The
// exported images are archived into a tar when written to the output
// stream. All images with the given tag and all versions containing
// the same tag are exported. names is the set of tags to export, and
// outStream is the writer which the images are written to.
//
// TODO(thaJeztah): produce JSON stream progress response and image events; see https://github.com/moby/moby/issues/43910
func (i *ImageService) ExportImage(ctx context.Context, names []string, outStream io.Writer) error {
	opts := []archive.ExportOpt{
		archive.WithPlatform(platforms.Ordered(platforms.DefaultSpec())),
		archive.WithSkipNonDistributableBlobs(),
	}
	is := i.client.ImageService()
	for _, imageRef := range names {
		named, err := reference.ParseDockerRef(imageRef)
		if err != nil {
			return err
		}
		opts = append(opts, archive.WithImage(is, named.String()))
	}
	return i.client.Export(ctx, outStream, opts...)
}

// LoadImage uploads a set of images into the repository. This is the
// complement of ExportImage.  The input stream is an uncompressed tar
// ball containing images and metadata.
//
// TODO(thaJeztah): produce JSON stream progress response and image events; see https://github.com/moby/moby/issues/43910
func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outStream io.Writer, quiet bool) error {
	platform := platforms.DefaultStrict()
	imgs, err := i.client.Import(ctx, inTar, containerd.WithImportPlatform(platform))

	if err != nil {
		// TODO(thaJeztah): remove this log or change to debug once we can; see https://github.com/moby/moby/pull/43822#discussion_r937502405
		logrus.WithError(err).Warn("failed to import image to containerd")
		return errors.Wrap(err, "failed to import image")
	}

	for _, img := range imgs {
		platformImg := containerd.NewImageWithPlatform(i.client, img, platform)

		unpacked, err := platformImg.IsUnpacked(ctx, containerd.DefaultSnapshotter)
		if err != nil {
			// TODO(thaJeztah): remove this log or change to debug once we can; see https://github.com/moby/moby/pull/43822#discussion_r937502405
			logrus.WithError(err).WithField("image", img.Name).Debug("failed to check if image is unpacked")
			continue
		}

		if !unpacked {
			err := platformImg.Unpack(ctx, containerd.DefaultSnapshotter)
			if err != nil {
				// TODO(thaJeztah): remove this log or change to debug once we can; see https://github.com/moby/moby/pull/43822#discussion_r937502405
				logrus.WithError(err).WithField("image", img.Name).Warn("failed to unpack image")
				return errors.Wrap(err, "failed to unpack image")
			}
		}
	}
	return nil
}
<commit_msg>containerd/load: Load all platforms<commit_after>package containerd

import (
	"context"
	"io"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/images/archive"
	"github.com/containerd/containerd/platforms"
	"github.com/docker/distribution/reference"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// ExportImage exports a list of images to the given output stream. The
// exported images are archived into a tar when written to the output
// stream. All images with the given tag and all versions containing
// the same tag are exported. names is the set of tags to export, and
// outStream is the writer which the images are written to.
//
// TODO(thaJeztah): produce JSON stream progress response and image events; see https://github.com/moby/moby/issues/43910
func (i *ImageService) ExportImage(ctx context.Context, names []string, outStream io.Writer) error {
	opts := []archive.ExportOpt{
		archive.WithPlatform(platforms.Ordered(platforms.DefaultSpec())),
		archive.WithSkipNonDistributableBlobs(),
	}
	is := i.client.ImageService()
	for _, imageRef := range names {
		named, err := reference.ParseDockerRef(imageRef)
		if err != nil {
			return err
		}
		opts = append(opts, archive.WithImage(is, named.String()))
	}
	return i.client.Export(ctx, outStream, opts...)
}

// LoadImage uploads a set of images into the repository. This is the
// complement of ExportImage.  The input stream is an uncompressed tar
// ball containing images and metadata.
//
// TODO(thaJeztah): produce JSON stream progress response and image events; see https://github.com/moby/moby/issues/43910
func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outStream io.Writer, quiet bool) error {
	platform := platforms.All
	imgs, err := i.client.Import(ctx, inTar, containerd.WithImportPlatform(platform))

	if err != nil {
		// TODO(thaJeztah): remove this log or change to debug once we can; see https://github.com/moby/moby/pull/43822#discussion_r937502405
		logrus.WithError(err).Warn("failed to import image to containerd")
		return errors.Wrap(err, "failed to import image")
	}

	for _, img := range imgs {
		platformImg := containerd.NewImageWithPlatform(i.client, img, platform)

		unpacked, err := platformImg.IsUnpacked(ctx, containerd.DefaultSnapshotter)
		if err != nil {
			// TODO(thaJeztah): remove this log or change to debug once we can; see https://github.com/moby/moby/pull/43822#discussion_r937502405
			logrus.WithError(err).WithField("image", img.Name).Debug("failed to check if image is unpacked")
			continue
		}

		if !unpacked {
			err := platformImg.Unpack(ctx, containerd.DefaultSnapshotter)
			if err != nil {
				// TODO(thaJeztah): remove this log or change to debug once we can; see https://github.com/moby/moby/pull/43822#discussion_r937502405
				logrus.WithError(err).WithField("image", img.Name).Warn("failed to unpack image")
				return errors.Wrap(err, "failed to unpack image")
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package remote

import (
	"os"
	"testing"

	"github.com/hashicorp/terraform/terraform"
)

func TestAtlasRemote_Interface(t *testing.T) {
	var client interface{} = &AtlasRemoteClient{}
	if _, ok := client.(RemoteClient); !ok {
		t.Fatalf("does not implement interface")
	}
}

func checkAtlas(t *testing.T) {
	if os.Getenv("ATLAS_TOKEN") == "" {
		t.SkipNow()
	}
}

func TestAtlasRemote_Validate(t *testing.T) {
	conf := map[string]string{}
	if _, err := NewAtlasRemoteClient(conf); err == nil {
		t.Fatalf("expect error")
	}

	conf["access_token"] = "test"
	conf["name"] = "hashicorp/test-state"
	if _, err := NewAtlasRemoteClient(conf); err != nil {
		t.Fatalf("err: %v", err)
	}
}

func TestAtlasRemote(t *testing.T) {
	checkAtlas(t)
	remote := &terraform.RemoteState{
		Type: "atlas",
		Config: map[string]string{
			"access_token": os.Getenv("ATLAS_TOKEN"),
			"name":         "hashicorp/test-remote-state",
		},
	}
	r, err := NewClientByState(remote)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	// Get a valid input
	inp, err := blankState(remote)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	// Delete the state, should be none
	err = r.DeleteState()
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Ensure no state
	payload, err := r.GetState()
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	if payload != nil {
		t.Fatalf("unexpected payload")
	}

	// Put the state
	err = r.PutState(inp, false)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Get it back
	payload, err = r.GetState()
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	if payload == nil {
		t.Fatalf("unexpected payload")
	}

	// Check the payload
	//if !bytes.Equal(payload.MD5, hash) {
	//    t.Fatalf("bad hash: %x %x", payload.MD5, hash)
	//}
	//if !bytes.Equal(payload.State, inp) {
	//    t.Errorf("inp: %s", inp)
	//    t.Fatalf("bad response: %s", payload.State)
	//}

	// Delete the state
	err = r.DeleteState()
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Should be gone
	payload, err = r.GetState()
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	if payload != nil {
		t.Fatalf("unexpected payload")
	}
}
<commit_msg>remote: Re-assert the MD5's match<commit_after>package remote

import (
	"bytes"
	"crypto/md5"
	"os"
	"testing"

	"github.com/hashicorp/terraform/terraform"
)

func TestAtlasRemote_Interface(t *testing.T) {
	var client interface{} = &AtlasRemoteClient{}
	if _, ok := client.(RemoteClient); !ok {
		t.Fatalf("does not implement interface")
	}
}

func checkAtlas(t *testing.T) {
	if os.Getenv("ATLAS_TOKEN") == "" {
		t.SkipNow()
	}
}

func TestAtlasRemote_Validate(t *testing.T) {
	conf := map[string]string{}
	if _, err := NewAtlasRemoteClient(conf); err == nil {
		t.Fatalf("expect error")
	}

	conf["access_token"] = "test"
	conf["name"] = "hashicorp/test-state"
	if _, err := NewAtlasRemoteClient(conf); err != nil {
		t.Fatalf("err: %v", err)
	}
}

func TestAtlasRemote(t *testing.T) {
	checkAtlas(t)
	remote := &terraform.RemoteState{
		Type: "atlas",
		Config: map[string]string{
			"access_token": os.Getenv("ATLAS_TOKEN"),
			"name":         "hashicorp/test-remote-state",
		},
	}
	r, err := NewClientByState(remote)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	// Get a valid input
	inp, err := blankState(remote)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	inpMD5 := md5.Sum(inp)
	hash := inpMD5[:16]

	// Delete the state, should be none
	err = r.DeleteState()
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Ensure no state
	payload, err := r.GetState()
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	if payload != nil {
		t.Fatalf("unexpected payload")
	}

	// Put the state
	err = r.PutState(inp, false)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Get it back
	payload, err = r.GetState()
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	if payload == nil {
		t.Fatalf("unexpected payload")
	}

	// Check the payload
	if !bytes.Equal(payload.MD5, hash) {
		t.Fatalf("bad hash: %x %x", payload.MD5, hash)
	}
	if !bytes.Equal(payload.State, inp) {
		t.Errorf("inp: %s", inp)
		t.Fatalf("bad response: %s", payload.State)
	}

	// Delete the state
	err = r.DeleteState()
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Should be gone
	payload, err = r.GetState()
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	if payload != nil {
		t.Fatalf("unexpected payload")
	}
}
<|endoftext|>


<commit_before>/*
This package is just a collection of use-case for the various aspects of the RIPE API.
Consider this both as an example on how to use the API and a testing tool for the API wrapper.
*/
package main

import (
	"github.com/keltia/ripe-atlas"
	"github.com/urfave/cli"
	"log"
	"os"
	"sort"
)

var (
	// flags
	fWant4 bool
	fWant6 bool

	// True by default
	fWantMine = true

	fAllProbes       bool
	fAllMeasurements bool

	fAsn         string
	fCountry     string
	fFieldList   string
	fFormat      string
	fOptFields   string
	fProtocol    string
	fSortOrder   string
	fMeasureType string

	fHTTPMethod  string
	fUserAgent   string
	fHTTPVersion string

	fBitCD         bool
	fDisableDNSSEC bool

	fDebug      bool
	fVerbose    bool
	fWantAnchor bool

	fMaxHops    int
	fPacketSize int

	mycnf *Config

	cliCommands []cli.Command

	client *atlas.Client
)

const (
	atlasVersion = "0.11"
	MyName       = "ripe-atlas"

	// WantBoth is the way to ask for both IPv4 & IPv6.
	WantBoth = "64"

	// Want4 only 4
	Want4 = "4"
	// Want6 only 6
	Want6 = "6"
)

// -4 & -6 are special, if neither is specified, then we turn both as true
// Check a few other things while we are here
func finalcheck(c *cli.Context) error {
	var err error

	// Load main configuration
	mycnf, err = LoadConfig("")
	if err != nil {
		if fVerbose {
			log.Printf("No configuration file found.")
		}
	}

	// Logical
	if fDebug {
		fVerbose = true
		log.Printf("config: %#v", mycnf)
	}

	// Various messages
	if fVerbose {
		if mycnf.APIKey != "" {
			log.Printf("Found API key!")
		} else {
			log.Printf("No API key!")
		}

		if mycnf.DefaultProbe != 0 {
			log.Printf("Found default probe: %d\n", mycnf.DefaultProbe)
		}
	}

	// Check whether we have proxy authentication (from a separate config file)
	auth, err := setupProxyAuth()
	if err != nil {
		if fVerbose {
			log.Printf("Invalid or no proxy auth credentials")
		}
	}

	// Wondering whether to move to the Functional options pattern
	// cf. https://dave.cheney.net/2016/11/13/do-not-fear-first-class-functions
	client, err = atlas.NewClient(atlas.Config{
		APIKey:       mycnf.APIKey,
		DefaultProbe: mycnf.DefaultProbe,
		PoolSize:     mycnf.PoolSize,
		ProxyAuth:    auth,
		Verbose:      fVerbose,
	})

	// No need to continue if this fails
	if err != nil {
		log.Fatalf("Error creating the Atlas client: %v", err)
	}

	if fWant4 {
		mycnf.WantAF = Want4
	}

	if fWant6 {
		mycnf.WantAF = Want6
	}

	// Both are fine
	if fWant4 && fWant6 {
		mycnf.WantAF = WantBoth
	}

	// So is neither — common case
	if !fWant4 && !fWant6 {
		mycnf.WantAF = WantBoth
	}

	return nil
}

// main is the starting point (and everything)
func main() {
	cli.VersionFlag = cli.BoolFlag{Name: "version, V"}

	cli.VersionPrinter = func(c *cli.Context) {
		log.Printf("API wrapper: %s Atlas API: %s\n", c.App.Version, atlas.GetVersion())
	}

	app := cli.NewApp()
	app.Name = "atlas"
	app.Usage = "RIPE Atlas CLI interface"
	app.Author = "Ollivier Robert <roberto@keltia.net>"
	app.Version = atlasVersion
	//app.HideVersion = true

	// General flags
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "format,f",
			Usage:       "specify output format",
			Destination: &fFormat,
		},
		cli.BoolFlag{
			Name:        "debug,D",
			Usage:       "debug mode",
			Destination: &fDebug,
		},
		cli.BoolFlag{
			Name:        "verbose,v",
			Usage:       "verbose mode",
			Destination: &fVerbose,
		},
		cli.StringFlag{
			Name:        "fields,F",
			Usage:       "specify which fields are wanted",
			Destination: &fFieldList,
		},
		cli.StringFlag{
			Name:        "opt-fields,O",
			Usage:       "specify which optional fields are wanted",
			Destination: &fOptFields,
		},
		cli.StringFlag{
			Name:        "sort,S",
			Usage:       "sort results",
			Destination: &fSortOrder,
		},
		cli.BoolFlag{
			Name:        "6, ipv6",
			Usage:       "Only IPv6",
			Destination: &fWant6,
		},
		cli.BoolFlag{
			Name:        "4, ipv4",
			Usage:       "Only IPv4",
			Destination: &fWant4,
		},
	}

	// Ensure -4 & -6 are treated properly & initialization is done
	app.Before = finalcheck

	sort.Sort(ByAlphabet(cliCommands))
	app.Commands = cliCommands
	app.Run(os.Args)
}
<commit_msg>Add new -I option for setting "include".<commit_after>/*
This package is just a collection of use-case for the various aspects of the RIPE API.
Consider this both as an example on how to use the API and a testing tool for the API wrapper.
*/
package main

import (
	"github.com/keltia/ripe-atlas"
	"github.com/urfave/cli"
	"log"
	"os"
	"sort"
)

var (
	// flags
	fWant4 bool
	fWant6 bool

	// True by default
	fWantMine = true

	fAllProbes       bool
	fAllMeasurements bool

	fAsn         string
	fCountry     string
	fFieldList   string
	fFormat      string
	fInclude     string
	fOptFields   string
	fProtocol    string
	fSortOrder   string
	fMeasureType string

	fHTTPMethod  string
	fUserAgent   string
	fHTTPVersion string

	fBitCD         bool
	fDisableDNSSEC bool

	fDebug      bool
	fVerbose    bool
	fWantAnchor bool

	fMaxHops    int
	fPacketSize int

	mycnf *Config

	cliCommands []cli.Command

	client *atlas.Client
)

const (
	atlasVersion = "0.11"
	MyName       = "ripe-atlas"

	// WantBoth is the way to ask for both IPv4 & IPv6.
	WantBoth = "64"

	// Want4 only 4
	Want4 = "4"
	// Want6 only 6
	Want6 = "6"
)

// -4 & -6 are special, if neither is specified, then we turn both as true
// Check a few other things while we are here
func finalcheck(c *cli.Context) error {
	var err error

	// Load main configuration
	mycnf, err = LoadConfig("")
	if err != nil {
		if fVerbose {
			log.Printf("No configuration file found.")
		}
	}

	// Logical
	if fDebug {
		fVerbose = true
		log.Printf("config: %#v", mycnf)
	}

	// Various messages
	if fVerbose {
		if mycnf.APIKey != "" {
			log.Printf("Found API key!")
		} else {
			log.Printf("No API key!")
		}

		if mycnf.DefaultProbe != 0 {
			log.Printf("Found default probe: %d\n", mycnf.DefaultProbe)
		}
	}

	// Check whether we have proxy authentication (from a separate config file)
	auth, err := setupProxyAuth()
	if err != nil {
		if fVerbose {
			log.Printf("Invalid or no proxy auth credentials")
		}
	}

	// Wondering whether to move to the Functional options pattern
	// cf. https://dave.cheney.net/2016/11/13/do-not-fear-first-class-functions
	client, err = atlas.NewClient(atlas.Config{
		APIKey:       mycnf.APIKey,
		DefaultProbe: mycnf.DefaultProbe,
		PoolSize:     mycnf.PoolSize,
		ProxyAuth:    auth,
		Verbose:      fVerbose,
	})

	// No need to continue if this fails
	if err != nil {
		log.Fatalf("Error creating the Atlas client: %v", err)
	}

	if fWant4 {
		mycnf.WantAF = Want4
	}

	if fWant6 {
		mycnf.WantAF = Want6
	}

	// Both are fine
	if fWant4 && fWant6 {
		mycnf.WantAF = WantBoth
	}

	// So is neither — common case
	if !fWant4 && !fWant6 {
		mycnf.WantAF = WantBoth
	}

	return nil
}

// main is the starting point (and everything)
func main() {
	cli.VersionFlag = cli.BoolFlag{Name: "version, V"}

	cli.VersionPrinter = func(c *cli.Context) {
		log.Printf("API wrapper: %s Atlas API: %s\n", c.App.Version, atlas.GetVersion())
	}

	app := cli.NewApp()
	app.Name = "atlas"
	app.Usage = "RIPE Atlas CLI interface"
	app.Author = "Ollivier Robert <roberto@keltia.net>"
	app.Version = atlasVersion
	//app.HideVersion = true

	// General flags
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "format,f",
			Usage:       "specify output format",
			Destination: &fFormat,
		},
		cli.BoolFlag{
			Name:        "debug,D",
			Usage:       "debug mode",
			Destination: &fDebug,
		},
		cli.BoolFlag{
			Name:        "verbose,v",
			Usage:       "verbose mode",
			Destination: &fVerbose,
		},
		cli.StringFlag{
			Name:        "fields,F",
			Usage:       "specify which fields are wanted",
			Destination: &fFieldList,
		},
		cli.StringFlag{
			Name:        "include,I",
			Usage:       "specify whether objects should be expanded",
			Destination: &fInclude,
		},
		cli.StringFlag{
			Name:        "opt-fields,O",
			Usage:       "specify which optional fields are wanted",
			Destination: &fOptFields,
		},
		cli.StringFlag{
			Name:        "sort,S",
			Usage:       "sort results",
			Destination: &fSortOrder,
		},
		cli.BoolFlag{
			Name:        "6, ipv6",
			Usage:       "Only IPv6",
			Destination: &fWant6,
		},
		cli.BoolFlag{
			Name:        "4, ipv4",
			Usage:       "Only IPv4",
			Destination: &fWant4,
		},
	}

	// Ensure -4 & -6 are treated properly & initialization is done
	app.Before = finalcheck

	sort.Sort(ByAlphabet(cliCommands))
	app.Commands = cliCommands
	app.Run(os.Args)
}
<|endoftext|>