<commit_before>package vfs

import (
	"context"
	"os"
	"testing"

	"github.com/rclone/rclone/fstest"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestCaseSensitivity(t *testing.T) {
	r := fstest.NewRun(t)
	defer r.Finalise()

	// Create test files
	ctx := context.Background()
	file1 := r.WriteObject(ctx, "FiLeA", "data1", t1)
	file2 := r.WriteObject(ctx, "FiLeB", "data2", t2)
	fstest.CheckItems(t, r.Fremote, file1, file2)

	// Create file3 with name differing from file2 name only by case.
	// On a case-Sensitive remote this will be a separate file.
	// On a case-INsensitive remote this file will either not exist
	// or overwrite file2 depending on how file system diverges.
	file3 := r.WriteObject(ctx, "FilEb", "data3", t3)

	// Create a case-Sensitive and case-INsensitive VFS
	optCS := DefaultOpt
	optCS.CaseInsensitive = false
	vfsCS := New(r.Fremote, &optCS)

	optCI := DefaultOpt
	optCI.CaseInsensitive = true
	vfsCI := New(r.Fremote, &optCI)

	// Run basic checks that must pass on VFS of any type.
	assertFileDataVFS(t, vfsCI, "FiLeA", "data1")
	assertFileDataVFS(t, vfsCS, "FiLeA", "data1")

	// Detect case sensitivity of the underlying remote.
	remoteIsOK := true
	if !checkFileDataVFS(t, vfsCS, "FiLeA", "data1") {
		remoteIsOK = false
	}
	if !checkFileDataVFS(t, vfsCS, "FiLeB", "data2") {
		remoteIsOK = false
	}
	if !checkFileDataVFS(t, vfsCS, "FilEb", "data3") {
		remoteIsOK = false
	}

	// The remaining test is only meaningful on a case-Sensitive file system.
	if !remoteIsOK {
		t.Logf("SKIP: TestCaseSensitivity - remote is not fully case-sensitive")
		return
	}

	// Continue with test as the underlying remote is fully case-Sensitive.
	fstest.CheckItems(t, r.Fremote, file1, file2, file3)

	// See how VFS handles case-INsensitive flag
	assertFileDataVFS(t, vfsCI, "FiLeA", "data1")
	assertFileDataVFS(t, vfsCI, "fileA", "data1")
	assertFileDataVFS(t, vfsCI, "filea", "data1")
	assertFileDataVFS(t, vfsCI, "FILEA", "data1")

	assertFileDataVFS(t, vfsCI, "FiLeB", "data2")
	assertFileDataVFS(t, vfsCI, "FilEb", "data3")

	fd, err := vfsCI.OpenFile("fileb", os.O_RDONLY, 0777)
	assert.Nil(t, fd)
	assert.Error(t, err)
	assert.NotEqual(t, err, ENOENT)

	fd, err = vfsCI.OpenFile("FILEB", os.O_RDONLY, 0777)
	assert.Nil(t, fd)
	assert.Error(t, err)
	assert.NotEqual(t, err, ENOENT)

	// Run the same set of checks with case-Sensitive VFS, for comparison.
	assertFileDataVFS(t, vfsCS, "FiLeA", "data1")

	assertFileAbsentVFS(t, vfsCS, "fileA")
	assertFileAbsentVFS(t, vfsCS, "filea")
	assertFileAbsentVFS(t, vfsCS, "FILEA")

	assertFileDataVFS(t, vfsCS, "FiLeB", "data2")
	assertFileDataVFS(t, vfsCS, "FilEb", "data3")

	assertFileAbsentVFS(t, vfsCS, "fileb")
	assertFileAbsentVFS(t, vfsCS, "FILEB")
}

func checkFileDataVFS(t *testing.T, vfs *VFS, name string, expect string) bool {
	fd, err := vfs.OpenFile(name, os.O_RDONLY, 0777)
	if fd == nil || err != nil {
		return false
	}
	defer func() {
		// File must be closed - otherwise Run.cleanUp() will fail on Windows.
		_ = fd.Close()
	}()

	fh, ok := fd.(*ReadFileHandle)
	if !ok {
		return false
	}

	size := len(expect)
	buf := make([]byte, size)
	num, err := fh.Read(buf)
	if err != nil || num != size {
		return false
	}

	return string(buf) == expect
}

func assertFileDataVFS(t *testing.T, vfs *VFS, name string, expect string) {
	fd, errOpen := vfs.OpenFile(name, os.O_RDONLY, 0777)
	assert.NotNil(t, fd)
	assert.NoError(t, errOpen)

	defer func() {
		// File must be closed - otherwise Run.cleanUp() will fail on Windows.
		if errOpen == nil && fd != nil {
			_ = fd.Close()
		}
	}()

	fh, ok := fd.(*ReadFileHandle)
	require.True(t, ok)

	size := len(expect)
	buf := make([]byte, size)
	numRead, errRead := fh.Read(buf)
	assert.NoError(t, errRead)
	assert.Equal(t, numRead, size)

	assert.Equal(t, string(buf), expect)
}

func assertFileAbsentVFS(t *testing.T, vfs *VFS, name string) {
	fd, err := vfs.OpenFile(name, os.O_RDONLY, 0777)
	defer func() {
		// File must be closed - otherwise Run.cleanUp() will fail on Windows.
		if err == nil && fd != nil {
			_ = fd.Close()
		}
	}()
	assert.Nil(t, fd)
	assert.Error(t, err)
	assert.Equal(t, err, ENOENT)
}
<commit_msg>vfs: skip TestCaseSensitivity on case insensitive backends<commit_after>package vfs

import (
	"context"
	"os"
	"testing"

	"github.com/rclone/rclone/fstest"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestCaseSensitivity(t *testing.T) {
	r := fstest.NewRun(t)
	defer r.Finalise()

	if r.Fremote.Features().CaseInsensitive {
		t.Skip("Can't test case sensitivity - this remote is officially not case-sensitive")
	}

	// Create test files
	ctx := context.Background()
	file1 := r.WriteObject(ctx, "FiLeA", "data1", t1)
	file2 := r.WriteObject(ctx, "FiLeB", "data2", t2)
	fstest.CheckItems(t, r.Fremote, file1, file2)

	// Create file3 with name differing from file2 name only by case.
	// On a case-Sensitive remote this will be a separate file.
	// On a case-INsensitive remote this file will either not exist
	// or overwrite file2 depending on how file system diverges.
	// On a box.com remote this step will even fail.
	file3 := r.WriteObject(ctx, "FilEb", "data3", t3)

	// Create a case-Sensitive and case-INsensitive VFS
	optCS := DefaultOpt
	optCS.CaseInsensitive = false
	vfsCS := New(r.Fremote, &optCS)

	optCI := DefaultOpt
	optCI.CaseInsensitive = true
	vfsCI := New(r.Fremote, &optCI)

	// Run basic checks that must pass on VFS of any type.
	assertFileDataVFS(t, vfsCI, "FiLeA", "data1")
	assertFileDataVFS(t, vfsCS, "FiLeA", "data1")

	// Detect case sensitivity of the underlying remote.
	remoteIsOK := true
	if !checkFileDataVFS(t, vfsCS, "FiLeA", "data1") {
		remoteIsOK = false
	}
	if !checkFileDataVFS(t, vfsCS, "FiLeB", "data2") {
		remoteIsOK = false
	}
	if !checkFileDataVFS(t, vfsCS, "FilEb", "data3") {
		remoteIsOK = false
	}

	// The remaining test is only meaningful on a case-Sensitive file system.
	if !remoteIsOK {
		t.Skip("Can't test case sensitivity - this remote doesn't comply as case-sensitive")
	}

	// Continue with test as the underlying remote is fully case-Sensitive.
	fstest.CheckItems(t, r.Fremote, file1, file2, file3)

	// See how VFS handles case-INsensitive flag
	assertFileDataVFS(t, vfsCI, "FiLeA", "data1")
	assertFileDataVFS(t, vfsCI, "fileA", "data1")
	assertFileDataVFS(t, vfsCI, "filea", "data1")
	assertFileDataVFS(t, vfsCI, "FILEA", "data1")

	assertFileDataVFS(t, vfsCI, "FiLeB", "data2")
	assertFileDataVFS(t, vfsCI, "FilEb", "data3")

	fd, err := vfsCI.OpenFile("fileb", os.O_RDONLY, 0777)
	assert.Nil(t, fd)
	assert.Error(t, err)
	assert.NotEqual(t, err, ENOENT)

	fd, err = vfsCI.OpenFile("FILEB", os.O_RDONLY, 0777)
	assert.Nil(t, fd)
	assert.Error(t, err)
	assert.NotEqual(t, err, ENOENT)

	// Run the same set of checks with case-Sensitive VFS, for comparison.
	assertFileDataVFS(t, vfsCS, "FiLeA", "data1")

	assertFileAbsentVFS(t, vfsCS, "fileA")
	assertFileAbsentVFS(t, vfsCS, "filea")
	assertFileAbsentVFS(t, vfsCS, "FILEA")

	assertFileDataVFS(t, vfsCS, "FiLeB", "data2")
	assertFileDataVFS(t, vfsCS, "FilEb", "data3")

	assertFileAbsentVFS(t, vfsCS, "fileb")
	assertFileAbsentVFS(t, vfsCS, "FILEB")
}

func checkFileDataVFS(t *testing.T, vfs *VFS, name string, expect string) bool {
	fd, err := vfs.OpenFile(name, os.O_RDONLY, 0777)
	if fd == nil || err != nil {
		return false
	}
	defer func() {
		// File must be closed - otherwise Run.cleanUp() will fail on Windows.
		_ = fd.Close()
	}()

	fh, ok := fd.(*ReadFileHandle)
	if !ok {
		return false
	}

	size := len(expect)
	buf := make([]byte, size)
	num, err := fh.Read(buf)
	if err != nil || num != size {
		return false
	}

	return string(buf) == expect
}

func assertFileDataVFS(t *testing.T, vfs *VFS, name string, expect string) {
	fd, errOpen := vfs.OpenFile(name, os.O_RDONLY, 0777)
	assert.NotNil(t, fd)
	assert.NoError(t, errOpen)

	defer func() {
		// File must be closed - otherwise Run.cleanUp() will fail on Windows.
		if errOpen == nil && fd != nil {
			_ = fd.Close()
		}
	}()

	fh, ok := fd.(*ReadFileHandle)
	require.True(t, ok)

	size := len(expect)
	buf := make([]byte, size)
	numRead, errRead := fh.Read(buf)
	assert.NoError(t, errRead)
	assert.Equal(t, numRead, size)

	assert.Equal(t, string(buf), expect)
}

func assertFileAbsentVFS(t *testing.T, vfs *VFS, name string) {
	fd, err := vfs.OpenFile(name, os.O_RDONLY, 0777)
	defer func() {
		// File must be closed - otherwise Run.cleanUp() will fail on Windows.
		if err == nil && fd != nil {
			_ = fd.Close()
		}
	}()
	assert.Nil(t, fd)
	assert.Error(t, err)
	assert.Equal(t, err, ENOENT)
}
<|endoftext|>


<commit_before>package middlewares

import (
	"encoding/base64"
	"github.com/elithrar/simple-scrypt"
	"github.com/zenazn/goji/web"
	. "github.com/eywa/configs"
	. "github.com/eywa/models"
	. "github.com/eywa/utils"
	"net/http"
)

func AdminAuthenticator(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/login" || r.Method == "OPTIONS" {
			h.ServeHTTP(w, r)
		} else {
			if len(r.Header.Get("Authentication")) != 0 {
				tokenStr := r.Header.Get("Authentication")
				auth, err := DecryptAuthToken(tokenStr)
				if err != nil {
					Render.JSON(w, http.StatusUnauthorized, map[string]string{"error": err.Error()})
				} else {
					if auth.Username == Config().Security.Dashboard.Username {
						if asBytes, err := base64.URLEncoding.DecodeString(auth.TokenString); err != nil {
							Render.JSON(w, http.StatusUnauthorized, map[string]string{"error": err.Error()})
						} else {
							if scrypt.CompareHashAndPassword(asBytes, []byte(Config().Security.Dashboard.Password)) != nil {
								Render.JSON(w, http.StatusUnauthorized, map[string]string{"error": "invalid username or password"})
							} else {
								c.Env["auth_token"] = auth
								h.ServeHTTP(w, r)
							}
						}
					} else {
						Render.JSON(w, http.StatusUnauthorized, map[string]string{"error": "invalid username or password"})
					}
				}
			} else {
				Render.JSON(w, http.StatusUnauthorized, map[string]string{"error": "empty Authentication header"})
			}
		}
	}

	return http.HandlerFunc(fn)
}
<commit_msg>Bypass authentication check for attach websocket<commit_after>package middlewares

import (
	"encoding/base64"
	"github.com/elithrar/simple-scrypt"
	"github.com/zenazn/goji/web"
	. "github.com/eywa/configs"
	. "github.com/eywa/models"
	. "github.com/eywa/utils"
	"net/http"
	"regexp"
)

func AdminAuthenticator(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
	  // Since javascript only supports basic authentication, we'll bypass the authentication token
	  var attachUrl = regexp.MustCompile(`/channels/.*/devices/.*/attach`)

		if r.URL.Path == "/login" || r.Method == "OPTIONS" || attachUrl.MatchString(r.URL.Path) {
			h.ServeHTTP(w, r)
		} else {
			if len(r.Header.Get("Authentication")) != 0 {
				tokenStr := r.Header.Get("Authentication")
				auth, err := DecryptAuthToken(tokenStr)
				if err != nil {
					Render.JSON(w, http.StatusUnauthorized, map[string]string{"error": err.Error()})
				} else {
					if auth.Username == Config().Security.Dashboard.Username {
						if asBytes, err := base64.URLEncoding.DecodeString(auth.TokenString); err != nil {
							Render.JSON(w, http.StatusUnauthorized, map[string]string{"error": err.Error()})
						} else {
							if scrypt.CompareHashAndPassword(asBytes, []byte(Config().Security.Dashboard.Password)) != nil {
								Render.JSON(w, http.StatusUnauthorized, map[string]string{"error": "invalid username or password"})
							} else {
								c.Env["auth_token"] = auth
								h.ServeHTTP(w, r)
							}
						}
					} else {
						Render.JSON(w, http.StatusUnauthorized, map[string]string{"error": "invalid username or password"})
					}
				}
			} else {
				Render.JSON(w, http.StatusUnauthorized, map[string]string{"error": "empty Authentication header"})
			}
		}
	}

	return http.HandlerFunc(fn)
}
<|endoftext|>


<commit_before>package database

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"log"
	"reflect"
	"strings"

	"github.com/juju/errors"
	"golang.org/x/net/context"
)

// Query helps preparing and executing queries.
type Query struct {
	// Where conditions and replacement values
	conditions []string
	values     []interface{}

	limit int64
	order string
}

// NewQuery starts a new query to the database.
func NewQuery() *Query {
	return &Query{}
}

// Clone makes a copy of the query keeping all the internal state up to that moment.
func (q *Query) Clone() *Query {
	conditions := make([]string, len(q.conditions))
	for i := range conditions {
		conditions[i] = q.conditions[i]
	}
	values := make([]interface{}, len(q.values))
	for i := range values {
		values[i] = q.values[i]
	}

	return &Query{
		conditions: conditions,
		values:     values,
		limit:      q.limit,
		order:      q.order,
	}
}

// Where filters by a new column adding some placeholders if needed.
func (q *Query) Where(column string, args ...interface{}) *Query {
	q = q.Clone()

	nargs := strings.Count(column, "?")
	if len(args) != nargs {
		panic(fmt.Sprintf("expected %d parameters in the query and received %d", nargs, len(args)))
	}
	if !hasOperator(column) {
		panic(fmt.Sprintf("column does not have an operator: %s", column))
	}

	if strings.HasSuffix(column, "IN (?)") && len(args) == 1 && reflect.TypeOf(args[0]).Kind() == reflect.Slice {
		argValue := reflect.ValueOf(args[0])
		placeholders := make([]string, argValue.Len())
		for i := range placeholders {
			placeholders[i] = "?"
		}
		newColumn := fmt.Sprintf("IN (%s)", strings.Join(placeholders, ", "))

		q.conditions = append(q.conditions, strings.Replace(column, "IN (?)", newColumn, -1))

		for i := 0; i < argValue.Len(); i++ {
			q.values = append(q.values, argValue.Index(i).Interface())
		}
	} else {
		q.conditions = append(q.conditions, column)
		q.values = append(q.values, args...)
	}

	return q
}

// GetAll returns all the results that matchs the query putting it in the output slice.
func (q *Query) GetAll(ctx context.Context, output interface{}) error {
	outputValue := reflect.ValueOf(output)
	outputType := reflect.TypeOf(output)

	// Some sanity checks about the output value
	if outputValue.Kind() != reflect.Ptr || outputValue.Elem().Kind() != reflect.Slice {
		return errors.New("output should be a pointer to a slice")
	}
	sliceElemType := outputType.Elem().Elem()
	if sliceElemType.Kind() != reflect.Ptr || sliceElemType.Elem().Kind() != reflect.Struct {
		return errors.New("output should be a pointer to a slice of struct pointers")
	}

	// Build the table name
	tableName := getTableName(sliceElemType.Elem())

	// Get the list of field names
	fields, columns, err := getSerializableFields(sliceElemType.Elem())
	if err != nil {
		return errors.Trace(err)
	}

	// Prepare the query string
	query := fmt.Sprintf("SELECT %s FROM `%s`", columns, tableName)
	if len(q.conditions) > 0 {
		query = fmt.Sprintf("%s WHERE %s", query, strings.Join(q.conditions, " AND "))
	}
	if q.order != "" {
		query = fmt.Sprintf("%s ORDER BY %s", query, q.order)
	}

	// Run the query and fetch the rows
	conn := FromContext(ctx)
	rows, err := conn.DB.Query(query, q.values...)
	if err != nil {
		return errors.Annotate(err, query)
	}
	defer rows.Close()

	scan := reflect.ValueOf(rows.Scan)
	for rows.Next() {
		// Create a new element
		elem := reflect.New(sliceElemType.Elem())

		// Prepare space to save the bytes of serialized fields
		serializedFields := [][]byte{}
		for _, field := range fields {
			if field.json || field.gob {
				serializedFields = append(serializedFields, []byte{})
			}
		}

		// Prepare the pointers to all its fields
		pointers := []reflect.Value{}
		serializedIdx := 0
		for _, field := range fields {
			if field.json || field.gob {
				pointers = append(pointers, reflect.ValueOf(&serializedFields[serializedIdx]))
				serializedIdx++
			} else {
				pointers = append(pointers, elem.Elem().FieldByName(field.name).Addr())
			}
		}

		// Scan the row into the struct
		if err := scan.Call(pointers)[0]; !err.IsNil() {
			return errors.Trace(err.Interface().(error))
		}

		// Read serialized fields
		serializedIdx = 0
		for _, field := range fields {
			switch {
			case field.json && len(serializedFields[serializedIdx]) > 0:
				decoder := json.NewDecoder(bytes.NewReader(serializedFields[serializedIdx]))
				dest := elem.Elem().FieldByName(field.name).Addr().Interface()
				if err := decoder.Decode(dest); err != nil {
					return errors.Trace(err)
				}

				serializedIdx++

			case field.gob:
				decoder := gob.NewDecoder(bytes.NewReader(serializedFields[serializedIdx]))
				dest := elem.Elem().FieldByName(field.name).Addr().Interface()
				if err := decoder.Decode(dest); err != nil {
					return errors.Trace(err)
				}

				serializedIdx++
			}
		}

		// Run hooks
		if err := runAfterFindHook(elem); err != nil {
			return errors.Trace(err)
		}

		// Append to the result
		outputValue.Elem().Set(reflect.Append(outputValue.Elem(), elem))
	}

	return nil
}

// Get returns the first result that matchs the query putting it in the output model.
func (q *Query) Get(ctx context.Context, output interface{}) error {
	outputValue := reflect.ValueOf(output)
	outputType := reflect.TypeOf(output)

	// Some sanity checks about the output value
	if outputValue.Kind() != reflect.Ptr || outputValue.Elem().Kind() != reflect.Struct {
		return errors.New("output should be a pointer to a struct")
	}

	// Limit the request to a single result
	q.Limit(1)

	// Buid an empty list for the results
	result := reflect.New(reflect.SliceOf(outputType))

	// Fetch the result
	getAll := reflect.ValueOf(q.GetAll)
	params := []reflect.Value{
		reflect.ValueOf(ctx),
		result,
	}
	if err := getAll.Call(params)[0]; !err.IsNil() {
		return errors.Trace(err.Interface().(error))
	}

	resultElem := result.Elem()
	if resultElem.Len() == 0 {
		return ErrNoSuchEntity
	}

	// Output only the individual result, not the whole list
	outputValue.Elem().Set(resultElem.Index(0).Elem())

	return nil
}

// Limit returns only the specified number of results as a maximum
func (q *Query) Limit(limit int64) *Query {
	q = q.Clone()

	q.limit = limit

	return q
}

// Order sets the order of the rows in the result
func (q *Query) Order(order string) *Query {
	q = q.Clone()

	q.order = order

	return q
}

// Delete removes the models that match the query.
func (q *Query) Delete(ctx context.Context, model interface{}) error {
	modelValue := reflect.ValueOf(model)
	modelType := reflect.TypeOf(model)

	// Some sanity checks about the model
	if modelValue.Kind() != reflect.Ptr || modelValue.Elem().Kind() != reflect.Struct {
		return errors.New("model should be a pointer to a struct")
	}

	// Build the WHERE conditions of the query
	var conditions string
	if len(q.conditions) > 0 {
		conditions = fmt.Sprintf(" WHERE %s", strings.Join(q.conditions, " AND "))
	}

	// Build the table name
	tableName := getTableName(modelType.Elem())
	query := fmt.Sprintf("DELETE FROM `%s`%s", tableName, conditions)

	// Exec the query
	conn := FromContext(ctx)
	if conn.Debug {
		log.Println("Delete:", query, "-->", q.values)
	}
	if _, err := conn.DB.Exec(query, q.values...); err != nil {
		return errors.Annotate(err, query)
	}

	return nil
}
<commit_msg>Use the limit in the query. Allow to specify an offset too.<commit_after>package database

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"log"
	"reflect"
	"strings"

	"github.com/juju/errors"
	"golang.org/x/net/context"
)

// Query helps preparing and executing queries.
type Query struct {
	// Where conditions and replacement values
	conditions []string
	values     []interface{}

	limit, offset int64
	order         string
}

// NewQuery starts a new query to the database.
func NewQuery() *Query {
	return &Query{}
}

// Clone makes a copy of the query keeping all the internal state up to that moment.
func (q *Query) Clone() *Query {
	conditions := make([]string, len(q.conditions))
	for i := range conditions {
		conditions[i] = q.conditions[i]
	}
	values := make([]interface{}, len(q.values))
	for i := range values {
		values[i] = q.values[i]
	}

	return &Query{
		conditions: conditions,
		values:     values,
		limit:      q.limit,
		order:      q.order,
	}
}

// Where filters by a new column adding some placeholders if needed.
func (q *Query) Where(column string, args ...interface{}) *Query {
	q = q.Clone()

	nargs := strings.Count(column, "?")
	if len(args) != nargs {
		panic(fmt.Sprintf("expected %d parameters in the query and received %d", nargs, len(args)))
	}
	if !hasOperator(column) {
		panic(fmt.Sprintf("column does not have an operator: %s", column))
	}

	if strings.HasSuffix(column, "IN (?)") && len(args) == 1 && reflect.TypeOf(args[0]).Kind() == reflect.Slice {
		argValue := reflect.ValueOf(args[0])
		placeholders := make([]string, argValue.Len())
		for i := range placeholders {
			placeholders[i] = "?"
		}
		newColumn := fmt.Sprintf("IN (%s)", strings.Join(placeholders, ", "))

		q.conditions = append(q.conditions, strings.Replace(column, "IN (?)", newColumn, -1))

		for i := 0; i < argValue.Len(); i++ {
			q.values = append(q.values, argValue.Index(i).Interface())
		}
	} else {
		q.conditions = append(q.conditions, column)
		q.values = append(q.values, args...)
	}

	return q
}

// GetAll returns all the results that matchs the query putting it in the output slice.
func (q *Query) GetAll(ctx context.Context, output interface{}) error {
	outputValue := reflect.ValueOf(output)
	outputType := reflect.TypeOf(output)

	// Some sanity checks about the output value
	if outputValue.Kind() != reflect.Ptr || outputValue.Elem().Kind() != reflect.Slice {
		return errors.New("output should be a pointer to a slice")
	}
	sliceElemType := outputType.Elem().Elem()
	if sliceElemType.Kind() != reflect.Ptr || sliceElemType.Elem().Kind() != reflect.Struct {
		return errors.New("output should be a pointer to a slice of struct pointers")
	}

	// Build the table name
	tableName := getTableName(sliceElemType.Elem())

	// Get the list of field names
	fields, columns, err := getSerializableFields(sliceElemType.Elem())
	if err != nil {
		return errors.Trace(err)
	}

	// Prepare the query string
	query := fmt.Sprintf("SELECT %s FROM `%s`", columns, tableName)
	if len(q.conditions) > 0 {
		query = fmt.Sprintf("%s WHERE %s", query, strings.Join(q.conditions, " AND "))
	}
	if q.order != "" {
		query = fmt.Sprintf("%s ORDER BY %s", query, q.order)
	}
	if q.limit != 0 {
		query = fmt.Sprintf("%s LIMIT %s", query, q.limit)

		if q.offset != 0 {
			query = fmt.Sprintf("%s OFFSET %s", query, q.offset)
		}
	} else if q.offset != 0 {
		return errors.New("cannot specify an offset in the query without limit")
	}

	// Run the query and fetch the rows
	conn := FromContext(ctx)
	rows, err := conn.DB.Query(query, q.values...)
	if err != nil {
		return errors.Annotate(err, query)
	}
	defer rows.Close()

	scan := reflect.ValueOf(rows.Scan)
	for rows.Next() {
		// Create a new element
		elem := reflect.New(sliceElemType.Elem())

		// Prepare space to save the bytes of serialized fields
		serializedFields := [][]byte{}
		for _, field := range fields {
			if field.json || field.gob {
				serializedFields = append(serializedFields, []byte{})
			}
		}

		// Prepare the pointers to all its fields
		pointers := []reflect.Value{}
		serializedIdx := 0
		for _, field := range fields {
			if field.json || field.gob {
				pointers = append(pointers, reflect.ValueOf(&serializedFields[serializedIdx]))
				serializedIdx++
			} else {
				pointers = append(pointers, elem.Elem().FieldByName(field.name).Addr())
			}
		}

		// Scan the row into the struct
		if err := scan.Call(pointers)[0]; !err.IsNil() {
			return errors.Trace(err.Interface().(error))
		}

		// Read serialized fields
		serializedIdx = 0
		for _, field := range fields {
			switch {
			case field.json && len(serializedFields[serializedIdx]) > 0:
				decoder := json.NewDecoder(bytes.NewReader(serializedFields[serializedIdx]))
				dest := elem.Elem().FieldByName(field.name).Addr().Interface()
				if err := decoder.Decode(dest); err != nil {
					return errors.Trace(err)
				}

				serializedIdx++

			case field.gob:
				decoder := gob.NewDecoder(bytes.NewReader(serializedFields[serializedIdx]))
				dest := elem.Elem().FieldByName(field.name).Addr().Interface()
				if err := decoder.Decode(dest); err != nil {
					return errors.Trace(err)
				}

				serializedIdx++
			}
		}

		// Run hooks
		if err := runAfterFindHook(elem); err != nil {
			return errors.Trace(err)
		}

		// Append to the result
		outputValue.Elem().Set(reflect.Append(outputValue.Elem(), elem))
	}

	return nil
}

// Get returns the first result that matchs the query putting it in the output model.
func (q *Query) Get(ctx context.Context, output interface{}) error {
	outputValue := reflect.ValueOf(output)
	outputType := reflect.TypeOf(output)

	// Some sanity checks about the output value
	if outputValue.Kind() != reflect.Ptr || outputValue.Elem().Kind() != reflect.Struct {
		return errors.New("output should be a pointer to a struct")
	}

	// Limit the request to a single result
	q.Limit(1)

	// Buid an empty list for the results
	result := reflect.New(reflect.SliceOf(outputType))

	// Fetch the result
	getAll := reflect.ValueOf(q.GetAll)
	params := []reflect.Value{
		reflect.ValueOf(ctx),
		result,
	}
	if err := getAll.Call(params)[0]; !err.IsNil() {
		return errors.Trace(err.Interface().(error))
	}

	resultElem := result.Elem()
	if resultElem.Len() == 0 {
		return ErrNoSuchEntity
	}

	// Output only the individual result, not the whole list
	outputValue.Elem().Set(resultElem.Index(0).Elem())

	return nil
}

// Limit returns only the specified number of results as a maximum
func (q *Query) Limit(limit int64) *Query {
	q = q.Clone()

	q.limit = limit

	return q
}

// Offset returns results starting from the specified row
func (q *Query) Offset(offset int64) *Query {
	q = q.Clone()

	q.offset = offset

	return q
}

// Order sets the order of the rows in the result
func (q *Query) Order(order string) *Query {
	q = q.Clone()

	q.order = order

	return q
}

// Delete removes the models that match the query.
func (q *Query) Delete(ctx context.Context, model interface{}) error {
	modelValue := reflect.ValueOf(model)
	modelType := reflect.TypeOf(model)

	// Some sanity checks about the model
	if modelValue.Kind() != reflect.Ptr || modelValue.Elem().Kind() != reflect.Struct {
		return errors.New("model should be a pointer to a struct")
	}

	// Build the WHERE conditions of the query
	var conditions string
	if len(q.conditions) > 0 {
		conditions = fmt.Sprintf(" WHERE %s", strings.Join(q.conditions, " AND "))
	}

	// Build the table name
	tableName := getTableName(modelType.Elem())
	query := fmt.Sprintf("DELETE FROM `%s`%s", tableName, conditions)

	// Exec the query
	conn := FromContext(ctx)
	if conn.Debug {
		log.Println("Delete:", query, "-->", q.values)
	}
	if _, err := conn.DB.Exec(query, q.values...); err != nil {
		return errors.Annotate(err, query)
	}

	return nil
}
<|endoftext|>


<commit_before>//
// Copyright 2017 Pedro Salgado
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package resque

import (
	"fmt"

	"gopkg.in/redis.v5"
)

const (
	// Queues name of the key for the set where Resque stores the currently available queues.
	Queues = "resque:queues"
)

// Queue a job queue.
type Queue struct {
	redis        *redis.Client
	jobClassName string
	Name         string
}

// newQueue initializes a Queue struct and updates the set of available Resque queues.
func newQueue(jcn string, c *redis.Client) (*Queue, error) {
	q := &Queue{
		redis:        c,
		jobClassName: jcn,
		Name:         fmt.Sprintf("resque:queue:%s", jcn),
	}

	exists, err := c.SIsMember(Queues, jcn).Result()
	if err != nil {
		return nil, err
	}
	if !exists {
		_, err := c.SAdd(Queues, jcn).Result()
		if err != nil {
			return nil, err
		}
	}

	return q, nil
}

// Peek returns from the queue the jobs at position(s) [start, stop].
func (q Queue) Peek(start, stop int64) ([]Job, error) {

	var jobs []Job

	err := q.redis.LRange(q.Name, start, stop).ScanSlice(jobs)
	if err != nil {
		return nil, err
	}

	return jobs, nil
}

// Receive gets a job from the queue.
func (q Queue) Receive() (*Job, error) {

	var job Job

	err := q.redis.LPop(q.Name).Scan(&job)
	if err != nil {
		if err.Error() == "redis: nil" {
			return nil, nil
		}
		return nil, err
	}

	return &job, err
}

// Send places a job to the queue.
func (q Queue) Send(job Job) error {

	byteArr, err := job.MarshalBinary()
	if err != nil {
		return err
	}

	return q.redis.RPush(q.Name, string(byteArr)).Err()
}

// Size returns the number of jobs in the queue.
func (q Queue) Size() (int64, error) {
	v, err := q.redis.SCard(q.Name).Result()
	if err != nil {
		return 0, err
	}

	return v, err
}
<commit_msg>queue.go: - added timeout attribute - Receive() now uses BLPop() - Size() now uses LLen()<commit_after>//
// Copyright 2017 Pedro Salgado
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package resque

import (
	"fmt"
	"time"

	"gopkg.in/redis.v5"
)

const (
	// Queues name of the key for the set where Resque stores the currently available queues.
	Queues = "resque:queues"
)

// Queue a job queue.
type Queue struct {
	redis        *redis.Client
	jobClassName string
	Name         string
	timeout      time.Duration
}

// newQueue initializes a Queue struct and updates the set of available Resque queues.
func newQueue(jcn string, c *redis.Client, timeout time.Duration) (*Queue, error) {
	q := &Queue{
		redis:        c,
		jobClassName: jcn,
		Name:         fmt.Sprintf("resque:queue:%s", jcn),
		timeout:      timeout,
	}

	exists, err := c.SIsMember(Queues, jcn).Result()
	if err != nil {
		return nil, err
	}
	if !exists {
		_, err := c.SAdd(Queues, jcn).Result()
		if err != nil {
			return nil, err
		}
	}

	return q, nil
}

// Peek returns from the queue the jobs at position(s) [start, stop].
func (q Queue) Peek(start, stop int64) ([]Job, error) {

	var jobs []Job

	err := q.redis.LRange(q.Name, start, stop).ScanSlice(jobs)
	if err != nil {
		return nil, err
	}

	return jobs, nil
}

// Receive gets a job from the queue.
func (q Queue) Receive() (*Job, error) {

	var job Job

	v, err := q.redis.BLPop(q.timeout, q.Name).Result()
	if err != nil {
		if err.Error() == "redis: nil" {
			return nil, nil
		}
		return nil, err
	}

	err = job.UnmarshalBinary([]byte(v[1]))
	if err != nil {
		return nil, err
	}

	return &job, err
}

// Send places a job to the queue.
func (q Queue) Send(job Job) error {

	byteArr, err := job.MarshalBinary()
	if err != nil {
		return err
	}

	return q.redis.RPush(q.Name, string(byteArr)).Err()
}

// Size returns the number of jobs in the queue.
func (q Queue) Size() (int64, error) {
	v, err := q.redis.LLen(q.Name).Result()
	if err != nil {
		return 0, err
	}

	return v, err
}
<|endoftext|>


<commit_before>package v8_test

import (
	"fmt"
	"strconv"

	"github.com/augustoroman/v8"
)

// AddAllNumbers is the callback function that we'll make accessible the JS VM.
// It will accept 2 or more numbers and return the sum. If fewer than two args
// are passed or any of the args are not parsable as numbers, it will fail.
func AddAllNumbers(in v8.CallbackArgs) (*v8.Value, error) {
	if len(in.Args) < 2 {
		return nil, fmt.Errorf("add requires at least 2 numbers, but got %d args", len(in.Args))
	}
	result := 0.0
	for i, arg := range in.Args {
		n, err := strconv.ParseFloat(arg.String(), 64)
		if err != nil {
			return nil, fmt.Errorf("Arg %d [%q] cannot be parsed as a number: %v", i, arg.String(), err)
		}
		result += n
	}
	return in.Context.Create(result)
}

func ExampleContext_Bind() {
	ctx := v8.NewIsolate().NewContext()

	// First, we'll bind our callback function into a *v8.Value that we can
	// use as we please. The string "my_add_function" here is the used by V8 as
	// the name of the function. That is, we've defined:
	//   val.toString() = (function my_add_function() { [native code] });
	// However the name "my_add_function" isn't actually accessible in the V8
	// global scope anywhere yet.
	val := ctx.Bind("my_add_function", AddAllNumbers)

	fmt.Println("val.String() =", val.String())

	// Next we'll set that value into the global context to make it available to
	// the JS.
	if err := ctx.Global().Set("add", val); err != nil {
		panic(err)
	}

	// Now we'll call it!
	result, err := ctx.Eval(`add(1,2,3,4,5)`, `example.js`)
	if err != nil {
		panic(err)
	}
	fmt.Println(`add(1,2,3,4,5) =`, result)

	// output:
	// val.String() = function my_add_function() { [native code] }
	// add(1,2,3,4,5) = 15
}
<commit_msg>Remove confusing print statement from Bind example.<commit_after>package v8_test

import (
	"fmt"
	"strconv"

	"github.com/augustoroman/v8"
)

// AddAllNumbers is the callback function that we'll make accessible the JS VM.
// It will accept 2 or more numbers and return the sum. If fewer than two args
// are passed or any of the args are not parsable as numbers, it will fail.
func AddAllNumbers(in v8.CallbackArgs) (*v8.Value, error) {
	if len(in.Args) < 2 {
		return nil, fmt.Errorf("add requires at least 2 numbers, but got %d args", len(in.Args))
	}
	result := 0.0
	for i, arg := range in.Args {
		n, err := strconv.ParseFloat(arg.String(), 64)
		if err != nil {
			return nil, fmt.Errorf("Arg %d [%q] cannot be parsed as a number: %v", i, arg.String(), err)
		}
		result += n
	}
	return in.Context.Create(result)
}

func ExampleContext_Bind() {
	ctx := v8.NewIsolate().NewContext()

	// First, we'll bind our callback function into a *v8.Value that we can
	// use as we please. The string "my_add_function" here is the used by V8 as
	// the name of the function. That is, we've defined:
	//   val.toString() = (function my_add_function() { [native code] });
	// However the name "my_add_function" isn't actually accessible in the V8
	// global scope anywhere yet.
	val := ctx.Bind("my_add_function", AddAllNumbers)

	// Next we'll set that value into the global context to make it available to
	// the JS.
	if err := ctx.Global().Set("add", val); err != nil {
		panic(err)
	}

	// Now we'll call it!
	result, err := ctx.Eval(`add(1,2,3,4,5)`, `example.js`)
	if err != nil {
		panic(err)
	}
	fmt.Println(`add(1,2,3,4,5) =`, result)

	// output:
	// add(1,2,3,4,5) = 15
}
<|endoftext|>


<commit_before>// A set of tests for AGI in Go
//
// Copyright (C) 2013 - 2014, Lefteris Zafiris <zaf.000@gmail.com>
// This program is free software, distributed under the terms of
// the BSD 3-Clause License. See the LICENSE file
// at the top of the source tree.
//
// Based on agi-test.agi from asterisk source tree.
// Can be used both as standalone AGI app or a FastAGI server
// if called with the flag '-spawn_fagi'

package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/zaf/agi"
	"log"
	"net"
)

var listen = flag.Bool("spawn_fagi", false, "Spawn as a FastAGI server")

func main() {
	flag.Parse()
	if *listen {
		//If called as a FastAGI server
		ln, err := net.Listen("tcp", ":4573")
		if err != nil {
			log.Fatal(err)
		}
		defer ln.Close()
		for {
			conn, err := ln.Accept()
			if err != nil {
				log.Println(err)
				continue
			}
			go spawnAgi(conn)
		}
	} else {
		//If called as standalone AGI app
		spawnAgi(nil)
	}
}

func spawnAgi(c net.Conn) {
	var myAgi *agi.Session
	var err error
	if c != nil {
		//Create a new FastAGI session
		rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
		myAgi, err = agi.Init(rw)
		defer func() {
			c.Close()
			myAgi.Destroy()
		}()
	} else {
		//Create a new AGI session
		myAgi, err = agi.Init(nil)
		defer myAgi.Destroy()
	}
	if err != nil {
		log.Printf("Error Parsing AGI environment: %v\n", err)
		return
	}
	testAgi(myAgi)
	return
}

func testAgi(sess *agi.Session) {
	//Perform some tests
	var tests, pass, fail int
	sess.Verbose("1.  Testing streamfile...")
	sess.StreamFile("beep", "")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("2.  Testing sendtext...")
	sess.SendText("Hello World")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("3.  Testing sendimage...")
	sess.SendImage("asterisk-image")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("4.  Testing saynumber...")
	sess.SayNumber(192837465, "")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("5.  Testing waitdtmf...")
	sess.WaitForDigit(3000)
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("6.  Testing redord...")
	sess.RecordFile("testagi", "gsm", "any", 3000)
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("7.  Testing record playback...")
	sess.StreamFile("testagi", "")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("8.  Testing set variable...")
	sess.SetVariable("testagi", "foo")
	tests++
	if sess.Res == nil || sess.Res[0] != "1" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("9.  Testing get full variable...")
	sess.GetFullVariable("testagi")
	tests++
	if sess.Res == nil || sess.Res[0] != "1" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("10. Testing exec...")
	sess.Exec("Wait", "3")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("================== Complete ======================")
	sess.Verbose(fmt.Sprintf("%d tests completed, %d passed, %d failed", tests, pass, fail))
	sess.Verbose("==================================================")
	return
}
<commit_msg>Small fixes in agi-test.go<commit_after>// A set of tests for AGI in Go
//
// Copyright (C) 2013 - 2014, Lefteris Zafiris <zaf.000@gmail.com>
// This program is free software, distributed under the terms of
// the BSD 3-Clause License. See the LICENSE file
// at the top of the source tree.
//
// Based on agi-test.agi from asterisk source tree.
// Can be used both as standalone AGI app or a FastAGI server
// if called with the flag '-spawn_fagi'

package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/zaf/agi"
	"log"
	"net"
)

var listen = flag.Bool("spawn_fagi", false, "Spawn as a FastAGI server")

func main() {
	flag.Parse()
	if *listen {
		//If called as a FastAGI server
		ln, err := net.Listen("tcp", ":4573")
		if err != nil {
			log.Fatal(err)
		}
		defer ln.Close()
		for {
			conn, err := ln.Accept()
			if err != nil {
				log.Println(err)
				continue
			}
			go spawnAgi(conn)
		}
	} else {
		//If called as standalone AGI app
		spawnAgi(nil)
	}
}

func spawnAgi(c net.Conn) {
	var myAgi *agi.Session
	var err error
	if c != nil {
		//Create a new FastAGI session
		rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
		myAgi, err = agi.Init(rw)
		defer func() {
			c.Close()
			myAgi.Destroy()
		}()
	} else {
		//Create a new AGI session
		myAgi, err = agi.Init(nil)
		defer myAgi.Destroy()
	}
	if err != nil {
		log.Printf("Error Parsing AGI environment: %v\n", err)
		return
	}
	testAgi(myAgi)
	return
}

func testAgi(sess *agi.Session) {
	//Perform some tests
	var tests, pass, fail int
	sess.Verbose("1.  Testing streamfile...")
	sess.StreamFile("beep", "")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("2.  Testing sendtext...")
	sess.SendText("Hello World")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("3.  Testing sendimage...")
	sess.SendImage("asterisk-image")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("4.  Testing saynumber...")
	sess.SayNumber(192837465, "")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("5.  Testing waitdtmf...")
	sess.WaitForDigit(3000)
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("6.  Testing redord...")
	sess.RecordFile("/tmp/testagi", "alaw", "any", 3000)
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("7.  Testing record playback...")
	sess.StreamFile("/tmp/testagi", "")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("8.  Testing set variable...")
	sess.SetVariable("testagi", "foo")
	tests++
	if sess.Res == nil || sess.Res[0] != "1" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("9.  Testing get full variable...")
	sess.GetFullVariable("testagi")
	tests++
	if sess.Res == nil || sess.Res[0] != "1" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("10. Testing exec...")
	sess.Exec("Wait", "3")
	tests++
	if sess.Res == nil || sess.Res[0] != "0" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}

	sess.Verbose("================== Complete ======================")
	sess.Verbose(fmt.Sprintf("%d tests completed, %d passed, %d failed", tests, pass, fail))
	sess.Verbose("==================================================")
	return
}
<|endoftext|>


<commit_before>package examples

/// contains all the examples
<commit_msg>remove examples package<commit_after><|endoftext|>


<commit_before>// Package radix is a simple redis driver. It needs better docs
package radix

import (
	"bufio"
	"io"
	"net"
	"sync"
	"time"

	"github.com/mediocregopher/radix.v2/resp"
)

// Client describes an entity which can carry out Actions, e.g. a connection
// pool for a single redis instance or the cluster client.
type Client interface {
	Do(Action) error
	Close() error
}

// Conn is an entity which reads/writes data using the redis resp protocol. The
// methods are synchronous. Encode and Decode may be called at the same time by
// two different go-routines, but each should only be called once at a time
// (i.e. two routines shouldn't call Encode at the same time, same with Decode).
type Conn interface {
	Encode(resp.Marshaler) error
	Decode(resp.Unmarshaler) error

	// Close closes the Conn and cleans up its resources. No methods may be
	// called after Close.
	Close() error
}

type rwcWrap struct {
	rwc    io.ReadWriteCloser
	brw    *bufio.ReadWriter
	rp, wp *resp.Pool
	*sync.Once
}

// NewConn takes an existing io.ReadWriteCloser and wraps it to support the Conn
// interface. The original io.ReadWriteCloser should not be used after calling
// this.
//
// In both the Encode and Decode methods of the returned Conn, if a net.Error is
// encountered the Conn will have Close called on it automatically.
func NewConn(rwc io.ReadWriteCloser) Conn {
	return rwcWrap{
		rwc:  rwc,
		brw:  bufio.NewReadWriter(bufio.NewReader(rwc), bufio.NewWriter(rwc)),
		rp:   new(resp.Pool),
		wp:   new(resp.Pool),
		Once: new(sync.Once),
	}
}

func (rwc rwcWrap) Encode(m resp.Marshaler) error {
	err := m.MarshalRESP(rwc.wp, rwc.brw)
	defer func() {
		if _, ok := err.(net.Error); ok {
			rwc.Close()
		}
	}()

	if err != nil {
		return err
	}
	err = rwc.brw.Flush()
	return err
}

func (rwc rwcWrap) Decode(u resp.Unmarshaler) error {
	err := u.UnmarshalRESP(rwc.rp, rwc.brw.Reader)
	if _, ok := err.(net.Error); ok {
		rwc.Close()
	}
	return err
}

func (rwc rwcWrap) Close() error {
	var err error
	rwc.Once.Do(func() {
		err = rwc.rwc.Close()
	})
	return err
}

// DialFunc is a function which returns an initialized, ready-to-be-used Conn.
// Functions like NewPool or NewCluster take in a DialFunc in order to allow for
// things like calls to AUTH on each new connection, setting timeouts, custom
// Conn implementations, etc...
type DialFunc func(network, addr string) (Conn, error)

// Dial creates a network connection using net.Dial and passes it into NewConn.
func Dial(network, addr string) (Conn, error) {
	c, err := net.Dial(network, addr)
	if err != nil {
		return nil, err
	}
	return NewConn(c), nil
}

type timeoutConn struct {
	net.Conn
	timeout time.Duration
}

func (tc *timeoutConn) setDeadline() {
	if tc.timeout > 0 {
		tc.Conn.SetDeadline(time.Now().Add(tc.timeout))
	}
}

func (tc *timeoutConn) Read(b []byte) (int, error) {
	tc.setDeadline()
	return tc.Conn.Read(b)
}

func (tc *timeoutConn) Write(b []byte) (int, error) {
	tc.setDeadline()
	return tc.Conn.Write(b)
}

// DialTimeout is like Dial, but the given timeout is used to set read/write
// deadlines on all reads/writes
func DialTimeout(network, addr string, timeout time.Duration) (Conn, error) {
	c, err := net.DialTimeout(network, addr, timeout)
	if err != nil {
		return nil, err
	}
	return NewConn(&timeoutConn{Conn: c, timeout: timeout}), nil
}
<commit_msg>make radix.Conn inherit from net.Conn<commit_after>// Package radix is a simple redis driver. It needs better docs
package radix

import (
	"bufio"
	"net"
	"sync"
	"time"

	"github.com/mediocregopher/radix.v2/resp"
)

// Client describes an entity which can carry out Actions, e.g. a connection
// pool for a single redis instance or the cluster client.
type Client interface {
	Do(Action) error
	Close() error
}

// Conn is an entity which reads/writes data using the redis resp protocol. The
// methods are synchronous. Encode and Decode may be called at the same time by
// two different go-routines, but each should only be called once at a time
// (i.e. two routines shouldn't call Encode at the same time, same with Decode).
//
// NOTE the Read/Write methods inherited from net.Conn should not be used
// directly, though its other methods may.
type Conn interface {
	net.Conn
	Encode(resp.Marshaler) error
	Decode(resp.Unmarshaler) error
}

type connWrap struct {
	net.Conn
	brw    *bufio.ReadWriter
	rp, wp *resp.Pool
	*sync.Once
}

// NewConn takes an existing net.Conn and wraps it to support the Conn interface
// of this package. The Read and Write methods on the original net.Conn should
// not be used after calling this method.
//
// In both the Encode and Decode methods of the returned Conn, if a net.Error is
// encountered the Conn will have Close called on it automatically.
func NewConn(conn net.Conn) Conn {
	return connWrap{
		Conn: conn,
		brw:  bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)),
		rp:   new(resp.Pool),
		wp:   new(resp.Pool),
		Once: new(sync.Once),
	}
}

func (cw connWrap) Encode(m resp.Marshaler) error {
	err := m.MarshalRESP(cw.wp, cw.brw)
	defer func() {
		if _, ok := err.(net.Error); ok {
			cw.Close()
		}
	}()

	if err != nil {
		return err
	}
	err = cw.brw.Flush()
	return err
}

func (cw connWrap) Decode(u resp.Unmarshaler) error {
	err := u.UnmarshalRESP(cw.rp, cw.brw.Reader)
	if _, ok := err.(net.Error); ok {
		cw.Close()
	}
	return err
}

// DialFunc is a function which returns an initialized, ready-to-be-used Conn.
// Functions like NewPool or NewCluster take in a DialFunc in order to allow for
// things like calls to AUTH on each new connection, setting timeouts, custom
// Conn implementations, etc...
type DialFunc func(network, addr string) (Conn, error)

// Dial creates a network connection using net.Dial and passes it into NewConn.
func Dial(network, addr string) (Conn, error) {
	c, err := net.Dial(network, addr)
	if err != nil {
		return nil, err
	}
	return NewConn(c), nil
}

type timeoutConn struct {
	net.Conn
	timeout time.Duration
}

func (tc *timeoutConn) setDeadline() {
	if tc.timeout > 0 {
		tc.Conn.SetDeadline(time.Now().Add(tc.timeout))
	}
}

func (tc *timeoutConn) Read(b []byte) (int, error) {
	tc.setDeadline()
	return tc.Conn.Read(b)
}

func (tc *timeoutConn) Write(b []byte) (int, error) {
	tc.setDeadline()
	return tc.Conn.Write(b)
}

// DialTimeout is like Dial, but the given timeout is used to set read/write
// deadlines on all reads/writes
func DialTimeout(network, addr string, timeout time.Duration) (Conn, error) {
	c, err := net.DialTimeout(network, addr, timeout)
	if err != nil {
		return nil, err
	}
	return NewConn(&timeoutConn{Conn: c, timeout: timeout}), nil
}
<|endoftext|>


<commit_before>package weed_server

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"net/url"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/syndtr/goleveldb/leveldb"
)

type FilerPostResult struct {
	Name  string `json:"name,omitempty"`
	Size  uint32 `json:"size,omitempty"`
	Error string `json:"error,omitempty"`
	Fid   string `json:"fid,omitempty"`
	Url   string `json:"url,omitempty"`
}

var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")

func escapeQuotes(s string) string {
	return quoteEscaper.Replace(s)
}

func createFormFile(writer *multipart.Writer, fieldname, filename, mime string) (io.Writer, error) {
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition",
		fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
			escapeQuotes(fieldname), escapeQuotes(filename)))
	if len(mime) == 0 {
		mime = "application/octet-stream"
	}
	h.Set("Content-Type", mime)
	return writer.CreatePart(h)
}

func makeFormData(filename, mimeType string, content io.Reader) (formData io.Reader, contentType string, err error) {
	buf := new(bytes.Buffer)
	writer := multipart.NewWriter(buf)
	defer writer.Close()

	part, err := createFormFile(writer, "file", filename, mimeType)
	if err != nil {
		glog.V(0).Infoln(err)
		return
	}
	_, err = io.Copy(part, content)
	if err != nil {
		glog.V(0).Infoln(err)
		return
	}

	formData = buf
	contentType = writer.FormDataContentType()

	return
}

func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
	query := r.URL.Query()
	replication := query.Get("replication")
	if replication == "" {
		replication = fs.defaultReplication
	}
	collection := query.Get("collection")
	if collection == "" {
		collection = fs.collection
	}

	var fileId string
	var err error
	var urlLocation string

	if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data; boundary=") {
		//Default handle way for http multipart
		if r.Method == "PUT" {
			buf, _ := ioutil.ReadAll(r.Body)
			r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
			fileName, _, _, _, _, _, _, pe := storage.ParseUpload(r)
			if pe != nil {
				glog.V(0).Infoln("failing to parse post body", pe.Error())
				writeJsonError(w, r, http.StatusInternalServerError, pe)
				return
			}
			//reconstruct http request body for following new request to volume server
			r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))

			path := r.URL.Path
			if strings.HasSuffix(path, "/") {
				if fileName != "" {
					path += fileName
				}
			}

			if fileId, err = fs.filer.FindFile(path); err != nil && err != leveldb.ErrNotFound {
				glog.V(0).Infoln("failing to find path in filer store", path, err.Error())
				writeJsonError(w, r, http.StatusInternalServerError, err)
				return
			} else if fileId != "" && err == nil {
				var le error
				urlLocation, le = operation.LookupFileId(fs.getMasterNode(), fileId)
				if le != nil {
					glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, le.Error())
					w.WriteHeader(http.StatusNotFound)
					return
				}
			}
		} else {
			assignResult, ae := operation.Assign(fs.getMasterNode(), 1, replication, collection, query.Get("ttl"))
			if ae != nil {
				glog.V(0).Infoln("failing to assign a file id", ae.Error())
				writeJsonError(w, r, http.StatusInternalServerError, ae)
				return
			}
			fileId = assignResult.Fid
			urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid
		}
	} else {
		/*
			Amazon S3 ref link:[http://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html]
			There is a long way to provide a completely compatibility against all Amazon S3 API, I just made
			a simple data stream adapter between S3 PUT API and seaweedfs's volume storage Write API
			1. The request url format should be http://$host:$port/$bucketName/$objectName
			2. bucketName will be mapped to seaweedfs's collection name
		*/
		lastPos := strings.LastIndex(r.URL.Path, "/")
		if lastPos == -1 || lastPos == 0 || lastPos == len(r.URL.Path)-1 {
			glog.V(0).Infoln("URL Path [%s] is invalid, could not retrieve file name", r.URL.Path)
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("URL Path is invalid"))
			return
		}

		secondPos := strings.Index(r.URL.Path[1:], "/") + 1
		collection = r.URL.Path[1:secondPos]
		path := r.URL.Path

		if fileId, err = fs.filer.FindFile(path); err != nil && err != leveldb.ErrNotFound {
			glog.V(0).Infoln("failing to find path in filer store", path, err.Error())
			writeJsonError(w, r, http.StatusInternalServerError, err)
			return
		} else if fileId != "" && err == nil {
			var le error
			urlLocation, le = operation.LookupFileId(fs.getMasterNode(), fileId)
			if le != nil {
				glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, le.Error())
				w.WriteHeader(http.StatusNotFound)
				return
			}
		} else {
			assignResult, ae := operation.Assign(fs.getMasterNode(), 1, replication, collection, query.Get("ttl"))
			if ae != nil {
				glog.V(0).Infoln("failing to assign a file id", ae.Error())
				writeJsonError(w, r, http.StatusInternalServerError, ae)
				return
			}
			fileId = assignResult.Fid
			urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid
		}

		if contentMD5 := r.Header.Get("Content-MD5"); contentMD5 != "" {
			buf, _ := ioutil.ReadAll(r.Body)
			//checkMD5
			sum := md5.Sum(buf)
			fileDataMD5 := base64.StdEncoding.EncodeToString(sum[0:len(sum)])
			if strings.ToLower(fileDataMD5) != strings.ToLower(contentMD5) {
				glog.V(0).Infof("fileDataMD5 [%s] is not equal to Content-MD5 [%s]", fileDataMD5, contentMD5)
				writeJsonError(w, r, http.StatusNotAcceptable, fmt.Errorf("MD5 check failed"))
				return
			}
			//reconstruct http request body for following new request to volume server
			r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
		}

		fileName := r.URL.Path[lastPos+1:]
		body, contentType, te := makeFormData(fileName, r.Header.Get("Content-Type"), r.Body)
		if te != nil {
			glog.V(0).Infoln("S3 protocol to raw seaweed protocol failed", te.Error())
			writeJsonError(w, r, http.StatusInternalServerError, te)
			return
		}

		if body != nil {
			switch v := body.(type) {
			case *bytes.Buffer:
				r.ContentLength = int64(v.Len())
			case *bytes.Reader:
				r.ContentLength = int64(v.Len())
			case *strings.Reader:
				r.ContentLength = int64(v.Len())
			}
		}

		r.Header.Set("Content-Type", contentType)
		rc, ok := body.(io.ReadCloser)
		if !ok && body != nil {
			rc = ioutil.NopCloser(body)
		}
		r.Body = rc
	}

	u, _ := url.Parse(urlLocation)
	glog.V(4).Infoln("post to", u)
	request := &http.Request{
		Method:        r.Method,
		URL:           u,
		Proto:         r.Proto,
		ProtoMajor:    r.ProtoMajor,
		ProtoMinor:    r.ProtoMinor,
		Header:        r.Header,
		Body:          r.Body,
		Host:          r.Host,
		ContentLength: r.ContentLength,
	}
	resp, do_err := util.Do(request)
	if do_err != nil {
		glog.V(0).Infoln("failing to connect to volume server", r.RequestURI, do_err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, do_err)
		return
	}
	defer resp.Body.Close()
	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, ra_err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, ra_err)
		return
	}
	glog.V(4).Infoln("post result", string(resp_body))
	var ret operation.UploadResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(resp_body))
		writeJsonError(w, r, http.StatusInternalServerError, unmarshal_err)
		return
	}
	if ret.Error != "" {
		glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error)
		writeJsonError(w, r, http.StatusInternalServerError, errors.New(ret.Error))
		return
	}
	path := r.URL.Path
	if strings.HasSuffix(path, "/") {
		if ret.Name != "" {
			path += ret.Name
		} else {
			operation.DeleteFile(fs.getMasterNode(), fileId, fs.jwt(fileId)) //clean up
			glog.V(0).Infoln("Can not to write to folder", path, "without a file name!")
			writeJsonError(w, r, http.StatusInternalServerError,
				errors.New("Can not to write to folder "+path+" without a file name"))
			return
		}
	}

	// also delete the old fid unless PUT operation
	if r.Method != "PUT" {
		if oldFid, err := fs.filer.FindFile(path); err == nil {
			operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid))
		}
	}

	glog.V(4).Infoln("saving", path, "=>", fileId)
	if db_err := fs.filer.CreateFile(path, fileId); db_err != nil {
		operation.DeleteFile(fs.getMasterNode(), fileId, fs.jwt(fileId)) //clean up
		glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err)
		writeJsonError(w, r, http.StatusInternalServerError, db_err)
		return
	}

	reply := FilerPostResult{
		Name:  ret.Name,
		Size:  ret.Size,
		Error: ret.Error,
		Fid:   fileId,
		Url:   urlLocation,
	}
	writeJsonQuiet(w, r, http.StatusCreated, reply)
}

// curl -X DELETE http://localhost:8888/path/to
// curl -X DELETE http://localhost:8888/path/to?recursive=true
func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
	var err error
	var fid string
	if strings.HasSuffix(r.URL.Path, "/") {
		isRecursive := r.FormValue("recursive") == "true"
		err = fs.filer.DeleteDirectory(r.URL.Path, isRecursive)
	} else {
		fid, err = fs.filer.DeleteFile(r.URL.Path)
		if err == nil && fid != "" {
			err = operation.DeleteFile(fs.getMasterNode(), fid, fs.jwt(fid))
		}
	}
	if err == nil {
		writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""})
	} else {
		glog.V(4).Infoln("deleting", r.URL.Path, ":", err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, err)
	}
}
<commit_msg>refactor work for filer write handler<commit_after>package weed_server

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"net/url"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/syndtr/goleveldb/leveldb"
)

type FilerPostResult struct {
	Name  string `json:"name,omitempty"`
	Size  uint32 `json:"size,omitempty"`
	Error string `json:"error,omitempty"`
	Fid   string `json:"fid,omitempty"`
	Url   string `json:"url,omitempty"`
}

var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")

func escapeQuotes(s string) string {
	return quoteEscaper.Replace(s)
}

func createFormFile(writer *multipart.Writer, fieldname, filename, mime string) (io.Writer, error) {
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition",
		fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
			escapeQuotes(fieldname), escapeQuotes(filename)))
	if len(mime) == 0 {
		mime = "application/octet-stream"
	}
	h.Set("Content-Type", mime)
	return writer.CreatePart(h)
}

func makeFormData(filename, mimeType string, content io.Reader) (formData io.Reader, contentType string, err error) {
	buf := new(bytes.Buffer)
	writer := multipart.NewWriter(buf)
	defer writer.Close()

	part, err := createFormFile(writer, "file", filename, mimeType)
	if err != nil {
		glog.V(0).Infoln(err)
		return
	}
	_, err = io.Copy(part, content)
	if err != nil {
		glog.V(0).Infoln(err)
		return
	}

	formData = buf
	contentType = writer.FormDataContentType()

	return
}

func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Request, path string) (fileId, urlLocation string, err error) {
	if fileId, err = fs.filer.FindFile(path); err != nil && err != leveldb.ErrNotFound {
		glog.V(0).Infoln("failing to find path in filer store", path, err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	} else if fileId != "" && err == nil {
		urlLocation, err = operation.LookupFileId(fs.getMasterNode(), fileId)
		if err != nil {
			glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error())
			w.WriteHeader(http.StatusNotFound)
			return
		}
	}
	return
}

func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) {
	assignResult, ae := operation.Assign(fs.getMasterNode(), 1, replication, collection, r.URL.Query().Get("ttl"))
	if ae != nil {
		glog.V(0).Infoln("failing to assign a file id", ae.Error())
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		err = ae
		return
	}
	fileId = assignResult.Fid
	urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid
	return
}

func (fs *FilerServer) multipartUploadAnalyzer(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) {
	//Default handle way for http multipart
	if r.Method == "PUT" {
		buf, _ := ioutil.ReadAll(r.Body)
		r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
		fileName, _, _, _, _, _, _, pe := storage.ParseUpload(r)
		if pe != nil {
			glog.V(0).Infoln("failing to parse post body", pe.Error())
			writeJsonError(w, r, http.StatusInternalServerError, pe)
			err = pe
			return
		}
		//reconstruct http request body for following new request to volume server
		r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))

		path := r.URL.Path
		if strings.HasSuffix(path, "/") {
			if fileName != "" {
				path += fileName
			}
		}
		fileId, urlLocation, err = fs.queryFileInfoByPath(w, r, path)
	} else {
		fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection)
	}
	return
}

func multipartHttpBodyBuilder(w http.ResponseWriter, r *http.Request, fileName string) (err error) {
	body, contentType, te := makeFormData(fileName, r.Header.Get("Content-Type"), r.Body)
	if te != nil {
		glog.V(0).Infoln("S3 protocol to raw seaweed protocol failed", te.Error())
		writeJsonError(w, r, http.StatusInternalServerError, te)
		err = te
		return
	}

	if body != nil {
		switch v := body.(type) {
		case *bytes.Buffer:
			r.ContentLength = int64(v.Len())
		case *bytes.Reader:
			r.ContentLength = int64(v.Len())
		case *strings.Reader:
			r.ContentLength = int64(v.Len())
		}
	}

	r.Header.Set("Content-Type", contentType)
	rc, ok := body.(io.ReadCloser)
	if !ok && body != nil {
		rc = ioutil.NopCloser(body)
	}
	r.Body = rc
	return
}

func checkContentMD5(w http.ResponseWriter, r *http.Request) (err error) {
	if contentMD5 := r.Header.Get("Content-MD5"); contentMD5 != "" {
		buf, _ := ioutil.ReadAll(r.Body)
		//checkMD5
		sum := md5.Sum(buf)
		fileDataMD5 := base64.StdEncoding.EncodeToString(sum[0:len(sum)])
		if strings.ToLower(fileDataMD5) != strings.ToLower(contentMD5) {
			glog.V(0).Infof("fileDataMD5 [%s] is not equal to Content-MD5 [%s]", fileDataMD5, contentMD5)
			err = fmt.Errorf("MD5 check failed")
			writeJsonError(w, r, http.StatusNotAcceptable, err)
			return
		}
		//reconstruct http request body for following new request to volume server
		r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
	}
	return
}

func (fs *FilerServer) monolithicUploadAnalyzer(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) {
	/*
		Amazon S3 ref link:[http://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html]
		There is a long way to provide a completely compatibility against all Amazon S3 API, I just made
		a simple data stream adapter between S3 PUT API and seaweedfs's volume storage Write API
		1. The request url format should be http://$host:$port/$bucketName/$objectName
		2. bucketName will be mapped to seaweedfs's collection name
		3. You could customize and make your enhancement.
	*/
	lastPos := strings.LastIndex(r.URL.Path, "/")
	if lastPos == -1 || lastPos == 0 || lastPos == len(r.URL.Path)-1 {
		glog.V(0).Infoln("URL Path [%s] is invalid, could not retrieve file name", r.URL.Path)
		err = fmt.Errorf("URL Path is invalid")
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	if err = checkContentMD5(w, r); err != nil {
		return
	}

	fileName := r.URL.Path[lastPos+1:]
	if err = multipartHttpBodyBuilder(w, r, fileName); err != nil {
		return
	}

	secondPos := strings.Index(r.URL.Path[1:], "/") + 1
	collection = r.URL.Path[1:secondPos]
	path := r.URL.Path

	if fileId, urlLocation, err = fs.queryFileInfoByPath(w, r, path); err == nil && fileId == "" {
		fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection)
	}
	return
}

func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
	query := r.URL.Query()
	replication := query.Get("replication")
	if replication == "" {
		replication = fs.defaultReplication
	}
	collection := query.Get("collection")
	if collection == "" {
		collection = fs.collection
	}

	var fileId, urlLocation string
	var err error

	if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data; boundary=") {
		fileId, urlLocation, err = fs.multipartUploadAnalyzer(w, r, replication, collection)
		if err != nil {
			return
		}
	} else {
		fileId, urlLocation, err = fs.monolithicUploadAnalyzer(w, r, replication, collection)
		if err != nil {
			return
		}
	}

	u, _ := url.Parse(urlLocation)
	glog.V(4).Infoln("post to", u)
	request := &http.Request{
		Method:        r.Method,
		URL:           u,
		Proto:         r.Proto,
		ProtoMajor:    r.ProtoMajor,
		ProtoMinor:    r.ProtoMinor,
		Header:        r.Header,
		Body:          r.Body,
		Host:          r.Host,
		ContentLength: r.ContentLength,
	}
	resp, do_err := util.Do(request)
	if do_err != nil {
		glog.V(0).Infoln("failing to connect to volume server", r.RequestURI, do_err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, do_err)
		return
	}
	defer resp.Body.Close()
	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, ra_err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, ra_err)
		return
	}
	glog.V(4).Infoln("post result", string(resp_body))
	var ret operation.UploadResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(resp_body))
		writeJsonError(w, r, http.StatusInternalServerError, unmarshal_err)
		return
	}
	if ret.Error != "" {
		glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error)
		writeJsonError(w, r, http.StatusInternalServerError, errors.New(ret.Error))
		return
	}
	path := r.URL.Path
	if strings.HasSuffix(path, "/") {
		if ret.Name != "" {
			path += ret.Name
		} else {
			operation.DeleteFile(fs.getMasterNode(), fileId, fs.jwt(fileId)) //clean up
			glog.V(0).Infoln("Can not to write to folder", path, "without a file name!")
			writeJsonError(w, r, http.StatusInternalServerError,
				errors.New("Can not to write to folder "+path+" without a file name"))
			return
		}
	}

	// also delete the old fid unless PUT operation
	if r.Method != "PUT" {
		if oldFid, err := fs.filer.FindFile(path); err == nil {
			operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid))
		}
	}

	glog.V(4).Infoln("saving", path, "=>", fileId)
	if db_err := fs.filer.CreateFile(path, fileId); db_err != nil {
		operation.DeleteFile(fs.getMasterNode(), fileId, fs.jwt(fileId)) //clean up
		glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err)
		writeJsonError(w, r, http.StatusInternalServerError, db_err)
		return
	}

	reply := FilerPostResult{
		Name:  ret.Name,
		Size:  ret.Size,
		Error: ret.Error,
		Fid:   fileId,
		Url:   urlLocation,
	}
	writeJsonQuiet(w, r, http.StatusCreated, reply)
}

// curl -X DELETE http://localhost:8888/path/to
// curl -X DELETE http://localhost:8888/path/to?recursive=true
func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
	var err error
	var fid string
	if strings.HasSuffix(r.URL.Path, "/") {
		isRecursive := r.FormValue("recursive") == "true"
		err = fs.filer.DeleteDirectory(r.URL.Path, isRecursive)
	} else {
		fid, err = fs.filer.DeleteFile(r.URL.Path)
		if err == nil && fid != "" {
			err = operation.DeleteFile(fs.getMasterNode(), fid, fs.jwt(fid))
		}
	}
	if err == nil {
		writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""})
	} else {
		glog.V(4).Infoln("deleting", r.URL.Path, ":", err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, err)
	}
}
<|endoftext|>


<commit_before>// remap Takes a CSV file and a map file then generates
// a output file as defined within the map file
//
// \author  Matthew Cross <matthew@pmg.co>
// \package remap
// \version 1.0
package main

import (
  "os"
  "fmt"
  "flag"
  "bufio"
  "strings"
  "encoding/csv"
)

// Global constants
const DefaultBufferSize  = 4096
const DefaultHeaderCount = 128
const DefaultFilterCount = 24

var filterSet = make([]Filter, DefaultFilterCount)
var filterSetIndex = 0

type Filter struct {
  field string
  operation string
  value string
}

func (f *Filter) parseOperation(testValue string) bool {
  switch strings.ToLower(f.operation) {
    case "=":
      return f.value == testValue

    case "!=":
      return f.value != testValue

    case "<=":
      return f.value <= testValue

    case ">=":
      return f.value >= testValue

    case "<":
      return f.value < testValue

    case ">":
      return f.value > testValue

    case "like":
      return strings.Contains(f.value, testValue)

    case "notlike":
      return !strings.Contains(f.value, testValue)
  }
  return false
}

func (f *Filter) Apply(columns map[string]string) bool {
  return f.parseOperation(columns[f.field])
}

// Adds a filter to the filterset if one exists
//
// \since  1.0
// \param  line the extracted file line
// \return void
func addFilter(line string) string {
  if fields := strings.Split(line, " "); len(fields) == 3 {
    filterSet[filterSetIndex].field     = fields[0]
    filterSet[filterSetIndex].operation = fields[1]
    filterSet[filterSetIndex].value     = fields[2]
    filterSetIndex++
    return fields[0]
  }
  return line
}

// Checks to see if a line contains filter operations
//
// \since  1.0
// \param  filters the filters to apply
// \param  column the line parsed into columns
// \return bool true if the line passes all the filters
func checkFilters(filters []Filter, columns map[string]string) (bool) {
  for i := 0; i < filterSetIndex; i++ {
    if !filters[i].Apply(columns) {
      return false
    }
  }
  return true
}

// Reads a newline delimited map file that is used when outputing the results
func readMapFile(filename string) (mappedHeaders []string, errorMessage string) {
  mappedHeaders = make([]string, DefaultHeaderCount)

  file, err := os.Open(filename)
  if err != nil {
    errorMessage = "Cannot open mapped header file."
    return
  }
  defer file.Close()

  reader := bufio.NewReaderSize(file, DefaultBufferSize)

  index := 0
  line, isPrefix, err := reader.ReadLine()
  for err == nil && !isPrefix {
    if (index >= DefaultBufferSize) {
      errorMessage = "Maxim amount of headers reached."
      return
    }
    mappedHeaders[index] = addFilter(string(line))
    index++
    line, isPrefix, err = reader.ReadLine()
  }
  return mappedHeaders[:index], ""
}

// Converts two arrays into one associative array (logical equivelent to PHPs' array_combine() function)
//
// \since  1.0
// \param  mappedHeaders a slice of headers retrieved from the map file
// \param  columns an associative array of columns for the current input file line
// \return an array of column values mapped from mappedHeaders
func combineHeaders(mappedHeaders []string, columns map[string]string) (records []string) {
  records = make([]string, len(mappedHeaders))
  for index, header := range mappedHeaders {
    records[index] = columns[header]
  }
  return
}

// Read an input file and a map file then generate an output file
//
// \since  1.0
// \param  inputName the input filename
// \param  outputName the output filename
// \param  mapName the map filename
// \return void
func readFile(inputName string, outputName string, mapName string) {
  inputFile, err := os.Open(inputName)
  if err != nil {
    panic(err)
  }
  defer inputFile.Close()

  outputFile, err := os.Create(outputName)
  if err != nil {
    panic(err)
  }
  defer outputFile.Close()

  mappedHeaders, mappingError := readMapFile(mapName)
  if mappingError != "" {
    panic(err)
  }

  reader := csv.NewReader(inputFile)
  writer := csv.NewWriter(outputFile)

  fileHeaders, err := reader.Read()
  columns := make(map[string]string, len(fileHeaders))

  writer.Write(mappedHeaders)
  for err == nil {
    records, err := reader.Read()
    if err != nil {
      return
    }

    for index, header := range fileHeaders {
      columns[header] = records[index]
    }

    if checkFilters(filterSet, columns) {
      err = writer.Write(combineHeaders(mappedHeaders, columns))
      if err != nil {
        fmt.Println(err)
      }
      writer.Flush()
    }
  }
}

// Main entry point
func main() {
  flag.Parse()
  args := flag.Args()
  if len(args) < 3 {
      fmt.Println("remap [input file] [output file] [map file]")
      os.Exit(1)
  }
  readFile(args[0], args[1], args[2])
  fmt.Println("Finished!")
  os.Exit(0)
}
<commit_msg>Finish filter implementation.<commit_after>// remap Takes a CSV file and a map file then generates
// a output file as defined within the map file
//
// \author  Matthew Cross <matthew@pmg.co>
// \package remap
// \version 1.0
package main

import (
  "os"
  "fmt"
  "flag"
  "bufio"
  "strings"
  "encoding/csv"
)

// Global constants
const DefaultBufferSize  = 4096
const DefaultHeaderCount = 128
const DefaultFilterCount = 24

var filterSet = make([]Filter, DefaultFilterCount)
var filterSetIndex = 0

// Generic class to represent a filter operation
//
// \since  1.0
type Filter struct {
  // The column to filter on
  field string

  // The filter operation to perform
  operation string

  // The value to filter against
  value string
}

// Parse a the filter operation against testValue
//
// \since  1.0
// \access protected
// \param  testValue the value to filter on
// \return bool true if the filter passes; false otherwise
func (f *Filter) parseOperation(testValue string) bool {
  switch strings.ToLower(f.operation) {
    case "=":
      return f.value == testValue

    case "!=":
      return f.value != testValue

    case "<=":
      return !(f.value <= testValue)

    case ">=":
      return !(f.value >= testValue)

    case "<":
      return !(f.value < testValue)

    case ">":
      return !(f.value > testValue)

    case "like":
      return strings.Contains(f.value, testValue)

    case "notlike":
      return !strings.Contains(f.value, testValue)
  }
  return false
}

// Apply a filter on a given column set
//
// \since  1.0
// \access public
// \param  columns the column set to filter on
// \return true if the filter passes; false otherwise
func (f *Filter) Apply(columns map[string]string) bool {
  return f.parseOperation(columns[f.field])
}

// Adds a filter to the filterset if one exists
//
// \since  1.0
// \param  line the extracted file line
// \return void
func addFilter(line string) string {
  if fields := strings.Split(line, " "); len(fields) == 3 {
    filterSet[filterSetIndex].field     = fields[0]
    filterSet[filterSetIndex].operation = fields[1]
    filterSet[filterSetIndex].value     = fields[2]
    filterSetIndex++
    return fields[0]
  }
  return line
}

// Checks to see if a line contains filter operations
//
// \since  1.0
// \param  filters the filters to apply
// \param  column the line parsed into columns
// \return bool true if the line passes all the filters
func checkFilters(filters []Filter, columns map[string]string) (bool) {
  for i := 0; i < filterSetIndex; i++ {
    if !filters[i].Apply(columns) {
      return false
    }
  }
  return true
}

// Reads a newline delimited map file that is used when outputting the results
//
// \since  1.0
// \param  filename the map filename
// \return mapped array of headers otherwise error will be a non empty string
func readMapFile(filename string) (mappedHeaders []string, errorMessage string) {
  mappedHeaders = make([]string, DefaultHeaderCount)

  file, err := os.Open(filename)
  if err != nil {
    errorMessage = "Cannot open mapped header file."
    return
  }
  defer file.Close()

  reader := bufio.NewReaderSize(file, DefaultBufferSize)

  index := 0
  line, isPrefix, err := reader.ReadLine()
  for err == nil && !isPrefix {
    if (index >= DefaultBufferSize) {
      errorMessage = "Maxim amount of headers reached."
      return
    }
    mappedHeaders[index] = addFilter(string(line))
    index++
    line, isPrefix, err = reader.ReadLine()
  }
  return mappedHeaders[:index], ""
}

// Converts two arrays into one associative array (logical equivelent to PHPs' array_combine() function)
//
// \since  1.0
// \param  mappedHeaders a slice of headers retrieved from the map file
// \param  columns an associative array of columns for the current input file line
// \return an array of column values mapped from mappedHeaders
func combineHeaders(mappedHeaders []string, columns map[string]string) (records []string) {
  records = make([]string, len(mappedHeaders))
  for index, header := range mappedHeaders {
    records[index] = columns[header]
  }
  return
}

// Read an input file and a map file then generate an output file
//
// \since  1.0
// \param  inputName the input filename
// \param  outputName the output filename
// \param  mapName the map filename
// \return void
func readFile(inputName string, outputName string, mapName string) {
  inputFile, err := os.Open(inputName)
  if err != nil {
    panic(err)
  }
  defer inputFile.Close()

  outputFile, err := os.Create(outputName)
  if err != nil {
    panic(err)
  }
  defer outputFile.Close()

  mappedHeaders, mappingError := readMapFile(mapName)
  if mappingError != "" {
    panic(err)
  }

  reader := csv.NewReader(inputFile)
  writer := csv.NewWriter(outputFile)

  fileHeaders, err := reader.Read()
  columns := make(map[string]string, len(fileHeaders))

  writer.Write(mappedHeaders)
  for err == nil {
    records, err := reader.Read()
    if err != nil {
      return
    }

    for index, header := range fileHeaders {
      columns[header] = records[index]
    }

    if checkFilters(filterSet, columns) {
      err = writer.Write(combineHeaders(mappedHeaders, columns))
      if err != nil {
        fmt.Println(err)
      }
      writer.Flush()
    }
  }
}

// Main entry point
func main() {
  flag.Parse()
  args := flag.Args()
  if len(args) < 3 {
      fmt.Println("remap [input file] [output file] [map file]")
      os.Exit(1)
  }
  readFile(args[0], args[1], args[2])
  fmt.Println("Finished!")
  os.Exit(0)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package annotations

import (
	"strings"

	"github.com/onsi/ginkgo"

	"k8s.io/ingress-nginx/test/e2e/framework"
)

var _ = framework.DescribeAnnotation("disable-access-log disable-http-access-log disable-stream-access-log", func() {
	f := framework.NewDefaultFramework("disableaccesslog")

	ginkgo.BeforeEach(func() {
		f.NewEchoDeployment()
	})

	ginkgo.It("disable-access-log set access_log off", func() {
		host := "disableaccesslog.foo.com"
		annotations := map[string]string{
			"nginx.ingress.kubernetes.io/disable-access-log": "true",
		}

		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations)
		f.EnsureIngress(ing)

		f.WaitForNginxServer(host,
			func(server string) bool {
				str := `
                                multiline
                                string.`
				return strings.Contains(server, `access_log off;`)
			})
	})

	ginkgo.It("disable-http-access-log set access_log off", func() {
		host := "disablehttpaccesslog.foo.com"
		annotations := map[string]string{
			"nginx.ingress.kubernetes.io/disable-http-access-log": "true",
		}

		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations)
		f.EnsureIngress(ing)

		f.WaitForNginxServer(host,
			func(server string) bool {
				return strings.Contains(server, `access_log off;`)
			})
	})

	ginkgo.It("disable-stream-access-log set access_log off", func() {
		host := "disablehttpaccesslog.foo.com"
		annotations := map[string]string{
			"nginx.ingress.kubernetes.io/disable-stream-access-log": "true",
		}

		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations)
		f.EnsureIngress(ing)

		f.WaitForNginxServer(host,
			func(server string) bool {
				return strings.Contains(server, `access_log off;`)
			})
	})
})
<commit_msg>Fix e2e test error<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package annotations

import (
	"strings"

	"github.com/onsi/ginkgo"

	"k8s.io/ingress-nginx/test/e2e/framework"
)

var _ = framework.DescribeAnnotation("disable-access-log disable-http-access-log disable-stream-access-log", func() {
	f := framework.NewDefaultFramework("disableaccesslog")

	ginkgo.BeforeEach(func() {
		f.NewEchoDeployment()
	})

	ginkgo.It("disable-access-log set access_log off", func() {
		host := "disableaccesslog.foo.com"

		f.UpdateNginxConfigMapData("disable-access-log", "true")
		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)
		f.EnsureIngress(ing)

		f.WaitForNginxConfiguration(func(ngx string) bool {
			return strings.Contains(ngx, `access_log off;`)
		})
	})

	ginkgo.It("disable-http-access-log set access_log off", func() {
		host := "disablehttpaccesslog.foo.com"

		f.UpdateNginxConfigMapData("disable-http-access-log", "true")
		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)
		f.EnsureIngress(ing)

		f.WaitForNginxConfiguration(func(ngx string) bool {
			return strings.Contains(ngx, `access_log off;`)
		})
	})
})
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"
	"sigs.k8s.io/yaml"

	"knative.dev/eventing/pkg/apis/config"
	eventingduck "knative.dev/eventing/pkg/apis/duck/v1"
	eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1"
	testlib "knative.dev/eventing/test/lib"
	"knative.dev/pkg/reconciler"
	"knative.dev/pkg/system"
)

func TestBrokerNamespaceDefaulting(t *testing.T) {
	ctx := context.Background()

	c := testlib.Setup(t, true)
	defer testlib.TearDown(c)

	err := reconciler.RetryTestErrors(func(attempt int) error {

		t.Log("Updating defaulting ConfigMap attempt:", attempt)

		cm, err := c.Kube.CoreV1().ConfigMaps(system.Namespace()).Get(ctx, config.DefaultsConfigName, metav1.GetOptions{})
		assert.Nil(t, err)

		// Preserve existing namespace defaults.
		defaults := make(map[string]map[string]interface{})
		err = yaml.Unmarshal([]byte(cm.Data[config.BrokerDefaultsKey]), &defaults)
		assert.Nil(t, err)

		if _, ok := defaults["namespaceDefaults"]; !ok {
			defaults["namespaceDefaults"] = make(map[string]interface{})
		}

		defaults["namespaceDefaults"][c.Namespace] = map[string]interface{}{
			"apiVersion":  "v1",
			"kind":        "ConfigMap",
			"name":        "config-br-default-channel",
			"namespace":   "knative-eventing",
			"brokerClass": brokerClass,
			"delivery": map[string]interface{}{
				"retry":         5,
				"backoffPolicy": "exponential",
				"backoffDelay":  "PT0.5S",
			},
		}

		b, err := yaml.Marshal(defaults)
		assert.Nil(t, err)

		cm.Data[config.BrokerDefaultsKey] = string(b)

		cm, err = c.Kube.CoreV1().ConfigMaps(system.Namespace()).Update(ctx, cm, metav1.UpdateOptions{})
		if err != nil {
			return err
		}

		b, err = yaml.Marshal(cm.Data[config.BrokerDefaultsKey])
		if err != nil {
			t.Log("error", err)
		} else {
			t.Log("CM updated - new values:", string(b))
		}

		return nil
	})
	assert.Nil(t, err)

	// Create a Broker and check whether it has the DeliverySpec set as we specified above.
	// Since the webhook receives the updates at some undetermined time after the update to reduce flakiness retry after
	// a delay and check whether it has the shape we want it to have.

	namePrefix := "xyz"
	n := 0
	lastName := ""

	backoff := wait.Backoff{
		Duration: time.Second,
		Factor:   1.0,
		Jitter:   0.1,
		Steps:    5,
	}

	err = retry.OnError(backoff, func(err error) bool { return err != nil }, func() error {

		name := fmt.Sprintf("%s-%d", namePrefix, n)
		lastName = name

		obj := &unstructured.Unstructured{
			Object: map[string]interface{}{
				"apiVersion": "eventing.knative.dev/v1",
				"kind":       "Broker",
				"metadata": map[string]interface{}{
					"name":      name,
					"namespace": c.Namespace,
				},
			},
		}

		createdObj, err := c.Dynamic.
			Resource(schema.GroupVersionResource{Group: "eventing.knative.dev", Version: "v1", Resource: "brokers"}).
			Namespace(c.Namespace).
			Create(ctx, obj, metav1.CreateOptions{})
		assert.Nil(t, err)
		n = n + 1

		broker := &eventingv1.Broker{}
		err = runtime.DefaultUnstructuredConverter.FromUnstructured(createdObj.Object, broker)
		assert.Nil(t, err)

		if !webhookObservedBrokerUpdate(broker) {
			return fmt.Errorf("webhook hasn't seen the update: %+v", broker)
		}

		assert.Equal(t, brokerClass, broker.Annotations[eventingv1.BrokerClassAnnotationKey])

		if err != nil || !webhookObservedBrokerUpdateFromDeliverySpec(broker.Spec.Delivery) {
			return fmt.Errorf("webhook hasn't seen the update: %+v", broker.Spec.Delivery)
		}

		assert.Equal(t, "PT0.5S", *broker.Spec.Delivery.BackoffDelay)
		assert.Equal(t, int32(5), *broker.Spec.Delivery.Retry)
		assert.Equal(t, eventingduck.BackoffPolicyExponential, *broker.Spec.Delivery.BackoffPolicy)

		return nil
	})
	assert.Nil(t, err)

	err = wait.Poll(time.Second, time.Minute, func() (done bool, err error) {
		foundBroker, err := c.Eventing.EventingV1().Brokers(c.Namespace).Get(ctx, lastName, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return false, nil
		}
		assert.Nil(t, err)

		assert.Equal(t, brokerClass, foundBroker.Annotations[eventingv1.BrokerClassAnnotationKey])
		assert.Equal(t, "PT0.5S", *foundBroker.Spec.Delivery.BackoffDelay)
		assert.Equal(t, int32(5), *foundBroker.Spec.Delivery.Retry)
		assert.Equal(t, eventingduck.BackoffPolicyExponential, *foundBroker.Spec.Delivery.BackoffPolicy)

		return true, nil
	})
	assert.Nil(t, err)
}

func webhookObservedBrokerUpdate(br *eventingv1.Broker) bool {
	_, ok := br.Annotations[eventingv1.BrokerClassAnnotationKey]
	return ok
}

func webhookObservedBrokerUpdateFromDeliverySpec(d *eventingduck.DeliverySpec) bool {
	return d != nil && d.BackoffDelay != nil && d.Retry != nil && d.BackoffPolicy != nil
}
<commit_msg>[#4717] Update e2e broker defaults webhook test (#4720)<commit_after>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"
	"sigs.k8s.io/yaml"

	"knative.dev/eventing/pkg/apis/config"
	eventingduck "knative.dev/eventing/pkg/apis/duck/v1"
	eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1"
	testlib "knative.dev/eventing/test/lib"
	"knative.dev/pkg/reconciler"
	"knative.dev/pkg/system"
)

func TestBrokerNamespaceDefaulting(t *testing.T) {
	ctx := context.Background()

	c := testlib.Setup(t, true)
	defer testlib.TearDown(c)

	err := reconciler.RetryTestErrors(func(attempt int) error {

		t.Log("Updating defaulting ConfigMap attempt:", attempt)

		cm, err := c.Kube.CoreV1().ConfigMaps(system.Namespace()).Get(ctx, config.DefaultsConfigName, metav1.GetOptions{})
		assert.Nil(t, err)

		// Preserve existing namespace defaults.
		defaults := make(map[string]map[string]interface{})
		err = yaml.Unmarshal([]byte(cm.Data[config.BrokerDefaultsKey]), &defaults)
		assert.Nil(t, err)

		if _, ok := defaults["namespaceDefaults"]; !ok {
			defaults["namespaceDefaults"] = make(map[string]interface{})
		}

		defaults["namespaceDefaults"][c.Namespace] = map[string]interface{}{
			"apiVersion":  "v1",
			"kind":        "ConfigMap",
			"name":        "config-br-default-channel",
			"namespace":   "knative-eventing",
			"brokerClass": brokerClass,
			"delivery": map[string]interface{}{
				"retry":         5,
				"backoffPolicy": "exponential",
				"backoffDelay":  "PT0.5S",
			},
		}

		b, err := yaml.Marshal(defaults)
		assert.Nil(t, err)

		cm.Data[config.BrokerDefaultsKey] = string(b)

		cm, err = c.Kube.CoreV1().ConfigMaps(system.Namespace()).Update(ctx, cm, metav1.UpdateOptions{})
		if err != nil {
			return err
		}

		b, err = yaml.Marshal(cm.Data[config.BrokerDefaultsKey])
		if err != nil {
			t.Log("error", err)
		} else {
			t.Log("CM updated - new values:", string(b))
		}

		return nil
	})
	assert.Nil(t, err)

	// Create a Broker and check whether it has the DeliverySpec set as we specified above.
	// Since the webhook receives the updates at some undetermined time after the update to reduce flakiness retry after
	// a delay and check whether it has the shape we want it to have.

	namePrefix := "xyz"
	n := 0
	lastName := ""

	backoff := wait.Backoff{
		Duration: time.Second,
		Factor:   1.0,
		Jitter:   0.1,
		Steps:    5,
	}

	err = retry.OnError(backoff, func(err error) bool { return err != nil }, func() error {

		name := fmt.Sprintf("%s-%d", namePrefix, n)
		lastName = name

		obj := &unstructured.Unstructured{
			Object: map[string]interface{}{
				"apiVersion": "eventing.knative.dev/v1",
				"kind":       "Broker",
				"metadata": map[string]interface{}{
					"name":      name,
					"namespace": c.Namespace,
				},
			},
		}

		createdObj, err := c.Dynamic.
			Resource(schema.GroupVersionResource{Group: "eventing.knative.dev", Version: "v1", Resource: "brokers"}).
			Namespace(c.Namespace).
			Create(ctx, obj, metav1.CreateOptions{})
		require.Nil(t, err)
		n = n + 1

		broker := &eventingv1.Broker{}
		err = runtime.DefaultUnstructuredConverter.FromUnstructured(createdObj.Object, broker)
		require.Nil(t, err)

		if !webhookObservedBrokerUpdate(broker) {
			return fmt.Errorf("webhook hasn't seen the update: %+v", broker)
		}

		assert.Equal(t, brokerClass, broker.Annotations[eventingv1.BrokerClassAnnotationKey])

		if err != nil || !webhookObservedBrokerUpdateFromDeliverySpec(broker.Spec.Delivery) {
			return fmt.Errorf("webhook hasn't seen the update: %+v", broker.Spec.Delivery)
		}

		assert.Equal(t, "PT0.5S", *broker.Spec.Delivery.BackoffDelay)
		assert.Equal(t, int32(5), *broker.Spec.Delivery.Retry)
		assert.Equal(t, eventingduck.BackoffPolicyExponential, *broker.Spec.Delivery.BackoffPolicy)

		return nil
	})
	assert.Nil(t, err)

	err = wait.Poll(time.Second, time.Minute, func() (done bool, err error) {
		foundBroker, err := c.Eventing.EventingV1().Brokers(c.Namespace).Get(ctx, lastName, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return false, nil
		}
		require.Nil(t, err)

		assert.Equal(t, brokerClass, foundBroker.Annotations[eventingv1.BrokerClassAnnotationKey])
		assert.Equal(t, "PT0.5S", *foundBroker.Spec.Delivery.BackoffDelay)
		assert.Equal(t, int32(5), *foundBroker.Spec.Delivery.Retry)
		assert.Equal(t, eventingduck.BackoffPolicyExponential, *foundBroker.Spec.Delivery.BackoffPolicy)

		return true, nil
	})
	assert.Nil(t, err)
}

func webhookObservedBrokerUpdate(br *eventingv1.Broker) bool {
	_, ok := br.Annotations[eventingv1.BrokerClassAnnotationKey]
	return ok
}

func webhookObservedBrokerUpdateFromDeliverySpec(d *eventingduck.DeliverySpec) bool {
	return d != nil && d.BackoffDelay != nil && d.Retry != nil && d.BackoffPolicy != nil
}
<|endoftext|>