<commit_before>/*
offheap

An off-heap hash-table for Go (golang). Originally called go-offheap-hashtable,
but now shortened to just offheap.

The purpose here is to have hash table that can work away
from Go's Garbage Collector, to avoid long GC pause times.

We accomplish this by writing our own Malloc() and Free() implementation
(see malloc.go) which requests memory directly from the OS.

The keys, values, and entire hash table is kept on off-heap
storage. This storage can also optionally be backed by memory mapped file
for speedy persistence and fast startup times.

Initial HashTable implementation inspired by the public domain C++ code of
    https://github.com/preshing/CompareIntegerMaps
See also
    http://preshing.com/20130107/this-hash-table-is-faster-than-a-judy-array/
for performance studies of the C++ code.

HashTable

The implementation is mostly in offheap.go, read that to start.

Maps pointer-sized integers to Cell structures, which in turn hold Val_t
as well as Key_t structures.

Uses open addressing with linear probing. This makes it very cache
friendly and thus very fast.

In the t.Cells array, UnHashedKey = 0 is reserved to indicate an unused cell.
Actual value for key 0 (if any) is stored in t.ZeroCell.
The hash table automatically doubles in size when it becomes 75% full.
The hash table never shrinks in size, even after Clear(), unless you explicitly
call Compact().

Basic operations: Lookup(), Insert(), DeleteKey(). These are the
equivalent of the builtin map[uint64]interface{}.

As an example of how to specialize for a map[string]*Cell equivalent,
see the following functions in the bytekey.go file:

    func (t *HashTable) InsertStringKey(strkey string, value interface{}) bool
    func (t *HashTable) LookupStringKey(strkey string) (Val_t, bool)
    func (t *HashTable) DeleteStringKey(strkey string) bool

Example use:

    h := offheap.NewHashTable(2)

    // basic three operations are:
    h.InsertStringKey("My number", 43)
    val, ok := h.LookupStringKey("My number")
    h.DeleteStringKey("My number")

Note that this library is only a starting point of source code, and not intended to be used without customization. Users of the HashTable will have to customize it by changing the definitions of Key_t and Val_t to suite their needs. I'm experimenting next with storing objects in Capnproto serialized format, but this branch (branch capnp) isn't quite ready for use.

Related ideas:

https://gist.github.com/mish15/9822474

https://groups.google.com/forum/#!topic/golang-nuts/kCQP6S6ZGh0

*/
package offheap
<commit_msg>atg. doc++<commit_after>/*

Offheap

An off-heap hash-table for Go (golang). Originally called go-offheap-hashtable,
but now shortened to just offheap.

The purpose here is to have hash table that can work away
from Go's Garbage Collector, to avoid long GC pause times.

We accomplish this by writing our own Malloc() and Free() implementation
(see malloc.go) which requests memory directly from the OS.

The keys, values, and entire hash table is kept on off-heap
storage. This storage can also optionally be backed by memory mapped file
for speedy persistence and fast startup times.

Initial HashTable implementation inspired by the public domain C++ code of
    https://github.com/preshing/CompareIntegerMaps
See also
    http://preshing.com/20130107/this-hash-table-is-faster-than-a-judy-array/
for performance studies of the C++ code.

HashTable

The implementation is mostly in offheap.go, read that to start.

Maps pointer-sized integers to Cell structures, which in turn hold Val_t
as well as Key_t structures.

Uses open addressing with linear probing. This makes it very cache
friendly and thus very fast.

In the t.Cells array, UnHashedKey = 0 is reserved to indicate an unused cell.
Actual value for key 0 (if any) is stored in t.ZeroCell.
The hash table automatically doubles in size when it becomes 75% full.
The hash table never shrinks in size, even after Clear(), unless you explicitly
call Compact().

Basic operations: Lookup(), Insert(), DeleteKey(). These are the
equivalent of the builtin map[uint64]interface{}.

As an example of how to specialize for a map[string]*Cell equivalent,
see the following functions in the bytekey.go file:

    func (t *HashTable) InsertStringKey(strkey string, value interface{}) bool
    func (t *HashTable) LookupStringKey(strkey string) (Val_t, bool)
    func (t *HashTable) DeleteStringKey(strkey string) bool

Example use:

    h := offheap.NewHashTable(2)

    // basic three operations are:
    h.InsertStringKey("My number", 43)
    val, ok := h.LookupStringKey("My number")
    h.DeleteStringKey("My number")

Note that this library is only a starting point of source code, and not intended to be used without customization. Users of the HashTable will have to customize it by changing the definitions of Key_t and Val_t to suite their needs. I'm experimenting next with storing objects in Capnproto serialized format, but this branch (branch capnp) isn't quite ready for use.

Related ideas:

https://gist.github.com/mish15/9822474

https://groups.google.com/forum/#!topic/golang-nuts/kCQP6S6ZGh0

*/
package offheap
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	gosignal "os/signal"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/api"
	"github.com/docker/docker/autogen/dockerversion"
	"github.com/docker/docker/engine"
	"github.com/docker/docker/pkg/signal"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/docker/docker/pkg/term"
	"github.com/docker/docker/registry"
	"github.com/docker/docker/utils"
)

var (
	ErrConnectionRefused = errors.New("Cannot connect to the Docker daemon. Is 'docker -d' running on this host?")
)

func (cli *DockerCli) HTTPClient() *http.Client {
	return &http.Client{Transport: cli.transport}
}

func (cli *DockerCli) encodeData(data interface{}) (*bytes.Buffer, error) {
	params := bytes.NewBuffer(nil)
	if data != nil {
		if env, ok := data.(engine.Env); ok {
			if err := env.Encode(params); err != nil {
				return nil, err
			}
		} else {
			buf, err := json.Marshal(data)
			if err != nil {
				return nil, err
			}
			if _, err := params.Write(buf); err != nil {
				return nil, err
			}
		}
	}
	return params, nil
}

func (cli *DockerCli) clientRequest(method, path string, in io.Reader, headers map[string][]string) (io.ReadCloser, string, int, error) {
	expectedPayload := (method == "POST" || method == "PUT")
	if expectedPayload && in == nil {
		in = bytes.NewReader([]byte{})
	}
	req, err := http.NewRequest(method, fmt.Sprintf("/v%s%s", api.APIVERSION, path), in)
	if err != nil {
		return nil, "", -1, err
	}
	req.Header.Set("User-Agent", "Docker-Client/"+dockerversion.VERSION)
	req.URL.Host = cli.addr
	req.URL.Scheme = cli.scheme
	if headers != nil {
		for k, v := range headers {
			req.Header[k] = v
		}
	}
	if expectedPayload && req.Header.Get("Content-Type") == "" {
		req.Header.Set("Content-Type", "text/plain")
	}

	resp, err := cli.HTTPClient().Do(req)
	statusCode := -1
	if resp != nil {
		statusCode = resp.StatusCode
	}
	if err != nil {
		if strings.Contains(err.Error(), "connection refused") {
			return nil, "", statusCode, ErrConnectionRefused
		}

		if cli.tlsConfig == nil {
			return nil, "", statusCode, fmt.Errorf("%v. Are you trying to connect to a TLS-enabled daemon without TLS?", err)
		}

		return nil, "", statusCode, fmt.Errorf("An error occurred trying to connect: %v", err)
	}

	if statusCode < 200 || statusCode >= 400 {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, "", statusCode, err
		}
		if len(body) == 0 {
			return nil, "", statusCode, fmt.Errorf("Error: request returned %s for API route and version %s, check if the server supports the requested API version", http.StatusText(statusCode), req.URL)
		}
		return nil, "", statusCode, fmt.Errorf("Error response from daemon: %s", bytes.TrimSpace(body))
	}

	return resp.Body, resp.Header.Get("Content-Type"), statusCode, nil
}

func (cli *DockerCli) clientRequestAttemptLogin(method, path string, in io.Reader, out io.Writer, index *registry.IndexInfo, cmdName string) (io.ReadCloser, int, error) {
	cmdAttempt := func(authConfig registry.AuthConfig) (io.ReadCloser, int, error) {
		buf, err := json.Marshal(authConfig)
		if err != nil {
			return nil, -1, err
		}
		registryAuthHeader := []string{
			base64.URLEncoding.EncodeToString(buf),
		}

		// begin the request
		body, contentType, statusCode, err := cli.clientRequest(method, path, in, map[string][]string{
			"X-Registry-Auth": registryAuthHeader,
		})
		if err == nil && out != nil {
			// If we are streaming output, complete the stream since
			// errors may not appear until later.
			err = cli.streamBody(body, contentType, true, out, nil)
		}
		if err != nil {
			// Since errors in a stream appear after status 200 has been written,
			// we may need to change the status code.
			if strings.Contains(err.Error(), "Authentication is required") ||
				strings.Contains(err.Error(), "Status 401") ||
				strings.Contains(err.Error(), "status code 401") {
				statusCode = http.StatusUnauthorized
			}
		}
		return body, statusCode, err
	}

	// Resolve the Auth config relevant for this server
	authConfig := cli.configFile.ResolveAuthConfig(index)
	body, statusCode, err := cmdAttempt(authConfig)
	if statusCode == http.StatusUnauthorized {
		fmt.Fprintf(cli.out, "\nPlease login prior to %s:\n", cmdName)
		if err = cli.CmdLogin(index.GetAuthConfigKey()); err != nil {
			return nil, -1, err
		}
		authConfig = cli.configFile.ResolveAuthConfig(index)
		return cmdAttempt(authConfig)
	}
	return body, statusCode, err
}

func (cli *DockerCli) call(method, path string, data interface{}, headers map[string][]string) (io.ReadCloser, int, error) {
	params, err := cli.encodeData(data)
	if err != nil {
		return nil, -1, err
	}

	if data != nil {
		if headers == nil {
			headers = make(map[string][]string)
		}
		headers["Content-Type"] = []string{"application/json"}
	}

	body, _, statusCode, err := cli.clientRequest(method, path, params, headers)
	return body, statusCode, err
}
func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, headers map[string][]string) error {
	return cli.streamHelper(method, path, true, in, out, nil, headers)
}

func (cli *DockerCli) streamHelper(method, path string, setRawTerminal bool, in io.Reader, stdout, stderr io.Writer, headers map[string][]string) error {
	body, contentType, _, err := cli.clientRequest(method, path, in, headers)
	if err != nil {
		return err
	}
	return cli.streamBody(body, contentType, setRawTerminal, stdout, stderr)
}

func (cli *DockerCli) streamBody(body io.ReadCloser, contentType string, setRawTerminal bool, stdout, stderr io.Writer) error {
	defer body.Close()

	if api.MatchesContentType(contentType, "application/json") {
		return utils.DisplayJSONMessagesStream(body, stdout, cli.outFd, cli.isTerminalOut)
	}
	if stdout != nil || stderr != nil {
		// When TTY is ON, use regular copy
		var err error
		if setRawTerminal {
			_, err = io.Copy(stdout, body)
		} else {
			_, err = stdcopy.StdCopy(stdout, stderr, body)
		}
		log.Debugf("[stream] End of stdout")
		return err
	}
	return nil
}

func (cli *DockerCli) resizeTty(id string, isExec bool) {
	height, width := cli.getTtySize()
	if height == 0 && width == 0 {
		return
	}
	v := url.Values{}
	v.Set("h", strconv.Itoa(height))
	v.Set("w", strconv.Itoa(width))

	path := ""
	if !isExec {
		path = "/containers/" + id + "/resize?"
	} else {
		path = "/exec/" + id + "/resize?"
	}

	if _, _, err := readBody(cli.call("POST", path+v.Encode(), nil, nil)); err != nil {
		log.Debugf("Error resize: %s", err)
	}
}

func waitForExit(cli *DockerCli, containerID string) (int, error) {
	stream, _, err := cli.call("POST", "/containers/"+containerID+"/wait", nil, nil)
	if err != nil {
		return -1, err
	}

	var out engine.Env
	if err := out.Decode(stream); err != nil {
		return -1, err
	}
	return out.GetInt("StatusCode"), nil
}

// getExitCode perform an inspect on the container. It returns
// the running state and the exit code.
func getExitCode(cli *DockerCli, containerID string) (bool, int, error) {
	stream, _, err := cli.call("GET", "/containers/"+containerID+"/json", nil, nil)
	if err != nil {
		// If we can't connect, then the daemon probably died.
		if err != ErrConnectionRefused {
			return false, -1, err
		}
		return false, -1, nil
	}

	var result engine.Env
	if err := result.Decode(stream); err != nil {
		return false, -1, err
	}

	state := result.GetSubEnv("State")
	return state.GetBool("Running"), state.GetInt("ExitCode"), nil
}

// getExecExitCode perform an inspect on the exec command. It returns
// the running state and the exit code.
func getExecExitCode(cli *DockerCli, execID string) (bool, int, error) {
	stream, _, err := cli.call("GET", "/exec/"+execID+"/json", nil, nil)
	if err != nil {
		// If we can't connect, then the daemon probably died.
		if err != ErrConnectionRefused {
			return false, -1, err
		}
		return false, -1, nil
	}

	var result engine.Env
	if err := result.Decode(stream); err != nil {
		return false, -1, err
	}

	return result.GetBool("Running"), result.GetInt("ExitCode"), nil
}

func (cli *DockerCli) monitorTtySize(id string, isExec bool) error {
	cli.resizeTty(id, isExec)

	sigchan := make(chan os.Signal, 1)
	gosignal.Notify(sigchan, signal.SIGWINCH)
	go func() {
		for _ = range sigchan {
			cli.resizeTty(id, isExec)
		}
	}()
	return nil
}

func (cli *DockerCli) getTtySize() (int, int) {
	if !cli.isTerminalOut {
		return 0, 0
	}
	ws, err := term.GetWinsize(cli.outFd)
	if err != nil {
		log.Debugf("Error getting size: %s", err)
		if ws == nil {
			return 0, 0
		}
	}
	return int(ws.Height), int(ws.Width)
}

func readBody(stream io.ReadCloser, statusCode int, err error) ([]byte, int, error) {
	if stream != nil {
		defer stream.Close()
	}
	if err != nil {
		return nil, statusCode, err
	}
	body, err := ioutil.ReadAll(stream)
	if err != nil {
		return nil, -1, err
	}
	return body, statusCode, nil
}
<commit_msg>windows: monitorTtySize correctly by polling<commit_after>package client

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	gosignal "os/signal"
	"runtime"
	"strconv"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/api"
	"github.com/docker/docker/autogen/dockerversion"
	"github.com/docker/docker/engine"
	"github.com/docker/docker/pkg/signal"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/docker/docker/pkg/term"
	"github.com/docker/docker/registry"
	"github.com/docker/docker/utils"
)

var (
	ErrConnectionRefused = errors.New("Cannot connect to the Docker daemon. Is 'docker -d' running on this host?")
)

func (cli *DockerCli) HTTPClient() *http.Client {
	return &http.Client{Transport: cli.transport}
}

func (cli *DockerCli) encodeData(data interface{}) (*bytes.Buffer, error) {
	params := bytes.NewBuffer(nil)
	if data != nil {
		if env, ok := data.(engine.Env); ok {
			if err := env.Encode(params); err != nil {
				return nil, err
			}
		} else {
			buf, err := json.Marshal(data)
			if err != nil {
				return nil, err
			}
			if _, err := params.Write(buf); err != nil {
				return nil, err
			}
		}
	}
	return params, nil
}

func (cli *DockerCli) clientRequest(method, path string, in io.Reader, headers map[string][]string) (io.ReadCloser, string, int, error) {
	expectedPayload := (method == "POST" || method == "PUT")
	if expectedPayload && in == nil {
		in = bytes.NewReader([]byte{})
	}
	req, err := http.NewRequest(method, fmt.Sprintf("/v%s%s", api.APIVERSION, path), in)
	if err != nil {
		return nil, "", -1, err
	}
	req.Header.Set("User-Agent", "Docker-Client/"+dockerversion.VERSION)
	req.URL.Host = cli.addr
	req.URL.Scheme = cli.scheme
	if headers != nil {
		for k, v := range headers {
			req.Header[k] = v
		}
	}
	if expectedPayload && req.Header.Get("Content-Type") == "" {
		req.Header.Set("Content-Type", "text/plain")
	}

	resp, err := cli.HTTPClient().Do(req)
	statusCode := -1
	if resp != nil {
		statusCode = resp.StatusCode
	}
	if err != nil {
		if strings.Contains(err.Error(), "connection refused") {
			return nil, "", statusCode, ErrConnectionRefused
		}

		if cli.tlsConfig == nil {
			return nil, "", statusCode, fmt.Errorf("%v. Are you trying to connect to a TLS-enabled daemon without TLS?", err)
		}

		return nil, "", statusCode, fmt.Errorf("An error occurred trying to connect: %v", err)
	}

	if statusCode < 200 || statusCode >= 400 {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, "", statusCode, err
		}
		if len(body) == 0 {
			return nil, "", statusCode, fmt.Errorf("Error: request returned %s for API route and version %s, check if the server supports the requested API version", http.StatusText(statusCode), req.URL)
		}
		return nil, "", statusCode, fmt.Errorf("Error response from daemon: %s", bytes.TrimSpace(body))
	}

	return resp.Body, resp.Header.Get("Content-Type"), statusCode, nil
}

func (cli *DockerCli) clientRequestAttemptLogin(method, path string, in io.Reader, out io.Writer, index *registry.IndexInfo, cmdName string) (io.ReadCloser, int, error) {
	cmdAttempt := func(authConfig registry.AuthConfig) (io.ReadCloser, int, error) {
		buf, err := json.Marshal(authConfig)
		if err != nil {
			return nil, -1, err
		}
		registryAuthHeader := []string{
			base64.URLEncoding.EncodeToString(buf),
		}

		// begin the request
		body, contentType, statusCode, err := cli.clientRequest(method, path, in, map[string][]string{
			"X-Registry-Auth": registryAuthHeader,
		})
		if err == nil && out != nil {
			// If we are streaming output, complete the stream since
			// errors may not appear until later.
			err = cli.streamBody(body, contentType, true, out, nil)
		}
		if err != nil {
			// Since errors in a stream appear after status 200 has been written,
			// we may need to change the status code.
			if strings.Contains(err.Error(), "Authentication is required") ||
				strings.Contains(err.Error(), "Status 401") ||
				strings.Contains(err.Error(), "status code 401") {
				statusCode = http.StatusUnauthorized
			}
		}
		return body, statusCode, err
	}

	// Resolve the Auth config relevant for this server
	authConfig := cli.configFile.ResolveAuthConfig(index)
	body, statusCode, err := cmdAttempt(authConfig)
	if statusCode == http.StatusUnauthorized {
		fmt.Fprintf(cli.out, "\nPlease login prior to %s:\n", cmdName)
		if err = cli.CmdLogin(index.GetAuthConfigKey()); err != nil {
			return nil, -1, err
		}
		authConfig = cli.configFile.ResolveAuthConfig(index)
		return cmdAttempt(authConfig)
	}
	return body, statusCode, err
}

func (cli *DockerCli) call(method, path string, data interface{}, headers map[string][]string) (io.ReadCloser, int, error) {
	params, err := cli.encodeData(data)
	if err != nil {
		return nil, -1, err
	}

	if data != nil {
		if headers == nil {
			headers = make(map[string][]string)
		}
		headers["Content-Type"] = []string{"application/json"}
	}

	body, _, statusCode, err := cli.clientRequest(method, path, params, headers)
	return body, statusCode, err
}
func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, headers map[string][]string) error {
	return cli.streamHelper(method, path, true, in, out, nil, headers)
}

func (cli *DockerCli) streamHelper(method, path string, setRawTerminal bool, in io.Reader, stdout, stderr io.Writer, headers map[string][]string) error {
	body, contentType, _, err := cli.clientRequest(method, path, in, headers)
	if err != nil {
		return err
	}
	return cli.streamBody(body, contentType, setRawTerminal, stdout, stderr)
}

func (cli *DockerCli) streamBody(body io.ReadCloser, contentType string, setRawTerminal bool, stdout, stderr io.Writer) error {
	defer body.Close()

	if api.MatchesContentType(contentType, "application/json") {
		return utils.DisplayJSONMessagesStream(body, stdout, cli.outFd, cli.isTerminalOut)
	}
	if stdout != nil || stderr != nil {
		// When TTY is ON, use regular copy
		var err error
		if setRawTerminal {
			_, err = io.Copy(stdout, body)
		} else {
			_, err = stdcopy.StdCopy(stdout, stderr, body)
		}
		log.Debugf("[stream] End of stdout")
		return err
	}
	return nil
}

func (cli *DockerCli) resizeTty(id string, isExec bool) {
	height, width := cli.getTtySize()
	if height == 0 && width == 0 {
		return
	}
	v := url.Values{}
	v.Set("h", strconv.Itoa(height))
	v.Set("w", strconv.Itoa(width))

	path := ""
	if !isExec {
		path = "/containers/" + id + "/resize?"
	} else {
		path = "/exec/" + id + "/resize?"
	}

	if _, _, err := readBody(cli.call("POST", path+v.Encode(), nil, nil)); err != nil {
		log.Debugf("Error resize: %s", err)
	}
}

func waitForExit(cli *DockerCli, containerID string) (int, error) {
	stream, _, err := cli.call("POST", "/containers/"+containerID+"/wait", nil, nil)
	if err != nil {
		return -1, err
	}

	var out engine.Env
	if err := out.Decode(stream); err != nil {
		return -1, err
	}
	return out.GetInt("StatusCode"), nil
}

// getExitCode perform an inspect on the container. It returns
// the running state and the exit code.
func getExitCode(cli *DockerCli, containerID string) (bool, int, error) {
	stream, _, err := cli.call("GET", "/containers/"+containerID+"/json", nil, nil)
	if err != nil {
		// If we can't connect, then the daemon probably died.
		if err != ErrConnectionRefused {
			return false, -1, err
		}
		return false, -1, nil
	}

	var result engine.Env
	if err := result.Decode(stream); err != nil {
		return false, -1, err
	}

	state := result.GetSubEnv("State")
	return state.GetBool("Running"), state.GetInt("ExitCode"), nil
}

// getExecExitCode perform an inspect on the exec command. It returns
// the running state and the exit code.
func getExecExitCode(cli *DockerCli, execID string) (bool, int, error) {
	stream, _, err := cli.call("GET", "/exec/"+execID+"/json", nil, nil)
	if err != nil {
		// If we can't connect, then the daemon probably died.
		if err != ErrConnectionRefused {
			return false, -1, err
		}
		return false, -1, nil
	}

	var result engine.Env
	if err := result.Decode(stream); err != nil {
		return false, -1, err
	}

	return result.GetBool("Running"), result.GetInt("ExitCode"), nil
}

func (cli *DockerCli) monitorTtySize(id string, isExec bool) error {
	cli.resizeTty(id, isExec)

	if runtime.GOOS == "windows" {
		go func() {
			prevW, prevH := cli.getTtySize()
			for {
				time.Sleep(time.Millisecond * 250)
				w, h := cli.getTtySize()

				if prevW != w || prevH != h {
					cli.resizeTty(id, isExec)
				}
				prevW = w
				prevH = h
			}
		}()
	} else {
		sigchan := make(chan os.Signal, 1)
		gosignal.Notify(sigchan, signal.SIGWINCH)
		go func() {
			for _ = range sigchan {
				cli.resizeTty(id, isExec)
			}
		}()
	}
	return nil
}

func (cli *DockerCli) getTtySize() (int, int) {
	if !cli.isTerminalOut {
		return 0, 0
	}
	ws, err := term.GetWinsize(cli.outFd)
	if err != nil {
		log.Debugf("Error getting size: %s", err)
		if ws == nil {
			return 0, 0
		}
	}
	return int(ws.Height), int(ws.Width)
}

func readBody(stream io.ReadCloser, statusCode int, err error) ([]byte, int, error) {
	if stream != nil {
		defer stream.Close()
	}
	if err != nil {
		return nil, statusCode, err
	}
	body, err := ioutil.ReadAll(stream)
	if err != nil {
		return nil, -1, err
	}
	return body, statusCode, nil
}
<|endoftext|>


<commit_before>// Package splaytree implements the splay tree data structure,
// which is a self-balancing binary search tree, that is,
// it adapts its internal tree structure to how it is
// being used in order to optimize future operations.
// The tree contains only unique keys in sorted order.
// The self-balancing (called splaying) is done for every
// insert, lookup or remove operation. The key which is
// inserted/looked up/removed is splayed upwards to the
// root, by means of rotations over two or three nodes.
// The effect is that future accesses to this key and to
// its neighbors become cheap, as they will be at or near
// the root of the tree. Accesses to other non-neighboring
// keys diminish this benefit over time. On average the
// cost of accesses is optimal: O(log N). Splay trees
// are especially beneficial in applications which exhibit
// locality of reference. I.e. when accesses to the tree are
// related in location or time. This happens for instance
// for sequential (sorted) or clustered access patterns.
// See https://en.wikipedia.org/wiki/Splay_tree for details.
package splaytree
<commit_msg>comment<commit_after>// Package splaytree implements the splay tree data structure,
// which is a self-balancing binary search tree, that is,
// it adapts its internal tree structure to how it is
// being used in order to optimize future operations.
// The tree contains only unique keys in sorted order.
// The self-balancing (called splaying) is done for every
// insert, lookup or remove operation. Splaying is heavily
// optimized in a single loop. The key which is
// inserted/looked up/removed is splayed upwards to the
// root, by means of rotations over two or three nodes.
// The effect is that future accesses to this key and to
// its neighbors become cheap, as they will be at or near
// the root of the tree. Accesses to other non-neighboring
// keys diminish this benefit over time. On average the
// cost of accesses is optimal: O(log N). Splay trees
// are especially beneficial in applications which exhibit
// locality of reference. I.e. when accesses to the tree are
// related in location or time. This happens for instance
// for sequential (sorted) or clustered access patterns.
// See https://en.wikipedia.org/wiki/Splay_tree for details.
package splaytree
<|endoftext|>


<commit_before>/*
Package assert provides some basic assertion functions for testing and
also provides the building blocks for creating your own more complex
validations.

	package whatever

	import (
		"errors"
		"testing"
	)

	func AssertCustomErrorHandler(t *testing.T, errs map[string]string, key, expected string) {
		val, ok := errs[key]

		// using EqualSkip and NotEqualSkip as building blocks for my custom Assert function
		EqualSkip(t, 2, ok, true)
		NotEqualSkip(t, 2, val, nil)
		EqualSkip(t, 2, val, expected)
	}

	func TestEqual(t *testing.T) {

		// error comes from your package/library
		err := errors.New("my error")
		NotEqual(t, err, nil)
		Equal(t, err.Error(), "my error")

		err = nil
		Equal(t, err, nil)

		fn := func() {
			panic("omg omg omg!")
		}

		PanicMatches(t, func() { fn() }, "omg omg omg!")
		PanicMatches(t, func() { panic("omg omg omg!") }, "omg omg omg!")

		// errs would have come from your package/library
		errs := map[string]string{}
		errs["Name"] = "User Name Invalid"
		errs["Email"] = "User Email Invalid"

		AssertCustomErrorHandler(t, errs, "Name", "User Name Invalid")
		AssertCustomErrorHandler(t, errs, "Email", "User Email Invalid")
	}
*/
package assert
<commit_msg>Update doc.go<commit_after>/*
Package assert provides some basic assertion functions for testing and
also provides the building blocks for creating your own more complex
validations.

	package whatever

	import (
		"errors"
		"testing"
		. "gopkg.in/bluesuncorp/assert.v1"
	)

	func AssertCustomErrorHandler(t *testing.T, errs map[string]string, key, expected string) {
		val, ok := errs[key]

		// using EqualSkip and NotEqualSkip as building blocks for my custom Assert function
		EqualSkip(t, 2, ok, true)
		NotEqualSkip(t, 2, val, nil)
		EqualSkip(t, 2, val, expected)
	}

	func TestEqual(t *testing.T) {

		// error comes from your package/library
		err := errors.New("my error")
		NotEqual(t, err, nil)
		Equal(t, err.Error(), "my error")

		err = nil
		Equal(t, err, nil)

		fn := func() {
			panic("omg omg omg!")
		}

		PanicMatches(t, func() { fn() }, "omg omg omg!")
		PanicMatches(t, func() { panic("omg omg omg!") }, "omg omg omg!")

		// errs would have come from your package/library
		errs := map[string]string{}
		errs["Name"] = "User Name Invalid"
		errs["Email"] = "User Email Invalid"

		AssertCustomErrorHandler(t, errs, "Name", "User Name Invalid")
		AssertCustomErrorHandler(t, errs, "Email", "User Email Invalid")
	}
*/
package assert
<|endoftext|>


<commit_before>/*
	Package embd provides a superheroic hardware abstraction layer for doing embedded programming
	on supported platforms like the Raspberry Pi and BeagleBone Black. Most of the examples below
	will work without change (i.e. the same binary) on all supported platforms. How cool is that?

	Although samples are all present in the samples folder, we will show a few choice examples here.

	Use the LED driver to toggle LEDs on the BBB:

		import "github.com/kidoman/embd"
		...
		embd.InitLED()
		defer embd.CloseLED()
		...
		led, err := embd.NewLED("USR3")
		...
		led.Toggle()

	Even shorter while prototyping:

		import "github.com/kidoman/embd"
		...
		embd.InitLED()
		defer embd.CloseLED()
		...
		embd.ToggleLED(3)

	BBB + **PWM**:

		import "github.com/kidoman/embd"
		...
		embd.InitGPIO()
		defer embd.CloseGPIO()
		...
		pwm, _ := embd.NewPWMPin("P9_14")
		defer pwm.Close()
		...
		pwm.SetDuty(1000)

	Control GPIO pins on the RaspberryPi / BeagleBone Black:

		import "github.com/kidoman/embd"
		...
		embd.InitGPIO()
		defer embd.CloseGPIO()
		...
		embd.SetDirection(10, embd.Out)
		embd.DigitalWrite(10, embd.High)

	Could also do:

		import "github.com/kidoman/embd"
		...
		embd.InitGPIO()
		defer embd.CloseGPIO()
		...
		pin, err := embd.NewDigitalPin(10)
		...
		pin.SetDirection(embd.Out)
		pin.Write(embd.High)

	Or read data from the Bosch BMP085 barometric sensor:

		import "github.com/kidoman/embd"
		import "github.com/kidoman/embd/sensor/bmp085"
		...
		bus := embd.NewI2CBus(1)
		...
		baro := bmp085.New(bus)
		...
		temp, err := baro.Temperature()
		altitude, err := baro.Altitude()

	Even find out the heading from the LSM303 magnetometer:

		import "github.com/kidoman/embd"
		import "github.com/kidoman/embd/sensor/lsm303"
		...
		bus := embd.NewI2CBus(1)
		...
		mag := lsm303.New(bus)
		...
		heading, err := mag.Heading()

	The above two examples depend on I2C and therefore will work without change on almost all
	platforms.
*/
package embd
<commit_msg>doc: remove markdown from .go doc<commit_after>/*
	Package embd provides a superheroic hardware abstraction layer for doing embedded programming
	on supported platforms like the Raspberry Pi and BeagleBone Black. Most of the examples below
	will work without change (i.e. the same binary) on all supported platforms. How cool is that?

	Although samples are all present in the samples folder, we will show a few choice examples here.

	Use the LED driver to toggle LEDs on the BBB:

		import "github.com/kidoman/embd"
		...
		embd.InitLED()
		defer embd.CloseLED()
		...
		led, err := embd.NewLED("USR3")
		...
		led.Toggle()

	Even shorter while prototyping:

		import "github.com/kidoman/embd"
		...
		embd.InitLED()
		defer embd.CloseLED()
		...
		embd.ToggleLED(3)

	BBB + PWM:

		import "github.com/kidoman/embd"
		...
		embd.InitGPIO()
		defer embd.CloseGPIO()
		...
		pwm, _ := embd.NewPWMPin("P9_14")
		defer pwm.Close()
		...
		pwm.SetDuty(1000)

	Control GPIO pins on the RaspberryPi / BeagleBone Black:

		import "github.com/kidoman/embd"
		...
		embd.InitGPIO()
		defer embd.CloseGPIO()
		...
		embd.SetDirection(10, embd.Out)
		embd.DigitalWrite(10, embd.High)

	Could also do:

		import "github.com/kidoman/embd"
		...
		embd.InitGPIO()
		defer embd.CloseGPIO()
		...
		pin, err := embd.NewDigitalPin(10)
		...
		pin.SetDirection(embd.Out)
		pin.Write(embd.High)

	Or read data from the Bosch BMP085 barometric sensor:

		import "github.com/kidoman/embd"
		import "github.com/kidoman/embd/sensor/bmp085"
		...
		bus := embd.NewI2CBus(1)
		...
		baro := bmp085.New(bus)
		...
		temp, err := baro.Temperature()
		altitude, err := baro.Altitude()

	Even find out the heading from the LSM303 magnetometer:

		import "github.com/kidoman/embd"
		import "github.com/kidoman/embd/sensor/lsm303"
		...
		bus := embd.NewI2CBus(1)
		...
		mag := lsm303.New(bus)
		...
		heading, err := mag.Heading()

	The above two examples depend on I2C and therefore will work without change on almost all
	platforms.
*/
package embd
<|endoftext|>


<commit_before>/*
simpledb is a simple database which provides CRUD and search operations on records stored in Redis.
*/
package simpledb
<commit_msg>Update package description.<commit_after>/*
simpledb is a Golang package which provides CRUD and search operations on records stored in Redis. It's based on Redigo.
*/
package simpledb
<|endoftext|>


<commit_before>package main

import (
	"agent/handler"
	. "agent/types"
	"fmt"
	"lib/packet"
	"lib/utils"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/peterbourgon/g2s"
)

const (
	StatsdPrefix      = "API."
	EnvStatsd         = "STATSD_HOST"
	DefaultStatsdHost = "172.17.42.1:8125"
)

var _statter g2s.Statter

func init() {
	addr := DefaultStatsdHost
	if env := os.Getenv(EnvStatsd); env != "" {
		addr = env
	}

	s, err := g2s.Dial("udp", addr)
	if err != nil {
		log.Fatal(err)
		os.Exit(-1)
	}
	_statter = s
}

// client protocol handle proxy
func proxyUserRequest(sess *Session, p []byte) []byte {
	start := time.Now()
	defer utils.PrintPanicStack()

	//解密
	if sess.Flag&SessEncrypt != 0 {
		sess.Decoder.XORKeyStream(p, p)
	}

	//封装为reader
	reader := packet.Reader(p)

	// 读客户端数据包序列号(1,2,3...)
	// 客户端发送的数据包必须包含一个自增的序号，必须严格递增
	// 加密后，可避免重放攻击-REPLAY-ATTACK
	seq_id, err := reader.ReadU32()
	if err != nil {
		log.Error("read client timestamp failed:", err)
		sess.Flag |= SessKickOut
		return nil
	}

	// 数据包序列号验证
	if seq_id != sess.PacketCount {
		log.Errorf("illegal packet sequence id:%v should be:%v size:%v", seq_id, sess.PacketCount, len(p)-6)
		sess.Flag |= SessKickOut
		return nil
	}

	// 读协议号
	b, err := reader.ReadS16()
	if err != nil {
		log.Error("read protocol number failed.")
		sess.Flag |= SessKickOut
		return nil
	}

	log.Debugf("msg seq_id: %v msgid: %v ", seq_id, b)

	// 根据协议号断做服务划分
	// 协议号的划分采用分割协议区间, 用户可以自定义多个区间，用于转发到不同的后端服务
	var ret []byte
	if b > MaxProtoNum {
		if err := forward(sess, p[4:]); err != nil {
			log.Errorf("service id:%v execute failed, error:%v", b, err)
			sess.Flag |= SessKickOut
			return nil
		}
	} else {
		if h := handler.Handlers[b]; h != nil {
			ret = h(sess, reader)
		} else {
			log.Errorf("service id:%v not bind", b)
			sess.Flag |= SessKickOut
			return nil
		}
	}

	// 监控协议处理时间
	// 监控数值会发送到statsd,格式为:
	// API.XXX_REQ = 10ms
	elasped := time.Now().Sub(start)
	if b != 0 { // 排除心跳包日志
		log.Debug("[REQ]", handler.RCode[b])
		_statter.Timing(1.0, fmt.Sprintf("%v%v", StatsdPrefix, handler.RCode[b]), elasped)
	}

	return ret
}
<commit_msg>log req elasped<commit_after>package main

import (
	"agent/handler"
	. "agent/types"
	"fmt"
	"lib/packet"
	"lib/utils"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/peterbourgon/g2s"
)

const (
	StatsdPrefix      = "API."
	EnvStatsd         = "STATSD_HOST"
	DefaultStatsdHost = "172.17.42.1:8125"
)

var _statter g2s.Statter

func init() {
	addr := DefaultStatsdHost
	if env := os.Getenv(EnvStatsd); env != "" {
		addr = env
	}

	s, err := g2s.Dial("udp", addr)
	if err != nil {
		log.Fatal(err)
		os.Exit(-1)
	}
	_statter = s
}

// client protocol handle proxy
func proxyUserRequest(sess *Session, p []byte) []byte {
	start := time.Now()
	defer utils.PrintPanicStack()

	//解密
	if sess.Flag&SessEncrypt != 0 {
		sess.Decoder.XORKeyStream(p, p)
	}

	//封装为reader
	reader := packet.Reader(p)

	// 读客户端数据包序列号(1,2,3...)
	// 客户端发送的数据包必须包含一个自增的序号，必须严格递增
	// 加密后，可避免重放攻击-REPLAY-ATTACK
	seq_id, err := reader.ReadU32()
	if err != nil {
		log.Error("read client timestamp failed:", err)
		sess.Flag |= SessKickOut
		return nil
	}

	// 数据包序列号验证
	if seq_id != sess.PacketCount {
		log.Errorf("illegal packet sequence id:%v should be:%v size:%v", seq_id, sess.PacketCount, len(p)-6)
		sess.Flag |= SessKickOut
		return nil
	}

	// 读协议号
	b, err := reader.ReadS16()
	if err != nil {
		log.Error("read protocol number failed.")
		sess.Flag |= SessKickOut
		return nil
	}

	log.Debugf("msg seq_id: %v msgid: %v ", seq_id, b)

	// 根据协议号断做服务划分
	// 协议号的划分采用分割协议区间, 用户可以自定义多个区间，用于转发到不同的后端服务
	var ret []byte
	if b > MaxProtoNum {
		if err := forward(sess, p[4:]); err != nil {
			log.Errorf("service id:%v execute failed, error:%v", b, err)
			sess.Flag |= SessKickOut
			return nil
		}
	} else {
		if h := handler.Handlers[b]; h != nil {
			ret = h(sess, reader)
		} else {
			log.Errorf("service id:%v not bind", b)
			sess.Flag |= SessKickOut
			return nil
		}
	}

	// 监控协议处理时间
	// 监控数值会发送到statsd,格式为:
	// API.XXX_REQ = 10ms
	elasped := time.Now().Sub(start)
	if b != 0 { // 排除心跳包日志
		log.Debug("[REQ]", handler.RCode[b], " elasped:", elasped)
		_statter.Timing(1.0, fmt.Sprintf("%v%v", StatsdPrefix, handler.RCode[b]), elasped)
	}

	return ret
}
<|endoftext|>


<commit_before>package api

import (
	"golang.org/x/sys/windows"

	"io/ioutil"
	"os"
	"testing"
)

func TestGetNamedSecurityInfo(t *testing.T) {
	f, err := ioutil.TempFile(os.TempDir(), "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())
	var (
		ownerSid *windows.SID
		secDesc  windows.Handle
	)
	if err = GetNamedSecurityInfo(
		f.Name(),
		SE_FILE_OBJECT,
		OWNER_SECURITY_INFORMATION,
		&ownerSid,
		nil,
		nil,
		nil,
		&secDesc,
	); err != nil {
		t.Fatal(err)
	}
	defer windows.LocalFree(secDesc)
	token, err := windows.OpenCurrentProcessToken()
	if err != nil {
		t.Fatal(err)
	}
	defer token.Close()
	u, err := token.GetTokenUser()
	if err != nil {
		t.Fatal(err)
	}
	if !windows.EqualSid(ownerSid, u.User.Sid) {
		t.Log(ownerSid)
		t.Log(ownerSid.String())
		t.Log(u.User.Sid)
		t.Log(u.User.Sid.String())
		t.Fatal("SID of file does not match SID of current process")
	}
}

func TestSetNamedSecurityInfo(t *testing.T) {
	f, err := ioutil.TempFile(os.TempDir(), "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())
	token, err := windows.OpenCurrentProcessToken()
	if err != nil {
		t.Fatal(err)
	}
	defer token.Close()
	u, err := token.GetTokenUser()
	if err != nil {
		t.Fatal(err)
	}
	if err = SetNamedSecurityInfo(
		f.Name(),
		SE_FILE_OBJECT,
		OWNER_SECURITY_INFORMATION,
		u.User.Sid,
		nil,
		0,
		0,
	); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>Fixed secinfo tests.<commit_after>package api

import (
	"golang.org/x/sys/windows"

	"io/ioutil"
	"os"
	"testing"
)

func TestGetNamedSecurityInfo(t *testing.T) {
	f, err := ioutil.TempFile(os.TempDir(), "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())
	var (
		secDesc windows.Handle
	)
	if err = GetNamedSecurityInfo(
		f.Name(),
		SE_FILE_OBJECT,
		0,
		nil,
		nil,
		nil,
		nil,
		&secDesc,
	); err != nil {
		t.Fatal(err)
	}
	defer windows.LocalFree(secDesc)
}

func TestSetNamedSecurityInfo(t *testing.T) {
	f, err := ioutil.TempFile(os.TempDir(), "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())
	if err = SetNamedSecurityInfo(
		f.Name(),
		SE_FILE_OBJECT,
		DACL_SECURITY_INFORMATION,
		nil,
		nil,
		0,
		0,
	); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package amass

import (
	"strings"
	"time"

	"github.com/OWASP/Amass/amass/core"
	"github.com/OWASP/Amass/amass/utils"
	"github.com/miekg/dns"
)

// BruteForceService is the Service that handles all brute force name generation
// within the architecture. This is achieved by watching all the NEWSUB events.
type BruteForceService struct {
	core.BaseService

	filter *utils.StringFilter
}

// NewBruteForceService returns he object initialized, but not yet started.
func NewBruteForceService(config *core.Config, bus *core.EventBus) *BruteForceService {
	bfs := &BruteForceService{filter: utils.NewStringFilter()}

	bfs.BaseService = *core.NewBaseService(bfs, "Brute Forcing", config, bus)
	return bfs
}

// OnStart implements the Service interface
func (bfs *BruteForceService) OnStart() error {
	bfs.BaseService.OnStart()

	if bfs.Config().BruteForcing {
		if bfs.Config().Recursive {
			if bfs.Config().MinForRecursive == 0 {
				bfs.Bus().Subscribe(core.NameResolvedTopic, bfs.SendRequest)
			} else {
				bfs.Bus().Subscribe(core.NewSubdomainTopic, bfs.NewSubdomain)
			}
		}
		go bfs.startRootDomains()
	}
	go bfs.processRequests()
	return nil
}

func (bfs *BruteForceService) processRequests() {
	for {
		select {
		case <-bfs.PauseChan():
			<-bfs.ResumeChan()
		case <-bfs.Quit():
			return
		case req := <-bfs.RequestChan():
			if bfs.goodRequest(req) {
				bfs.performBruteForcing(req.Name, req.Domain)
			}
		}
	}
}

func (bfs *BruteForceService) goodRequest(req *core.Request) bool {
	if !bfs.Config().BruteForcing {
		return false
	}

	if !bfs.Config().IsDomainInScope(req.Name) {
		return false
	}

	bfs.SetActive()

	var ok bool
	for _, r := range req.Records {
		t := uint16(r.Type)

		if t == dns.TypeA || t == dns.TypeAAAA {
			ok = true
			break
		}
	}
	return ok
}

func (bfs *BruteForceService) startRootDomains() {
	// Look at each domain provided by the config
	for _, domain := range bfs.Config().Domains() {
		bfs.performBruteForcing(domain, domain)
	}
}

// NewSubdomain is called by the Name Service when proper subdomains are discovered.
func (bfs *BruteForceService) NewSubdomain(req *core.Request, times int) {
	if times == bfs.Config().MinForRecursive {
		bfs.SendRequest(req)
	}
}

func (bfs *BruteForceService) performBruteForcing(subdomain, root string) {
	if bfs.filter.Duplicate(subdomain) {
		return
	}

	bfs.SetActive()
	t := time.NewTicker(time.Second)
	defer t.Stop()
	for _, word := range bfs.Config().Wordlist {
		select {
		case <-t.C:
			bfs.SetActive()
		case <-bfs.Quit():
			return
		default:
			bfs.Bus().Publish(core.NewNameTopic, &core.Request{
				Name:   strings.ToLower(word + "." + subdomain),
				Domain: root,
				Tag:    core.BRUTE,
				Source: bfs.String(),
			})
		}
	}
}
<commit_msg>small fix for MinForRecursive<commit_after>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package amass

import (
	"strings"
	"time"

	"github.com/OWASP/Amass/amass/core"
	"github.com/OWASP/Amass/amass/utils"
	"github.com/miekg/dns"
)

// BruteForceService is the Service that handles all brute force name generation
// within the architecture. This is achieved by watching all the NEWSUB events.
type BruteForceService struct {
	core.BaseService

	filter *utils.StringFilter
}

// NewBruteForceService returns he object initialized, but not yet started.
func NewBruteForceService(config *core.Config, bus *core.EventBus) *BruteForceService {
	bfs := &BruteForceService{filter: utils.NewStringFilter()}

	bfs.BaseService = *core.NewBaseService(bfs, "Brute Forcing", config, bus)
	return bfs
}

// OnStart implements the Service interface
func (bfs *BruteForceService) OnStart() error {
	bfs.BaseService.OnStart()

	if bfs.Config().BruteForcing {
		if bfs.Config().Recursive {
			if bfs.Config().MinForRecursive == 0 {
				bfs.Bus().Subscribe(core.NameResolvedTopic, bfs.SendRequest)
			} else {
				bfs.Bus().Subscribe(core.NewSubdomainTopic, bfs.NewSubdomain)
			}
		}
		go bfs.startRootDomains()
	}
	go bfs.processRequests()
	return nil
}

func (bfs *BruteForceService) processRequests() {
	for {
		select {
		case <-bfs.PauseChan():
			<-bfs.ResumeChan()
		case <-bfs.Quit():
			return
		case req := <-bfs.RequestChan():
			if bfs.goodRequest(req) {
				bfs.performBruteForcing(req.Name, req.Domain)
			}
		}
	}
}

func (bfs *BruteForceService) goodRequest(req *core.Request) bool {
	if !bfs.Config().BruteForcing {
		return false
	}

	if !bfs.Config().IsDomainInScope(req.Name) {
		return false
	}

	bfs.SetActive()

	var ok bool
	for _, r := range req.Records {
		t := uint16(r.Type)

		if t == dns.TypeA || t == dns.TypeAAAA {
			ok = true
			break
		}
	}
	return ok
}

func (bfs *BruteForceService) startRootDomains() {
	// Look at each domain provided by the config
	for _, domain := range bfs.Config().Domains() {
		bfs.performBruteForcing(domain, domain)
	}
}

// NewSubdomain is called by the Name Service when proper subdomains are discovered.
func (bfs *BruteForceService) NewSubdomain(req *core.Request, times int) {
	if times >= bfs.Config().MinForRecursive {
		bfs.SendRequest(req)
	}
}

func (bfs *BruteForceService) performBruteForcing(subdomain, root string) {
	if bfs.filter.Duplicate(subdomain) {
		return
	}

	bfs.SetActive()
	t := time.NewTicker(time.Second)
	defer t.Stop()
	for _, word := range bfs.Config().Wordlist {
		select {
		case <-t.C:
			bfs.SetActive()
		case <-bfs.Quit():
			return
		default:
			bfs.Bus().Publish(core.NewNameTopic, &core.Request{
				Name:   strings.ToLower(word + "." + subdomain),
				Domain: root,
				Tag:    core.BRUTE,
				Source: bfs.String(),
			})
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package trace

import "sync/atomic"

// The process global tracer could have process-wide resource
// tags applied directly, or we can have a SetGlobal tracer to
// install a default tracer w/ resources.
var global atomic.Value

var _ Tracer = noopTracer{}

// GlobalTracer return tracer registered with global registry.
// If no tracer is registered then an instance of noop Tracer is returned.
func GlobalTracer() Tracer {
	if t := global.Load(); t != nil {
		return t.(Tracer)
	}
	return noopTracer{}
}

// SetGlobalTracer sets provided tracer as a global tracer.
func SetGlobalTracer(t Tracer) {
	global.Store(t)
}
<commit_msg>remove duplicate interface check (#87)<commit_after>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package trace

import "sync/atomic"

// The process global tracer could have process-wide resource
// tags applied directly, or we can have a SetGlobal tracer to
// install a default tracer w/ resources.
var global atomic.Value

// GlobalTracer return tracer registered with global registry.
// If no tracer is registered then an instance of noop Tracer is returned.
func GlobalTracer() Tracer {
	if t := global.Load(); t != nil {
		return t.(Tracer)
	}
	return noopTracer{}
}

// SetGlobalTracer sets provided tracer as a global tracer.
func SetGlobalTracer(t Tracer) {
	global.Store(t)
}
<|endoftext|>


<commit_before>package seeder

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"testing"
	"time"

	"github.com/dev-cloverlab/carpenter/builder"
	"github.com/dev-cloverlab/carpenter/dialect/mysql"
	_ "github.com/go-sql-driver/mysql"
)

var (
	db     *sql.DB
	schema = "test"
)

func init() {
	var err error
	db, err = sql.Open("mysql", fmt.Sprintf("root@/%s", schema))
	if err != nil {
		panic(err)
	}
	new, err := getTables("./_test/table1.json")
	if err != nil {
		panic(err)
	}
	createSQL, err := builder.Build(db, nil, new[0], true)
	if err != nil {
		panic(err)
	}
	for _, sql := range createSQL {
		if _, err := db.Exec(sql); err != nil {
			panic(err)
		}
	}
}

func TestMain(m *testing.M) {
	code := m.Run()
	db.Exec("drop table if exists `seed_test`")
	os.Exit(code)
}

func TestInsert(t *testing.T) {
	colName := "int"
	oldChunk, err := mysql.GetChunk(db, "seed_test", &colName)
	if err != nil {
		t.Fatal(err)
	}
	now := time.Now().Format(mysql.TimeFmt)
	newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{
		makeSeed([]interface{}{float64(10), "stringA", now, nil}),
		makeSeed([]interface{}{float64(20), "stringB", now, nil}),
	})

	expected := []string{
		"insert into `seed_test`(`int`,`string`,`time`,`null`)\n" +
			"values\n" +
			fmt.Sprintf("(10,\"stringA\",\"%v\",null),\n", now) +
			fmt.Sprintf("(20,\"stringB\",\"%v\",null)", now),
	}
	compString := "int"
	actual, err := Seed(db, oldChunk, newChunk, &compString)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected)
	}
	for _, sql := range actual {
		if _, err := db.Exec(sql); err != nil {
			t.Fatal(err)
		}
	}
}

func TestReplace(t *testing.T) {
	colName := "int"
	oldChunk, err := mysql.GetChunk(db, "seed_test", &colName)
	if err != nil {
		t.Fatal(err)
	}
	now := time.Now().Format(mysql.TimeFmt)
	newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{
		makeSeed([]interface{}{float64(10), "stringC", now, nil}),
		makeSeed([]interface{}{float64(20), "stringB", now, nil}),
	})

	expected := []string{
		"replace into `seed_test`(`int`,`string`,`time`,`null`)\n" +
			"values\n" +
			fmt.Sprintf("(10,\"stringC\",\"%v\",null)", now),
	}
	compString := "int"
	actual, err := Seed(db, oldChunk, newChunk, &compString)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected)
	}
	for _, sql := range actual {
		if _, err := db.Exec(sql); err != nil {
			t.Fatal(err)
		}
	}
}

func TestDelete(t *testing.T) {
	colName := "int"
	oldChunk, err := mysql.GetChunk(db, "seed_test", &colName)
	if err != nil {
		t.Fatal(err)
	}
	now := time.Now().Format(mysql.TimeFmt)
	newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{
		makeSeed([]interface{}{float64(20), "stringB", now, nil}),
	})

	expected := []string{
		"delete from `seed_test` where `int` in (\n" +
			"10\n" +
			")",
	}
	compString := "int"
	actual, err := Seed(db, oldChunk, newChunk, &compString)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected)
	}
	for _, sql := range actual {
		if _, err := db.Exec(sql); err != nil {
			t.Fatal(err)
		}
	}
}

func TestTruncate(t *testing.T) {
	colName := "int"
	oldChunk, err := mysql.GetChunk(db, "seed_test", &colName)
	if err != nil {
		t.Fatal(err)
	}
	newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{})

	expected := []string{
		"truncate table `seed_test`",
	}
	compString := "int"
	actual, err := Seed(db, oldChunk, newChunk, &compString)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected)
	}
	for _, sql := range actual {
		if _, err := db.Exec(sql); err != nil {
			t.Fatal(err)
		}
	}
}

func makeSeed(columnData []interface{}) mysql.Seed {
	return mysql.Seed{
		ColumnData: columnData,
	}
}

func makeChunk(tableName string, columnNames []string, seeds mysql.Seeds) *mysql.Chunk {
	return &mysql.Chunk{
		TableName:   tableName,
		ColumnNames: columnNames,
		Seeds:       seeds,
	}
}

func getTables(filename string) (mysql.Tables, error) {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	tables := mysql.Tables{}
	if err := json.Unmarshal(buf, &tables); err != nil {
		return nil, err
	}
	return tables, nil
}
<commit_msg>Fix the seeder test<commit_after>package seeder

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"testing"
	"time"

	"github.com/dev-cloverlab/carpenter/builder"
	"github.com/dev-cloverlab/carpenter/dialect/mysql"
	_ "github.com/go-sql-driver/mysql"
)

var (
	db     *sql.DB
	schema = "carpenter_test"
)

func init() {
	var err error
	db, err = sql.Open("mysql", "root@/")
	if err != nil {
		panic(err)
	}
}

func TestMain(m *testing.M) {
	_, err := db.Exec("CREATE DATABASE IF NOT EXISTS `" + schema + "`")
	if err != nil {
		panic(err)
	}
	_, err = db.Exec("USE `" + schema + "`")
	if err != nil {
		panic(err)
	}
	new, err := getTables("./_test/table1.json")
	if err != nil {
		panic(err)
	}
	createSQL, err := builder.Build(db, nil, new[0], true)
	if err != nil {
		panic(err)
	}
	for _, sql := range createSQL {
		if _, err := db.Exec(sql); err != nil {
			panic(err)
		}
	}
	code := m.Run()
	_, err = db.Exec("drop table if exists `seed_test`")
	if err != nil {
		panic(err)
	}
	_, err = db.Exec("DROP DATABASE `" + schema + "`")
	if err != nil {
		panic(err)
	}
	os.Exit(code)
}

func TestInsert(t *testing.T) {
	colName := "int"
	oldChunk, err := mysql.GetChunk(db, "seed_test", &colName)
	if err != nil {
		t.Fatal(err)
	}
	now := time.Now().Format(mysql.TimeFmt)
	newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{
		makeSeed([]interface{}{float64(10), "stringA", now, nil}),
		makeSeed([]interface{}{float64(20), "stringB", now, nil}),
	})

	expected := []string{
		"insert into `seed_test`(`int`,`string`,`time`,`null`)\n" +
			"values\n" +
			fmt.Sprintf("(10,\"stringA\",\"%v\",null),\n", now) +
			fmt.Sprintf("(20,\"stringB\",\"%v\",null)", now),
	}
	compString := "int"
	actual, err := Seed(db, oldChunk, newChunk, &compString)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected)
	}
	for _, sql := range actual {
		if _, err := db.Exec(sql); err != nil {
			t.Fatal(err)
		}
	}
}

func TestReplace(t *testing.T) {
	colName := "int"
	oldChunk, err := mysql.GetChunk(db, "seed_test", &colName)
	if err != nil {
		t.Fatal(err)
	}
	now := time.Now().Format(mysql.TimeFmt)
	newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{
		makeSeed([]interface{}{float64(10), "stringC", now, nil}),
		makeSeed([]interface{}{float64(20), "stringB", now, nil}),
	})

	expected := []string{
		"replace into `seed_test`(`int`,`string`,`time`,`null`)\n" +
			"values\n" +
			fmt.Sprintf("(10,\"stringC\",\"%v\",null)", now),
	}
	compString := "int"
	actual, err := Seed(db, oldChunk, newChunk, &compString)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected)
	}
	for _, sql := range actual {
		if _, err := db.Exec(sql); err != nil {
			t.Fatal(err)
		}
	}
}

func TestDelete(t *testing.T) {
	colName := "int"
	oldChunk, err := mysql.GetChunk(db, "seed_test", &colName)
	if err != nil {
		t.Fatal(err)
	}
	now := time.Now().Format(mysql.TimeFmt)
	newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{
		makeSeed([]interface{}{float64(20), "stringB", now, nil}),
	})

	expected := []string{
		"delete from `seed_test` where `int` in (\n" +
			"10\n" +
			")",
	}
	compString := "int"
	actual, err := Seed(db, oldChunk, newChunk, &compString)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected)
	}
	for _, sql := range actual {
		if _, err := db.Exec(sql); err != nil {
			t.Fatal(err)
		}
	}
}

func TestTruncate(t *testing.T) {
	colName := "int"
	oldChunk, err := mysql.GetChunk(db, "seed_test", &colName)
	if err != nil {
		t.Fatal(err)
	}
	newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{})

	expected := []string{
		"truncate table `seed_test`",
	}
	compString := "int"
	actual, err := Seed(db, oldChunk, newChunk, &compString)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected)
	}
	for _, sql := range actual {
		if _, err := db.Exec(sql); err != nil {
			t.Fatal(err)
		}
	}
}

func makeSeed(columnData []interface{}) mysql.Seed {
	return mysql.Seed{
		ColumnData: columnData,
	}
}

func makeChunk(tableName string, columnNames []string, seeds mysql.Seeds) *mysql.Chunk {
	return &mysql.Chunk{
		TableName:   tableName,
		ColumnNames: columnNames,
		Seeds:       seeds,
	}
}

func getTables(filename string) (mysql.Tables, error) {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	tables := mysql.Tables{}
	if err := json.Unmarshal(buf, &tables); err != nil {
		return nil, err
	}
	return tables, nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package segment

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"reflect"
	"strings"
	"testing"
)

func TestAdhocSegmentsWithType(t *testing.T) {

	tests := []struct {
		input         []byte
		output        [][]byte
		outputStrings []string
		outputTypes   []int
	}{
		{
			input: []byte("Now is the.\n End."),
			output: [][]byte{
				[]byte("Now"),
				[]byte(" "),
				[]byte("is"),
				[]byte(" "),
				[]byte("the"),
				[]byte("."),
				[]byte("\n"),
				[]byte(" "),
				[]byte("End"),
				[]byte("."),
			},
			outputStrings: []string{
				"Now",
				" ",
				"is",
				" ",
				"the",
				".",
				"\n",
				" ",
				"End",
				".",
			},
			outputTypes: []int{
				Letter,
				None,
				Letter,
				None,
				Letter,
				None,
				None,
				None,
				Letter,
				None,
			},
		},
		{
			input: []byte("3.5"),
			output: [][]byte{
				[]byte("3.5"),
			},
			outputStrings: []string{
				"3.5",
			},
			outputTypes: []int{
				Number,
			},
		},
		{
			input: []byte("cat3.5"),
			output: [][]byte{
				[]byte("cat3.5"),
			},
			outputStrings: []string{
				"cat3.5",
			},
			outputTypes: []int{
				Letter,
			},
		},
		{
			input: []byte("c"),
			output: [][]byte{
				[]byte("c"),
			},
			outputStrings: []string{
				"c",
			},
			outputTypes: []int{
				Letter,
			},
		},
		{
			input: []byte("こんにちは世界"),
			output: [][]byte{
				[]byte("こ"),
				[]byte("ん"),
				[]byte("に"),
				[]byte("ち"),
				[]byte("は"),
				[]byte("世"),
				[]byte("界"),
			},
			outputStrings: []string{
				"こ",
				"ん",
				"に",
				"ち",
				"は",
				"世",
				"界",
			},
			outputTypes: []int{
				Ideo,
				Ideo,
				Ideo,
				Ideo,
				Ideo,
				Ideo,
				Ideo,
			},
		},
		{
			input: []byte("你好世界"),
			output: [][]byte{
				[]byte("你"),
				[]byte("好"),
				[]byte("世"),
				[]byte("界"),
			},
			outputStrings: []string{
				"你",
				"好",
				"世",
				"界",
			},
			outputTypes: []int{
				Ideo,
				Ideo,
				Ideo,
				Ideo,
			},
		},
		{
			input: []byte("サッカ"),
			output: [][]byte{
				[]byte("サッカ"),
			},
			outputStrings: []string{
				"サッカ",
			},
			outputTypes: []int{
				Ideo,
			},
		},
	}

	for _, test := range tests {
		rv := make([][]byte, 0)
		rvstrings := make([]string, 0)
		rvtypes := make([]int, 0)
		segmenter := NewWordSegmenter(bytes.NewReader(test.input))
		// Set the split function for the scanning operation.
		for segmenter.Segment() {
			rv = append(rv, segmenter.Bytes())
			rvstrings = append(rvstrings, segmenter.Text())
			rvtypes = append(rvtypes, segmenter.Type())
		}
		if err := segmenter.Err(); err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(rv, test.output) {
			t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s'", test.output, rv, test.input)
		}
		if !reflect.DeepEqual(rvstrings, test.outputStrings) {
			t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s'", test.outputStrings, rvstrings, test.input)
		}
		if !reflect.DeepEqual(rvtypes, test.outputTypes) {
			t.Fatalf("expeced:\n%#v\ngot:\n%#v\nfor: '%s'", test.outputTypes, rvtypes, test.input)
		}
	}

}

func TestUnicodeSegments(t *testing.T) {

	for _, test := range unicodeWordTests {
		rv := make([][]byte, 0)
		scanner := bufio.NewScanner(bytes.NewReader(test.input))
		// Set the split function for the scanning operation.
		scanner.Split(SplitWords)
		for scanner.Scan() {
			rv = append(rv, scanner.Bytes())
		}
		if err := scanner.Err(); err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(rv, test.output) {
			t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s'", test.output, rv, test.input)
		}
	}
}

// Tests borrowed from Scanner to test Segmenter

// slowReader is a reader that returns only a few bytes at a time, to test the incremental
// reads in Scanner.Scan.
type slowReader struct {
	max int
	buf io.Reader
}

func (sr *slowReader) Read(p []byte) (n int, err error) {
	if len(p) > sr.max {
		p = p[0:sr.max]
	}
	return sr.buf.Read(p)
}

// genLine writes to buf a predictable but non-trivial line of text of length
// n, including the terminal newline and an occasional carriage return.
// If addNewline is false, the \r and \n are not emitted.
func genLine(buf *bytes.Buffer, lineNum, n int, addNewline bool) {
	buf.Reset()
	doCR := lineNum%5 == 0
	if doCR {
		n--
	}
	for i := 0; i < n-1; i++ { // Stop early for \n.
		c := 'a' + byte(lineNum+i)
		if c == '\n' || c == '\r' { // Don't confuse us.
			c = 'N'
		}
		buf.WriteByte(c)
	}
	if addNewline {
		if doCR {
			buf.WriteByte('\r')
		}
		buf.WriteByte('\n')
	}
	return
}

func wrapSplitFuncAsSegmentFuncForTesting(splitFunc bufio.SplitFunc) SegmentFunc {
	return func(data []byte, atEOF bool) (advance int, token []byte, typ int, err error) {
		typ = 0
		advance, token, err = splitFunc(data, atEOF)
		return
	}
}

// Test that the line segmenter errors out on a long line.
func TestSegmentTooLong(t *testing.T) {
	const smallMaxTokenSize = 256 // Much smaller for more efficient testing.
	// Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize.
	tmp := new(bytes.Buffer)
	buf := new(bytes.Buffer)
	lineNum := 0
	j := 0
	for i := 0; i < 2*smallMaxTokenSize; i++ {
		genLine(tmp, lineNum, j, true)
		j++
		buf.Write(tmp.Bytes())
		lineNum++
	}
	s := NewSegmenter(&slowReader{3, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(bufio.ScanLines))
	s.MaxTokenSize(smallMaxTokenSize)
	j = 0
	for lineNum := 0; s.Segment(); lineNum++ {
		genLine(tmp, lineNum, j, false)
		if j < smallMaxTokenSize {
			j++
		} else {
			j--
		}
		line := tmp.Bytes()
		if !bytes.Equal(s.Bytes(), line) {
			t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Bytes(), line)
		}
	}
	err := s.Err()
	if err != ErrTooLong {
		t.Fatalf("expected ErrTooLong; got %s", err)
	}
}

var testError = errors.New("testError")

// Test the correct error is returned when the split function errors out.
func TestSegmentError(t *testing.T) {
	// Create a split function that delivers a little data, then a predictable error.
	numSplits := 0
	const okCount = 7
	errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF {
			panic("didn't get enough data")
		}
		if numSplits >= okCount {
			return 0, nil, testError
		}
		numSplits++
		return 1, data[0:1], nil
	}
	// Read the data.
	const text = "abcdefghijklmnopqrstuvwxyz"
	buf := strings.NewReader(text)
	s := NewSegmenter(&slowReader{1, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(errorSplit))
	var i int
	for i = 0; s.Segment(); i++ {
		if len(s.Bytes()) != 1 || text[i] != s.Bytes()[0] {
			t.Errorf("#%d: expected %q got %q", i, text[i], s.Bytes()[0])
		}
	}
	// Check correct termination location and error.
	if i != okCount {
		t.Errorf("unexpected termination; expected %d tokens got %d", okCount, i)
	}
	err := s.Err()
	if err != testError {
		t.Fatalf("expected %q got %v", testError, err)
	}
}

// Test that Scan finishes if we have endless empty reads.
type endlessZeros struct{}

func (endlessZeros) Read(p []byte) (int, error) {
	return 0, nil
}

func TestBadReader(t *testing.T) {
	scanner := NewSegmenter(endlessZeros{})
	for scanner.Segment() {
		t.Fatal("read should fail")
	}
	err := scanner.Err()
	if err != io.ErrNoProgress {
		t.Errorf("unexpected error: %v", err)
	}
}

func TestSegmentAdvanceNegativeError(t *testing.T) {
	errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF {
			panic("didn't get enough data")
		}
		return -1, data[0:1], nil
	}
	// Read the data.
	const text = "abcdefghijklmnopqrstuvwxyz"
	buf := strings.NewReader(text)
	s := NewSegmenter(&slowReader{1, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(errorSplit))
	s.Segment()
	err := s.Err()
	if err != ErrNegativeAdvance {
		t.Fatalf("expected %q got %v", testError, err)
	}
}

func TestSegmentAdvanceTooFarError(t *testing.T) {
	errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF {
			panic("didn't get enough data")
		}
		return len(data) + 10, data[0:1], nil
	}
	// Read the data.
	const text = "abcdefghijklmnopqrstuvwxyz"
	buf := strings.NewReader(text)
	s := NewSegmenter(&slowReader{1, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(errorSplit))
	s.Segment()
	err := s.Err()
	if err != ErrAdvanceTooFar {
		t.Fatalf("expected %q got %v", testError, err)
	}
}

func TestSegmentLongTokens(t *testing.T) {
	// Read the data.
	text := bytes.Repeat([]byte("abcdefghijklmnop"), 257)
	buf := strings.NewReader(string(text))
	s := NewSegmenter(&slowReader{1, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(bufio.ScanLines))
	for s.Segment() {
		line := s.Bytes()
		if !bytes.Equal(text, line) {
			t.Errorf("expected %s, got %s", text, line)
		}
	}
	err := s.Err()
	if err != nil {
		t.Fatalf("unexpected error; got %s", err)
	}
}
<commit_msg>improving test coverage<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package segment

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"reflect"
	"strings"
	"testing"
)

func TestAdhocSegmentsWithType(t *testing.T) {

	tests := []struct {
		input         []byte
		output        [][]byte
		outputStrings []string
		outputTypes   []int
	}{
		{
			input: []byte("Now is the.\n End."),
			output: [][]byte{
				[]byte("Now"),
				[]byte(" "),
				[]byte("is"),
				[]byte(" "),
				[]byte("the"),
				[]byte("."),
				[]byte("\n"),
				[]byte(" "),
				[]byte("End"),
				[]byte("."),
			},
			outputStrings: []string{
				"Now",
				" ",
				"is",
				" ",
				"the",
				".",
				"\n",
				" ",
				"End",
				".",
			},
			outputTypes: []int{
				Letter,
				None,
				Letter,
				None,
				Letter,
				None,
				None,
				None,
				Letter,
				None,
			},
		},
		{
			input: []byte("3.5"),
			output: [][]byte{
				[]byte("3.5"),
			},
			outputStrings: []string{
				"3.5",
			},
			outputTypes: []int{
				Number,
			},
		},
		{
			input: []byte("cat3.5"),
			output: [][]byte{
				[]byte("cat3.5"),
			},
			outputStrings: []string{
				"cat3.5",
			},
			outputTypes: []int{
				Letter,
			},
		},
		{
			input: []byte("c"),
			output: [][]byte{
				[]byte("c"),
			},
			outputStrings: []string{
				"c",
			},
			outputTypes: []int{
				Letter,
			},
		},
		{
			input: []byte("こんにちは世界"),
			output: [][]byte{
				[]byte("こ"),
				[]byte("ん"),
				[]byte("に"),
				[]byte("ち"),
				[]byte("は"),
				[]byte("世"),
				[]byte("界"),
			},
			outputStrings: []string{
				"こ",
				"ん",
				"に",
				"ち",
				"は",
				"世",
				"界",
			},
			outputTypes: []int{
				Ideo,
				Ideo,
				Ideo,
				Ideo,
				Ideo,
				Ideo,
				Ideo,
			},
		},
		{
			input: []byte("你好世界"),
			output: [][]byte{
				[]byte("你"),
				[]byte("好"),
				[]byte("世"),
				[]byte("界"),
			},
			outputStrings: []string{
				"你",
				"好",
				"世",
				"界",
			},
			outputTypes: []int{
				Ideo,
				Ideo,
				Ideo,
				Ideo,
			},
		},
		{
			input: []byte("サッカ"),
			output: [][]byte{
				[]byte("サッカ"),
			},
			outputStrings: []string{
				"サッカ",
			},
			outputTypes: []int{
				Ideo,
			},
		},
	}

	for _, test := range tests {
		rv := make([][]byte, 0)
		rvstrings := make([]string, 0)
		rvtypes := make([]int, 0)
		segmenter := NewWordSegmenter(bytes.NewReader(test.input))
		// Set the split function for the scanning operation.
		for segmenter.Segment() {
			rv = append(rv, segmenter.Bytes())
			rvstrings = append(rvstrings, segmenter.Text())
			rvtypes = append(rvtypes, segmenter.Type())
		}
		if err := segmenter.Err(); err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(rv, test.output) {
			t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s'", test.output, rv, test.input)
		}
		if !reflect.DeepEqual(rvstrings, test.outputStrings) {
			t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s'", test.outputStrings, rvstrings, test.input)
		}
		if !reflect.DeepEqual(rvtypes, test.outputTypes) {
			t.Fatalf("expeced:\n%#v\ngot:\n%#v\nfor: '%s'", test.outputTypes, rvtypes, test.input)
		}
	}

}

func TestUnicodeSegments(t *testing.T) {

	for _, test := range unicodeWordTests {
		rv := make([][]byte, 0)
		scanner := bufio.NewScanner(bytes.NewReader(test.input))
		// Set the split function for the scanning operation.
		scanner.Split(SplitWords)
		for scanner.Scan() {
			rv = append(rv, scanner.Bytes())
		}
		if err := scanner.Err(); err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(rv, test.output) {
			t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s'", test.output, rv, test.input)
		}
	}
}

// Tests borrowed from Scanner to test Segmenter

// slowReader is a reader that returns only a few bytes at a time, to test the incremental
// reads in Scanner.Scan.
type slowReader struct {
	max int
	buf io.Reader
}

func (sr *slowReader) Read(p []byte) (n int, err error) {
	if len(p) > sr.max {
		p = p[0:sr.max]
	}
	return sr.buf.Read(p)
}

// genLine writes to buf a predictable but non-trivial line of text of length
// n, including the terminal newline and an occasional carriage return.
// If addNewline is false, the \r and \n are not emitted.
func genLine(buf *bytes.Buffer, lineNum, n int, addNewline bool) {
	buf.Reset()
	doCR := lineNum%5 == 0
	if doCR {
		n--
	}
	for i := 0; i < n-1; i++ { // Stop early for \n.
		c := 'a' + byte(lineNum+i)
		if c == '\n' || c == '\r' { // Don't confuse us.
			c = 'N'
		}
		buf.WriteByte(c)
	}
	if addNewline {
		if doCR {
			buf.WriteByte('\r')
		}
		buf.WriteByte('\n')
	}
	return
}

func wrapSplitFuncAsSegmentFuncForTesting(splitFunc bufio.SplitFunc) SegmentFunc {
	return func(data []byte, atEOF bool) (advance int, token []byte, typ int, err error) {
		typ = 0
		advance, token, err = splitFunc(data, atEOF)
		return
	}
}

// Test that the line segmenter errors out on a long line.
func TestSegmentTooLong(t *testing.T) {
	const smallMaxTokenSize = 256 // Much smaller for more efficient testing.
	// Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize.
	tmp := new(bytes.Buffer)
	buf := new(bytes.Buffer)
	lineNum := 0
	j := 0
	for i := 0; i < 2*smallMaxTokenSize; i++ {
		genLine(tmp, lineNum, j, true)
		j++
		buf.Write(tmp.Bytes())
		lineNum++
	}
	s := NewSegmenter(&slowReader{3, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(bufio.ScanLines))
	s.MaxTokenSize(smallMaxTokenSize)
	j = 0
	for lineNum := 0; s.Segment(); lineNum++ {
		genLine(tmp, lineNum, j, false)
		if j < smallMaxTokenSize {
			j++
		} else {
			j--
		}
		line := tmp.Bytes()
		if !bytes.Equal(s.Bytes(), line) {
			t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Bytes(), line)
		}
	}
	err := s.Err()
	if err != ErrTooLong {
		t.Fatalf("expected ErrTooLong; got %s", err)
	}
}

var testError = errors.New("testError")

// Test the correct error is returned when the split function errors out.
func TestSegmentError(t *testing.T) {
	// Create a split function that delivers a little data, then a predictable error.
	numSplits := 0
	const okCount = 7
	errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF {
			panic("didn't get enough data")
		}
		if numSplits >= okCount {
			return 0, nil, testError
		}
		numSplits++
		return 1, data[0:1], nil
	}
	// Read the data.
	const text = "abcdefghijklmnopqrstuvwxyz"
	buf := strings.NewReader(text)
	s := NewSegmenter(&slowReader{1, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(errorSplit))
	var i int
	for i = 0; s.Segment(); i++ {
		if len(s.Bytes()) != 1 || text[i] != s.Bytes()[0] {
			t.Errorf("#%d: expected %q got %q", i, text[i], s.Bytes()[0])
		}
	}
	// Check correct termination location and error.
	if i != okCount {
		t.Errorf("unexpected termination; expected %d tokens got %d", okCount, i)
	}
	err := s.Err()
	if err != testError {
		t.Fatalf("expected %q got %v", testError, err)
	}
}

// Test that Scan finishes if we have endless empty reads.
type endlessZeros struct{}

func (endlessZeros) Read(p []byte) (int, error) {
	return 0, nil
}

func TestBadReader(t *testing.T) {
	scanner := NewSegmenter(endlessZeros{})
	for scanner.Segment() {
		t.Fatal("read should fail")
	}
	err := scanner.Err()
	if err != io.ErrNoProgress {
		t.Errorf("unexpected error: %v", err)
	}
}

func TestSegmentAdvanceNegativeError(t *testing.T) {
	errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF {
			panic("didn't get enough data")
		}
		return -1, data[0:1], nil
	}
	// Read the data.
	const text = "abcdefghijklmnopqrstuvwxyz"
	buf := strings.NewReader(text)
	s := NewSegmenter(&slowReader{1, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(errorSplit))
	s.Segment()
	err := s.Err()
	if err != ErrNegativeAdvance {
		t.Fatalf("expected %q got %v", testError, err)
	}
}

func TestSegmentAdvanceTooFarError(t *testing.T) {
	errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF {
			panic("didn't get enough data")
		}
		return len(data) + 10, data[0:1], nil
	}
	// Read the data.
	const text = "abcdefghijklmnopqrstuvwxyz"
	buf := strings.NewReader(text)
	s := NewSegmenter(&slowReader{1, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(errorSplit))
	s.Segment()
	err := s.Err()
	if err != ErrAdvanceTooFar {
		t.Fatalf("expected %q got %v", testError, err)
	}
}

func TestSegmentLongTokens(t *testing.T) {
	// Read the data.
	text := bytes.Repeat([]byte("abcdefghijklmnop"), 257)
	buf := strings.NewReader(string(text))
	s := NewSegmenter(&slowReader{1, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(bufio.ScanLines))
	for s.Segment() {
		line := s.Bytes()
		if !bytes.Equal(text, line) {
			t.Errorf("expected %s, got %s", text, line)
		}
	}
	err := s.Err()
	if err != nil {
		t.Fatalf("unexpected error; got %s", err)
	}
}

func TestSegmentLongTokensDontDouble(t *testing.T) {
	// Read the data.
	text := bytes.Repeat([]byte("abcdefghijklmnop"), 257)
	buf := strings.NewReader(string(text))
	s := NewSegmenter(&slowReader{1, buf})
	// change to line segmenter for testing
	s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(bufio.ScanLines))
	s.MaxTokenSize(6144)
	for s.Segment() {
		line := s.Bytes()
		if !bytes.Equal(text, line) {
			t.Errorf("expected %s, got %s", text, line)
		}
	}
	err := s.Err()
	if err != nil {
		t.Fatalf("unexpected error; got %s", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package core

import (
	"crypto/x509"
	"github.com/letsencrypt/boulder/jose"
	"net/http"
)

// A WebFrontEnd object supplies methods that can be hooked into
// the Go http module's server functions, principally http.HandleFunc()
//
// It also provides methods to configure the base for authorization and
// certificate URLs.
//
// It is assumed that the ACME server is laid out as follows:
// * One URL for new-authorization -> NewAuthz
// * One URL for new-certificate -> NewCert
// * One path for authorizations -> Authz
// * One path for certificates -> Cert
type WebFrontEnd interface {
	// Set the base URL for authorizations
	SetAuthzBase(path string)

	// Set the base URL for certificates
	SetCertBase(path string)

	// This method represents the ACME new-registration resource
	NewRegistration(response http.ResponseWriter, request *http.Request)

	// This method represents the ACME new-authorization resource
	NewAuthz(response http.ResponseWriter, request *http.Request)

	// This method represents the ACME new-certificate resource
	NewCert(response http.ResponseWriter, request *http.Request)

	// Provide access to requests for registration resources
	Registration(response http.ResponseWriter, request *http.Request)

	// Provide access to requests for authorization resources
	Authz(response http.ResponseWriter, request *http.Request)

	// Provide access to requests for authorization resources
	Cert(response http.ResponseWriter, request *http.Request)
}

type RegistrationAuthority interface {
	// [WebFrontEnd]
	NewRegistration(Registration, jose.JsonWebKey) (Registration, error)

	// [WebFrontEnd]
	NewAuthorization(Authorization, jose.JsonWebKey) (Authorization, error)

	// [WebFrontEnd]
	NewCertificate(CertificateRequest, jose.JsonWebKey) (Certificate, error)

	// [WebFrontEnd]
	UpdateRegistration(Registration, Registration) (Registration, error)

	// [WebFrontEnd]
	UpdateAuthorization(Authorization, int, Challenge) (Authorization, error)

	// [WebFrontEnd]
	RevokeCertificate(x509.Certificate) error

	// [ValidationAuthority]
	OnValidationUpdate(Authorization)
}

type ValidationAuthority interface {
	// [RegistrationAuthority]
	UpdateValidations(Authorization) error
}

type CertificateAuthority interface {
	// [RegistrationAuthority]
	IssueCertificate(x509.CertificateRequest) (Certificate, error)
}

type StorageGetter interface {
	GetRegistration(string) (Registration, error)
	GetAuthorization(string) (Authorization, error)
	GetCertificate(string) ([]byte, error)
}

type StorageAdder interface {
	NewRegistration() (string, error)
	UpdateRegistration(Registration) error

	NewPendingAuthorization() (string, error)
	UpdatePendingAuthorization(Authorization) error
	FinalizeAuthorization(Authorization) error

	AddCertificate([]byte) (string, error)
}

// The StorageAuthority interface represnts a simple key/value
// store.  It is divided into StorageGetter and StorageUpdater
// interfaces for privilege separation.
type StorageAuthority interface {
	StorageGetter
	StorageAdder
}
<commit_msg>Identifier checking (syntax,blacklist,PSL)<commit_after>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package core

import (
	"crypto/x509"
	"github.com/letsencrypt/boulder/jose"
	"net/http"
)

// A WebFrontEnd object supplies methods that can be hooked into
// the Go http module's server functions, principally http.HandleFunc()
//
// It also provides methods to configure the base for authorization and
// certificate URLs.
//
// It is assumed that the ACME server is laid out as follows:
// * One URL for new-authorization -> NewAuthz
// * One URL for new-certificate -> NewCert
// * One path for authorizations -> Authz
// * One path for certificates -> Cert
type WebFrontEnd interface {
	// Set the base URL for authorizations
	SetAuthzBase(path string)

	// Set the base URL for certificates
	SetCertBase(path string)

	// This method represents the ACME new-registration resource
	NewRegistration(response http.ResponseWriter, request *http.Request)

	// This method represents the ACME new-authorization resource
	NewAuthz(response http.ResponseWriter, request *http.Request)

	// This method represents the ACME new-certificate resource
	NewCert(response http.ResponseWriter, request *http.Request)

	// Provide access to requests for registration resources
	Registration(response http.ResponseWriter, request *http.Request)

	// Provide access to requests for authorization resources
	Authz(response http.ResponseWriter, request *http.Request)

	// Provide access to requests for authorization resources
	Cert(response http.ResponseWriter, request *http.Request)
}

type RegistrationAuthority interface {
	// [WebFrontEnd]
	NewRegistration(Registration, jose.JsonWebKey) (Registration, error)

	// [WebFrontEnd]
	NewAuthorization(Authorization, jose.JsonWebKey) (Authorization, error)

	// [WebFrontEnd]
	NewCertificate(CertificateRequest, jose.JsonWebKey) (Certificate, error)

	// [WebFrontEnd]
	UpdateRegistration(Registration, Registration) (Registration, error)

	// [WebFrontEnd]
	UpdateAuthorization(Authorization, int, Challenge) (Authorization, error)

	// [WebFrontEnd]
	RevokeCertificate(x509.Certificate) error

	// [ValidationAuthority]
	OnValidationUpdate(Authorization)
}

type ValidationAuthority interface {
	// [RegistrationAuthority]
	UpdateValidations(Authorization) error
}

type CertificateAuthority interface {
	// [RegistrationAuthority]
	IssueCertificate(x509.CertificateRequest) (Certificate, error)
}

type PolicyAuthority interface {
	WellFormed(AcmeIdentifier) bool
	WillingToIssue(AcmeIdentifier) bool
	ChallengesFor(AcmeIdentifier) ([]Challenge, [][]int)
}

type StorageGetter interface {
	GetRegistration(string) (Registration, error)
	GetAuthorization(string) (Authorization, error)
	GetCertificate(string) ([]byte, error)
}

type StorageAdder interface {
	NewRegistration() (string, error)
	UpdateRegistration(Registration) error

	NewPendingAuthorization() (string, error)
	UpdatePendingAuthorization(Authorization) error
	FinalizeAuthorization(Authorization) error

	AddCertificate([]byte) (string, error)
}

// The StorageAuthority interface represnts a simple key/value
// store.  It is divided into StorageGetter and StorageUpdater
// interfaces for privilege separation.
type StorageAuthority interface {
	StorageGetter
	StorageAdder
}
<|endoftext|>


<commit_before>package upload

import (
	"bytes"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"testing"

	"code.google.com/p/go-uuid/uuid"

	"github.com/stretchr/testify/assert"
)

func TestUploadMultipart(t *testing.T) {
	assert := assert.New(t)

	var body bytes.Buffer
	mw := multipart.NewWriter(&body)

	if err := writeMPBody("../bin/coquelicot/dummy/32509211_news_bigpic.jpg", mw); err != nil {
		assert.Error(err)
	}
	if err := writeMPBody("../bin/coquelicot/dummy/kino.jpg", mw); err != nil {
		assert.Error(err)
	}

	mw.Close()

	req, _ := http.NewRequest("POST", "/files", &body)
	req.Header.Set("Content-Type", mw.FormDataContentType())
	req.AddCookie(&http.Cookie{Name: "pavo", Value: "abcdef"})

	files, err := Process(req, "../bin/coquelicot/dummy/root_storage")
	assert.Nil(err)
	assert.Equal("kino.jpg", files[1].Filename)
	assert.Equal("image", files[1].BaseMime)

}
func TestUploadBinary(t *testing.T) {
	assert := assert.New(t)

	req, _ := http.NewRequest("POST", "/files", nil)
	req.Header.Set("Content-Type", "application/octet-stream")
	req.Header.Set("X-File", "../bin/coquelicot/dummy/bin-data")
	req.Header.Set("Content-Disposition", `attachment; filename="basta.png"`)
	req.AddCookie(&http.Cookie{Name: "pavo", Value: "abcdef"})

	files, err := Process(req, "../bin/coquelicot/dummy/root_storage")
	assert.Nil(err)
	assert.Equal("basta.png", files[0].Filename)
	assert.Equal("image", files[0].BaseMime)

}

func TestUploadChunked(t *testing.T) {
	assert := assert.New(t)
	storage := "../dummy/root_storage"
	fname := "../dummy/kino.jpg"
	f, _ := os.Open(fname)
	defer f.Close()

	cookie := &http.Cookie{Name: "pavo", Value: uuid.New()}

	req := createChunkRequest(f, 0, 24999)
	req.AddCookie(cookie)
	files, err := Process(req, storage)
	assert.Equal(Incomplete, err)
	assert.Equal(25000, files[0].Size)

	req = createChunkRequest(f, 25000, 49999)
	req.AddCookie(cookie)
	files, err = Process(req, storage)
	assert.Equal(Incomplete, err)
	assert.Equal(50000, files[0].Size)

	req = createChunkRequest(f, 50000, 52096)
	req.AddCookie(cookie)
	files, err = Process(req, storage)
	assert.Nil(err)
	assert.Equal(52097, files[0].Size)
	assert.Equal("kino.jpg", files[0].Filename)
}

func createChunkRequest(f *os.File, start int64, end int64) *http.Request {
	var body bytes.Buffer
	mw := multipart.NewWriter(&body)
	fi, _ := f.Stat()
	fw, _ := mw.CreateFormFile("files[]", fi.Name())

	io.CopyN(fw, f, end-start+1)
	mw.Close()

	req, _ := http.NewRequest("POST", "/files", &body)
	req.Header.Set("Content-Type", mw.FormDataContentType())
	req.Header.Set("Content-Disposition", `attachment; filename="`+fi.Name()+`"`)
	req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, fi.Size()))

	return req
}

func TestTempFileChunks(t *testing.T) {
	assert := assert.New(t)

	file, err := TempFileChunks(0, "../dummy/root_storage", "abcdef", "kino.jpg")
	assert.Nil(err)
	assert.NotNil(file)
}

func writeMPBody(fname string, mw *multipart.Writer) error {
	fw, _ := mw.CreateFormFile("files[]", filepath.Base(fname))
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(fw, f)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Upload UT fixes<commit_after>package upload

import (
	"bytes"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"testing"

	"code.google.com/p/go-uuid/uuid"

	"github.com/stretchr/testify/assert"
)

const dummy = "../bin/coquelicot/dummy"

func TestUploadMultipart(t *testing.T) {
	assert := assert.New(t)

	var body bytes.Buffer
	mw := multipart.NewWriter(&body)

	if err := writeMPBody(dummy+"/32509211_news_bigpic.jpg", mw); err != nil {
		assert.Error(err)
	}
	if err := writeMPBody(dummy+"/kino.jpg", mw); err != nil {
		assert.Error(err)
	}

	mw.Close()

	req, _ := http.NewRequest("POST", "/files", &body)
	req.Header.Set("Content-Type", mw.FormDataContentType())
	req.AddCookie(&http.Cookie{Name: "pavo", Value: "abcdef"})

	files, err := Process(req, dummy+"/root_storage")
	assert.Nil(err)
	assert.Equal("kino.jpg", files[1].Filename)
	assert.Equal("image", files[1].BaseMime)

}
func TestUploadBinary(t *testing.T) {
	assert := assert.New(t)

	req, _ := http.NewRequest("POST", "/files", nil)
	req.Header.Set("Content-Type", "application/octet-stream")
	req.Header.Set("X-File", dummy+"/bin-data")
	req.Header.Set("Content-Disposition", `attachment; filename="basta.png"`)
	req.AddCookie(&http.Cookie{Name: "pavo", Value: "abcdef"})

	files, err := Process(req, dummy+"/root_storage")
	assert.Nil(err)
	assert.Equal("basta.png", files[0].Filename)
	assert.Equal("image", files[0].BaseMime)

}

func TestUploadChunked(t *testing.T) {
	assert := assert.New(t)
	storage := dummy + "/root_storage"
	fname := dummy + "/kino.jpg"
	f, _ := os.Open(fname)
	defer f.Close()

	cookie := &http.Cookie{Name: "pavo", Value: uuid.New()}

	req := createChunkRequest(f, 0, 24999)
	req.AddCookie(cookie)
	files, err := Process(req, storage)
	assert.Equal(Incomplete, err)
	assert.Equal(25000, int(files[0].Size))

	req = createChunkRequest(f, 25000, 49999)
	req.AddCookie(cookie)
	files, err = Process(req, storage)
	assert.Equal(Incomplete, err)
	assert.Equal(50000, int(files[0].Size))

	req = createChunkRequest(f, 50000, 52096)
	req.AddCookie(cookie)
	files, err = Process(req, storage)
	assert.Nil(err)
	assert.Equal(52097, int(files[0].Size))
	assert.Equal("kino.jpg", files[0].Filename)
}

func createChunkRequest(f *os.File, start int64, end int64) *http.Request {
	var body bytes.Buffer
	mw := multipart.NewWriter(&body)
	fi, _ := f.Stat()
	fw, _ := mw.CreateFormFile("files[]", fi.Name())

	io.CopyN(fw, f, end-start+1)
	mw.Close()

	req, _ := http.NewRequest("POST", "/files", &body)
	req.Header.Set("Content-Type", mw.FormDataContentType())
	req.Header.Set("Content-Disposition", `attachment; filename="`+fi.Name()+`"`)
	req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, fi.Size()))

	return req
}

func TestTempFileChunks(t *testing.T) {
	assert := assert.New(t)

	file, err := TempFileChunks(0, dummy+"/root_storage", "abcdef", "kino.jpg")
	assert.Nil(err)
	assert.NotNil(file)
}

func writeMPBody(fname string, mw *multipart.Writer) error {
	fw, _ := mw.CreateFormFile("files[]", filepath.Base(fname))
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(fw, f)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>