<commit_before>package gxgo

import (
	"github.com/maprost/gox/gxcfg"
	"github.com/maprost/gox/gxutil/gxbash"
	"github.com/maprost/gox/gxutil/gxdocker"
	"github.com/maprost/gox/gxutil/gxlog"
)

func GoDep() error {
	_, err := gxbash.Command(gxlog.LevelInfo, "godep", "save", "./...")
	return err
}

func Compile() error {
	cfg := gxcfg.GetConfig()
	docker := gxdocker.NewRunBuilder(cfg.Docker.Container, cfg.Docker.Image)

	// add project
	docker.Value(cfg.FullProjectPath, "/go/"+cfg.ProjectPath)
	docker.Execute("cd /go/" + cfg.CmdPath + " && echo \"build\" && go build -o "+cfg.Name+"_gx")

	_, err := docker.Run()

	return err
}

func Test() {

}

func BuildImage(cfgFile string) {

}

func Run() {

}

func Remove() error {
	return gxdocker.StopAndRemove(gxcfg.GetConfig().Docker.Container)
}

func PullImage() error {
	return gxdocker.Pull(gxcfg.GetConfig().Docker.Image)
}

func runDockerCommand(docker gxdocker.RunBuilder, command string) {

	//docker.Value("")
	//
	//docker_run.value(base.path(0), "/go/%s" % self.property.path())
	//docker_run.value("%s/project.json" % self.property.root_path(), "/go/project.json")
	//docker_run.value("%s/bin" % self.property.root_path(), "/go/bin")
	//
	//# add dependencies
	//for dep in self.property.dependencies():
	//system_path = "%s/%s" % (self.property.root_path(), self.property.dependency_path(dep))
	//docker_path = "/go/%s" % self.property.dependency_path(dep)
	//if self.property.is_dependency_type_service(dep):
	//system_path += "/client"
	//docker_path += "/client"
	//
	//docker_run.value(system_path, docker_path)
	//
	//docker_run.execute("cd /go/src/rpp.de/%s" % self.property.name() + " && " +
	//	command + " && echo 'go finish #Code445#'")
	//build_output = docker_run.run()
	//log.info(build_output)
	//self.remove()
	//
	//# check if there is an error
	//if "go finish #Code445#" not in build_output:
	//exit(1)
}
<commit_msg>improve build mode<commit_after>package gxgo

import (
	"github.com/maprost/gox/gxcfg"
	"github.com/maprost/gox/gxutil/gxbash"
	"github.com/maprost/gox/gxutil/gxdocker"
	"github.com/maprost/gox/gxutil/gxlog"
)

func GoDep() error {
	_, err := gxbash.Command(gxlog.LevelInfo, "godep", "save", "./...")
	return err
}

func Compile() error {
	cfg := gxcfg.GetConfig()
	docker := gxdocker.NewRunBuilder(cfg.Docker.Container, cfg.Docker.Image)

	// add project
	docker.Value(cfg.FullProjectPath, "/go/"+cfg.ProjectPath)
	docker.Execute("cd /go/" + cfg.CmdPath + " && echo \"build\" && go build -o " + cfg.Name + "_gx")

	_, err := docker.Run()

	return err
}

func Test() {

}

func BuildImage(cfgFile string) {

}

func Run() {

}

func Remove() error {
	return gxdocker.StopAndRemove(gxcfg.GetConfig().Docker.Container)
}

func PullImage() error {
	return gxdocker.Pull(gxcfg.GetConfig().Docker.Image)
}

func runDockerCommand(docker gxdocker.RunBuilder, command string) {

	//docker.Value("")
	//
	//docker_run.value(base.path(0), "/go/%s" % self.property.path())
	//docker_run.value("%s/project.json" % self.property.root_path(), "/go/project.json")
	//docker_run.value("%s/bin" % self.property.root_path(), "/go/bin")
	//
	//# add dependencies
	//for dep in self.property.dependencies():
	//system_path = "%s/%s" % (self.property.root_path(), self.property.dependency_path(dep))
	//docker_path = "/go/%s" % self.property.dependency_path(dep)
	//if self.property.is_dependency_type_service(dep):
	//system_path += "/client"
	//docker_path += "/client"
	//
	//docker_run.value(system_path, docker_path)
	//
	//docker_run.execute("cd /go/src/rpp.de/%s" % self.property.name() + " && " +
	//	command + " && echo 'go finish #Code445#'")
	//build_output = docker_run.run()
	//log.info(build_output)
	//self.remove()
	//
	//# check if there is an error
	//if "go finish #Code445#" not in build_output:
	//exit(1)
}
<|endoftext|>


<commit_before>package docker

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"time"

	"github.com/docker/docker/pkg/stdcopy"
	"github.com/docker/docker/pkg/term"
	"github.com/docker/docker/utils"
)

const (
	APIVERSION        = 1.9
	DEFAULTHTTPPORT   = 2375
	DEFAULTUNIXSOCKET = "/var/run/docker.sock"
	DEFAULTPROTOCOL   = "unix"
	DEFAULTTAG        = "latest"
	VERSION           = "0.8.0"
)

// Enables verbose logging to the Terminal window
var Logging = true

// New creates an instance of the Docker Client
func New() *Client {
	return NewHost("")
}

func NewHost(uri string) *Client {
	var cli, _ = NewHostCert(uri, nil, nil)
	return cli
}

func NewHostCertFile(uri, cert, key string) (*Client, error) {
	if len(key) == 0 || len(cert) == 0 {
		return NewHostCert(uri, nil, nil)
	}
	certfile, err := ioutil.ReadFile(cert)
	if err != nil {
		return nil, err
	}
	keyfile, err := ioutil.ReadFile(key)
	if err != nil {
		return nil, err
	}
	return NewHostCert(uri, certfile, keyfile)
}

func NewHostCert(uri string, cert, key []byte) (*Client, error) {
	var host = GetHost(uri)
	var proto, addr = SplitProtoAddr(host)

	var cli = new(Client)
	cli.proto = proto
	cli.addr = addr
	cli.scheme = "http"
	cli.Images = &ImageService{cli}
	cli.Containers = &ContainerService{cli}

	// if no certificate is provided returns the
	// client with no TLS configured.
	if cert == nil || key == nil || len(cert) == 0 || len(key) == 0 {
		cli.trans = &http.Transport{
			Dial: func(dial_network, dial_addr string) (net.Conn, error) {
				return net.DialTimeout(cli.proto, cli.addr, 32*time.Second)
			},
		}
		return cli, nil
	}

	// loads the key value pair in pem format
	pem, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return nil, err
	}

	// setup the client TLS and store the certificate.
	// also skip verification since we are (typically)
	// going to be using certs for IP addresses.
	cli.scheme = "https"
	cli.tls = new(tls.Config)
	cli.tls.InsecureSkipVerify = true
	cli.tls.Certificates = []tls.Certificate{pem}

	// disable compression for local socket communication.
	if cli.proto == DEFAULTPROTOCOL {
		cli.trans.DisableCompression = true
	}

	// creates a transport that uses the custom tls configuration
	// to securely connect to remote Docker clients.
	cli.trans = &http.Transport{
		TLSClientConfig: cli.tls,
		Dial: func(dial_network, dial_addr string) (net.Conn, error) {
			return net.DialTimeout(cli.proto, cli.addr, 32*time.Second)
		},
	}

	return cli, nil
}

// GetHost returns the Docker Host address in order to
// connect to the Docker Daemon. It implements a very
// simple set of fallthrough logic to determine which
// address to use.
func GetHost(host string) string {
	// if a default value was provided this
	// shoudl be used
	if len(host) != 0 {
		return host
	}
	// else attempt to use the DOCKER_HOST
	// environment variable
	var env = os.Getenv("DOCKER_HOST")
	if len(env) != 0 {
		return env
	}
	// else check to see if the default unix
	// socket exists and return
	_, err := os.Stat(DEFAULTUNIXSOCKET)
	if err == nil {
		return fmt.Sprintf("%s://%s", DEFAULTPROTOCOL, DEFAULTUNIXSOCKET)
	}
	// else return the standard TCP address
	return fmt.Sprintf("tcp://0.0.0.0:%d", DEFAULTHTTPPORT)
}

// SplitProtoAddr is a helper function that splits
// a host into Protocol and Address.
func SplitProtoAddr(host string) (string, string) {
	var parts = strings.Split(host, "://")
	var proto, addr string
	switch {
	case len(parts) == 2:
		proto = parts[0]
		addr = parts[1]
	default:
		proto = "tcp"
		addr = parts[0]
	}
	return proto, addr
}

type Client struct {
	tls    *tls.Config
	trans  *http.Transport
	scheme string
	proto  string
	addr   string

	Images     *ImageService
	Containers *ContainerService
}

var (
	// Returned if the specified resource does not exist.
	ErrNotFound = errors.New("Not Found")

	// Return if something going wrong
	ErrInternalServer = errors.New("Internal Server Error")

	// Returned if the caller attempts to make a call or modify a resource
	// for which the caller is not authorized.
	//
	// The request was a valid request, the caller's authentication credentials
	// succeeded but those credentials do not grant the caller permission to
	// access the resource.
	ErrForbidden = errors.New("Forbidden")

	// Returned if the call requires authentication and either the credentials
	// provided failed or no credentials were provided.
	ErrNotAuthorized = errors.New("Unauthorized")

	// Returned if the caller submits a badly formed request. For example,
	// the caller can receive this return if you forget a required parameter.
	ErrBadRequest = errors.New("Bad Request")
)

// helper function used to make HTTP requests to the Docker daemon.
func (c *Client) do(method, path string, in, out interface{}) error {
	// if data input is provided, serialize to JSON
	var payload io.Reader
	if in != nil {
		buf, err := json.Marshal(in)
		if err != nil {
			return err
		}
		payload = bytes.NewBuffer(buf)
	}

	// create the request
	req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), payload)
	if err != nil {
		return err
	}

	// set the appropariate headers
	req.Header = http.Header{}
	req.Header.Set("User-Agent", "Docker-Client/"+VERSION)
	req.Header.Set("Content-Type", "application/json")

	// dial the host server
	req.URL.Host = c.addr
	req.URL.Scheme = "http"
	if c.tls != nil {
		req.URL.Scheme = "https"
	}

	resp, err := c.HTTPClient().Do(req)
	if err != nil {
		return err
	}

	// make sure we defer close the body
	defer resp.Body.Close()

	// Check for an http error status (ie not 200 StatusOK)
	switch resp.StatusCode {
	case 500:
		return ErrInternalServer
	case 404:
		return ErrNotFound
	case 403:
		return ErrForbidden
	case 401:
		return ErrNotAuthorized
	case 400:
		return ErrBadRequest
	}

	// Decode the JSON response
	if out != nil {
		return json.NewDecoder(resp.Body).Decode(out)
	}

	return nil
}

func (c *Client) hijack(method, path string, setRawTerminal bool, out io.Writer) error {
	req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), nil)
	if err != nil {
		return err
	}

	req.Header.Set("User-Agent", "Docker-Client/"+VERSION)
	req.Header.Set("Content-Type", "plain/text")
	req.Host = c.addr

	dial, err := c.Dial()
	if err != nil {
		if strings.Contains(err.Error(), "connection refused") {
			return fmt.Errorf("Can't connect to docker daemon. Is 'docker -d' running on this host?")
		}
		return err
	}
	clientconn := httputil.NewClientConn(dial, nil)
	defer clientconn.Close()

	// Server hijacks the connection, error 'connection closed' expected
	clientconn.Do(req)

	// Hijack the connection to read / write
	rwc, br := clientconn.Hijack()
	defer rwc.Close()

	// launch a goroutine to copy the stream
	// of build output to the writer.
	errStdout := make(chan error, 1)
	go func() {
		var err error
		if setRawTerminal {
			_, err = io.Copy(out, br)
		} else {
			_, err = stdcopy.StdCopy(out, out, br)
		}

		errStdout <- err
	}()

	// wait for a response
	if err := <-errStdout; err != nil {
		return err
	}
	return nil
}

func (c *Client) stream(method, path string, in io.Reader, out io.Writer, headers http.Header) error {
	if (method == "POST" || method == "PUT") && in == nil {
		in = bytes.NewReader(nil)
	}

	// setup the request
	req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), in)
	if err != nil {
		return err
	}

	// set default headers
	req.Header = headers
	req.Header.Set("User-Agent", "Docker-Client/0.6.4")
	req.Header.Set("Content-Type", "plain/text")

	// dial the host server
	req.URL.Host = c.addr
	req.URL.Scheme = "http"
	if c.tls != nil {
		req.URL.Scheme = "https"
	}

	resp, err := c.HTTPClient().Do(req)
	if err != nil {
		return err
	}

	// make sure we defer close the body
	defer resp.Body.Close()

	// Check for an http error status (ie not 200 StatusOK)
	switch resp.StatusCode {
	case 500:
		return ErrInternalServer
	case 404:
		return ErrNotFound
	case 403:
		return ErrForbidden
	case 401:
		return ErrNotAuthorized
	case 400:
		return ErrBadRequest
	}

	// If no output we exit now with no errors
	if out == nil {
		io.Copy(ioutil.Discard, resp.Body)
		return nil
	}

	// copy the output stream to the writer
	if resp.Header.Get("Content-Type") == "application/json" {
		var terminalFd = os.Stdin.Fd()
		var isTerminal = term.IsTerminal(terminalFd)

		// it may not make sense to put this code here, but it works for
		// us at the moment, and I don't feel like refactoring
		return utils.DisplayJSONMessagesStream(resp.Body, out, terminalFd, isTerminal)
	}
	// otherwise plain text
	if _, err := io.Copy(out, resp.Body); err != nil {
		return err
	}

	return nil
}

func (c *Client) HTTPClient() *http.Client {
	if c.trans != nil {
		return &http.Client{Transport: c.trans}
	}
	return &http.Client{
		// WARN Leak Transport's Pooling Connection
		Transport: &http.Transport{
			Dial: func(dial_network, dial_addr string) (net.Conn, error) {
				return net.DialTimeout(c.proto, c.addr, 32*time.Second)
			},
		},
	}
}

func (c *Client) Dial() (net.Conn, error) {
	if c.tls != nil && c.proto != "unix" {
		return tls.Dial(c.proto, c.addr, c.tls)
	}
	return net.Dial(c.proto, c.addr)
}

func (c *Client) CloseIdleConnections() {
	if c.trans != nil {
		c.trans.CloseIdleConnections()
	}
}
<commit_msg>Fix dependency: github.com/docker/docker/utils was moved to pkg/jsonmessage<commit_after>package docker

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"time"

	"github.com/docker/docker/pkg/jsonmessage"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/docker/docker/pkg/term"
)

const (
	APIVERSION        = 1.9
	DEFAULTHTTPPORT   = 2375
	DEFAULTUNIXSOCKET = "/var/run/docker.sock"
	DEFAULTPROTOCOL   = "unix"
	DEFAULTTAG        = "latest"
	VERSION           = "0.8.0"
)

// Enables verbose logging to the Terminal window
var Logging = true

// New creates an instance of the Docker Client
func New() *Client {
	return NewHost("")
}

func NewHost(uri string) *Client {
	var cli, _ = NewHostCert(uri, nil, nil)
	return cli
}

func NewHostCertFile(uri, cert, key string) (*Client, error) {
	if len(key) == 0 || len(cert) == 0 {
		return NewHostCert(uri, nil, nil)
	}
	certfile, err := ioutil.ReadFile(cert)
	if err != nil {
		return nil, err
	}
	keyfile, err := ioutil.ReadFile(key)
	if err != nil {
		return nil, err
	}
	return NewHostCert(uri, certfile, keyfile)
}

func NewHostCert(uri string, cert, key []byte) (*Client, error) {
	var host = GetHost(uri)
	var proto, addr = SplitProtoAddr(host)

	var cli = new(Client)
	cli.proto = proto
	cli.addr = addr
	cli.scheme = "http"
	cli.Images = &ImageService{cli}
	cli.Containers = &ContainerService{cli}

	// if no certificate is provided returns the
	// client with no TLS configured.
	if cert == nil || key == nil || len(cert) == 0 || len(key) == 0 {
		cli.trans = &http.Transport{
			Dial: func(dial_network, dial_addr string) (net.Conn, error) {
				return net.DialTimeout(cli.proto, cli.addr, 32*time.Second)
			},
		}
		return cli, nil
	}

	// loads the key value pair in pem format
	pem, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return nil, err
	}

	// setup the client TLS and store the certificate.
	// also skip verification since we are (typically)
	// going to be using certs for IP addresses.
	cli.scheme = "https"
	cli.tls = new(tls.Config)
	cli.tls.InsecureSkipVerify = true
	cli.tls.Certificates = []tls.Certificate{pem}

	// disable compression for local socket communication.
	if cli.proto == DEFAULTPROTOCOL {
		cli.trans.DisableCompression = true
	}

	// creates a transport that uses the custom tls configuration
	// to securely connect to remote Docker clients.
	cli.trans = &http.Transport{
		TLSClientConfig: cli.tls,
		Dial: func(dial_network, dial_addr string) (net.Conn, error) {
			return net.DialTimeout(cli.proto, cli.addr, 32*time.Second)
		},
	}

	return cli, nil
}

// GetHost returns the Docker Host address in order to
// connect to the Docker Daemon. It implements a very
// simple set of fallthrough logic to determine which
// address to use.
func GetHost(host string) string {
	// if a default value was provided this
	// shoudl be used
	if len(host) != 0 {
		return host
	}
	// else attempt to use the DOCKER_HOST
	// environment variable
	var env = os.Getenv("DOCKER_HOST")
	if len(env) != 0 {
		return env
	}
	// else check to see if the default unix
	// socket exists and return
	_, err := os.Stat(DEFAULTUNIXSOCKET)
	if err == nil {
		return fmt.Sprintf("%s://%s", DEFAULTPROTOCOL, DEFAULTUNIXSOCKET)
	}
	// else return the standard TCP address
	return fmt.Sprintf("tcp://0.0.0.0:%d", DEFAULTHTTPPORT)
}

// SplitProtoAddr is a helper function that splits
// a host into Protocol and Address.
func SplitProtoAddr(host string) (string, string) {
	var parts = strings.Split(host, "://")
	var proto, addr string
	switch {
	case len(parts) == 2:
		proto = parts[0]
		addr = parts[1]
	default:
		proto = "tcp"
		addr = parts[0]
	}
	return proto, addr
}

type Client struct {
	tls    *tls.Config
	trans  *http.Transport
	scheme string
	proto  string
	addr   string

	Images     *ImageService
	Containers *ContainerService
}

var (
	// Returned if the specified resource does not exist.
	ErrNotFound = errors.New("Not Found")

	// Return if something going wrong
	ErrInternalServer = errors.New("Internal Server Error")

	// Returned if the caller attempts to make a call or modify a resource
	// for which the caller is not authorized.
	//
	// The request was a valid request, the caller's authentication credentials
	// succeeded but those credentials do not grant the caller permission to
	// access the resource.
	ErrForbidden = errors.New("Forbidden")

	// Returned if the call requires authentication and either the credentials
	// provided failed or no credentials were provided.
	ErrNotAuthorized = errors.New("Unauthorized")

	// Returned if the caller submits a badly formed request. For example,
	// the caller can receive this return if you forget a required parameter.
	ErrBadRequest = errors.New("Bad Request")
)

// helper function used to make HTTP requests to the Docker daemon.
func (c *Client) do(method, path string, in, out interface{}) error {
	// if data input is provided, serialize to JSON
	var payload io.Reader
	if in != nil {
		buf, err := json.Marshal(in)
		if err != nil {
			return err
		}
		payload = bytes.NewBuffer(buf)
	}

	// create the request
	req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), payload)
	if err != nil {
		return err
	}

	// set the appropariate headers
	req.Header = http.Header{}
	req.Header.Set("User-Agent", "Docker-Client/"+VERSION)
	req.Header.Set("Content-Type", "application/json")

	// dial the host server
	req.URL.Host = c.addr
	req.URL.Scheme = "http"
	if c.tls != nil {
		req.URL.Scheme = "https"
	}

	resp, err := c.HTTPClient().Do(req)
	if err != nil {
		return err
	}

	// make sure we defer close the body
	defer resp.Body.Close()

	// Check for an http error status (ie not 200 StatusOK)
	switch resp.StatusCode {
	case 500:
		return ErrInternalServer
	case 404:
		return ErrNotFound
	case 403:
		return ErrForbidden
	case 401:
		return ErrNotAuthorized
	case 400:
		return ErrBadRequest
	}

	// Decode the JSON response
	if out != nil {
		return json.NewDecoder(resp.Body).Decode(out)
	}

	return nil
}

func (c *Client) hijack(method, path string, setRawTerminal bool, out io.Writer) error {
	req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), nil)
	if err != nil {
		return err
	}

	req.Header.Set("User-Agent", "Docker-Client/"+VERSION)
	req.Header.Set("Content-Type", "plain/text")
	req.Host = c.addr

	dial, err := c.Dial()
	if err != nil {
		if strings.Contains(err.Error(), "connection refused") {
			return fmt.Errorf("Can't connect to docker daemon. Is 'docker -d' running on this host?")
		}
		return err
	}
	clientconn := httputil.NewClientConn(dial, nil)
	defer clientconn.Close()

	// Server hijacks the connection, error 'connection closed' expected
	clientconn.Do(req)

	// Hijack the connection to read / write
	rwc, br := clientconn.Hijack()
	defer rwc.Close()

	// launch a goroutine to copy the stream
	// of build output to the writer.
	errStdout := make(chan error, 1)
	go func() {
		var err error
		if setRawTerminal {
			_, err = io.Copy(out, br)
		} else {
			_, err = stdcopy.StdCopy(out, out, br)
		}

		errStdout <- err
	}()

	// wait for a response
	if err := <-errStdout; err != nil {
		return err
	}
	return nil
}

func (c *Client) stream(method, path string, in io.Reader, out io.Writer, headers http.Header) error {
	if (method == "POST" || method == "PUT") && in == nil {
		in = bytes.NewReader(nil)
	}

	// setup the request
	req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), in)
	if err != nil {
		return err
	}

	// set default headers
	req.Header = headers
	req.Header.Set("User-Agent", "Docker-Client/0.6.4")
	req.Header.Set("Content-Type", "plain/text")

	// dial the host server
	req.URL.Host = c.addr
	req.URL.Scheme = "http"
	if c.tls != nil {
		req.URL.Scheme = "https"
	}

	resp, err := c.HTTPClient().Do(req)
	if err != nil {
		return err
	}

	// make sure we defer close the body
	defer resp.Body.Close()

	// Check for an http error status (ie not 200 StatusOK)
	switch resp.StatusCode {
	case 500:
		return ErrInternalServer
	case 404:
		return ErrNotFound
	case 403:
		return ErrForbidden
	case 401:
		return ErrNotAuthorized
	case 400:
		return ErrBadRequest
	}

	// If no output we exit now with no errors
	if out == nil {
		io.Copy(ioutil.Discard, resp.Body)
		return nil
	}

	// copy the output stream to the writer
	if resp.Header.Get("Content-Type") == "application/json" {
		var terminalFd = os.Stdin.Fd()
		var isTerminal = term.IsTerminal(terminalFd)

		// it may not make sense to put this code here, but it works for
		// us at the moment, and I don't feel like refactoring
		return jsonmessage.DisplayJSONMessagesStream(resp.Body, out, terminalFd, isTerminal)
	}
	// otherwise plain text
	if _, err := io.Copy(out, resp.Body); err != nil {
		return err
	}

	return nil
}

func (c *Client) HTTPClient() *http.Client {
	if c.trans != nil {
		return &http.Client{Transport: c.trans}
	}
	return &http.Client{
		// WARN Leak Transport's Pooling Connection
		Transport: &http.Transport{
			Dial: func(dial_network, dial_addr string) (net.Conn, error) {
				return net.DialTimeout(c.proto, c.addr, 32*time.Second)
			},
		},
	}
}

func (c *Client) Dial() (net.Conn, error) {
	if c.tls != nil && c.proto != "unix" {
		return tls.Dial(c.proto, c.addr, c.tls)
	}
	return net.Dial(c.proto, c.addr)
}

func (c *Client) CloseIdleConnections() {
	if c.trans != nil {
		c.trans.CloseIdleConnections()
	}
}
<|endoftext|>


<commit_before>package output

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/Jeffail/benthos/v3/lib/log"
	"github.com/Jeffail/benthos/v3/lib/metrics"
	"github.com/Jeffail/benthos/v3/lib/types"
)

//------------------------------------------------------------------------------

func init() {
	Constructors[TypeResource] = TypeSpec{
		constructor: NewResource,
		Summary: `
Resource is an output type that runs a resource output by its name. This output
allows you to run the same configured output resource in multiple places.`,
		Description: `
Resource outputs also have the advantage of name based metrics and logging. For
example, the config:

` + "``` yaml" + `
output:
  broker:
    pattern: fan_out
    outputs:
    - kafka:
        addresses: [ TODO ]
        topic: foo
    - gcp_pubsub:
        project: bar
        topic: baz
` + "```" + `

Is equivalent to:

` + "``` yaml" + `
output:
  broker:
    pattern: fan_out
    outputs:
    - resource: foo
    - resource: bar

resources:
  outputs:
    foo:
      kafka_balanced:
        addresses: [ TODO ]
        topic: foo

    bar:
      gcp_pubsub:
        project: bar
        topic: baz
 ` + "```" + `

But now the metrics path of Kafka output will be
` + "`resources.outputs.foo`" + `, this way of flattening observability
labels becomes more useful as configs get larger and more nested.`,
	}
}

//------------------------------------------------------------------------------

type outputProvider interface {
	GetOutput(name string) (types.OutputWriter, error)
}

//------------------------------------------------------------------------------

// Resource is a processor that returns the result of a output resource.
type Resource struct {
	mgr   outputProvider
	name  string
	log   log.Modular
	stats metrics.Type

	transactions <-chan types.Transaction

	ctx  context.Context
	done func()
}

// NewResource returns a resource output.
func NewResource(
	conf Config, mgr types.Manager, log log.Modular, stats metrics.Type,
) (Type, error) {
	// TODO: V4 Remove this
	outputProvider, ok := mgr.(outputProvider)
	if !ok {
		return nil, errors.New("manager does not support output resources")
	}

	if _, err := outputProvider.GetOutput(conf.Resource); err != nil {
		return nil, fmt.Errorf("failed to obtain output resource '%v': %v", conf.Resource, err)
	}

	ctx, done := context.WithCancel(context.Background())
	return &Resource{
		mgr:  outputProvider,
		name: conf.Resource,
		log:  log,
		ctx:  ctx,
		done: done,
	}, nil
}

//------------------------------------------------------------------------------

func (r *Resource) loop() {
	// Metrics paths
	var (
		mCount       = r.stats.GetCounter("count")
		mErrNotFound = r.stats.GetCounter("error_not_found")
	)

	var ts *types.Transaction
	for {
		if ts == nil {
			select {
			case t, open := <-r.transactions:
				if !open {
					r.done()
					return
				}
				ts = &t
			case <-r.ctx.Done():
				return
			}
		}
		mCount.Incr(1)
		out, err := r.mgr.GetOutput(r.name)
		if err != nil {
			r.log.Debugf("Failed to obtain output resource '%v': %v", r.name, err)
			mErrNotFound.Incr(1)
			select {
			case <-time.After(time.Second):
			case <-r.ctx.Done():
				return
			}
		} else {
			out.WriteTransaction(r.ctx, *ts)
			ts = nil
		}
	}
}

//------------------------------------------------------------------------------

// Consume assigns a messages channel for the output to read.
func (r *Resource) Consume(ts <-chan types.Transaction) error {
	if r.transactions != nil {
		return types.ErrAlreadyStarted
	}
	r.transactions = ts
	go r.loop()
	return nil
}

// Connected returns a boolean indicating whether this output is currently
// connected to its target.
func (r *Resource) Connected() bool {
	out, err := r.mgr.GetOutput(r.name)
	if err != nil {
		r.log.Debugf("Failed to obtain output resource '%v': %v", r.name, err)
		r.mErrNotFound.Incr(1)
		return false
	}
	return out.Connected()
}

// CloseAsync shuts down the output and stops processing requests.
func (r *Resource) CloseAsync() {
	r.done()
}

// WaitForClose blocks until the output has closed down.
func (r *Resource) WaitForClose(timeout time.Duration) error {
	select {
	case <-r.ctx.Done():
	case <-time.After(timeout):
		return types.ErrTimeout
	}
	return nil
}

//------------------------------------------------------------------------------
<commit_msg>Fix test<commit_after>package output

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/Jeffail/benthos/v3/lib/log"
	"github.com/Jeffail/benthos/v3/lib/metrics"
	"github.com/Jeffail/benthos/v3/lib/types"
)

//------------------------------------------------------------------------------

func init() {
	Constructors[TypeResource] = TypeSpec{
		constructor: NewResource,
		Summary: `
Resource is an output type that runs a resource output by its name. This output
allows you to run the same configured output resource in multiple places.`,
		Description: `
Resource outputs also have the advantage of name based metrics and logging. For
example, the config:

` + "``` yaml" + `
output:
  broker:
    pattern: fan_out
    outputs:
    - kafka:
        addresses: [ TODO ]
        topic: foo
    - gcp_pubsub:
        project: bar
        topic: baz
` + "```" + `

Is equivalent to:

` + "``` yaml" + `
output:
  broker:
    pattern: fan_out
    outputs:
    - resource: foo
    - resource: bar

resources:
  outputs:
    foo:
      kafka_balanced:
        addresses: [ TODO ]
        topic: foo

    bar:
      gcp_pubsub:
        project: bar
        topic: baz
 ` + "```" + `

But now the metrics path of Kafka output will be
` + "`resources.outputs.foo`" + `, this way of flattening observability
labels becomes more useful as configs get larger and more nested.`,
	}
}

//------------------------------------------------------------------------------

type outputProvider interface {
	GetOutput(name string) (types.OutputWriter, error)
}

//------------------------------------------------------------------------------

// Resource is a processor that returns the result of a output resource.
type Resource struct {
	mgr   outputProvider
	name  string
	log   log.Modular
	stats metrics.Type

	transactions <-chan types.Transaction

	ctx  context.Context
	done func()

	mErrNotFound metrics.StatCounter
}

// NewResource returns a resource output.
func NewResource(
	conf Config, mgr types.Manager, log log.Modular, stats metrics.Type,
) (Type, error) {
	// TODO: V4 Remove this
	outputProvider, ok := mgr.(outputProvider)
	if !ok {
		return nil, errors.New("manager does not support output resources")
	}

	if _, err := outputProvider.GetOutput(conf.Resource); err != nil {
		return nil, fmt.Errorf("failed to obtain output resource '%v': %v", conf.Resource, err)
	}

	ctx, done := context.WithCancel(context.Background())
	return &Resource{
		mgr:          outputProvider,
		name:         conf.Resource,
		log:          log,
		stats:        stats,
		ctx:          ctx,
		done:         done,
		mErrNotFound: stats.GetCounter("error_not_found"),
	}, nil
}

//------------------------------------------------------------------------------

func (r *Resource) loop() {
	// Metrics paths
	var (
		mCount = r.stats.GetCounter("count")
	)

	var ts *types.Transaction
	for {
		if ts == nil {
			select {
			case t, open := <-r.transactions:
				if !open {
					r.done()
					return
				}
				ts = &t
			case <-r.ctx.Done():
				return
			}
		}
		mCount.Incr(1)
		out, err := r.mgr.GetOutput(r.name)
		if err != nil {
			r.log.Debugf("Failed to obtain output resource '%v': %v", r.name, err)
			r.mErrNotFound.Incr(1)
			select {
			case <-time.After(time.Second):
			case <-r.ctx.Done():
				return
			}
		} else {
			out.WriteTransaction(r.ctx, *ts)
			ts = nil
		}
	}
}

//------------------------------------------------------------------------------

// Consume assigns a messages channel for the output to read.
func (r *Resource) Consume(ts <-chan types.Transaction) error {
	if r.transactions != nil {
		return types.ErrAlreadyStarted
	}
	r.transactions = ts
	go r.loop()
	return nil
}

// Connected returns a boolean indicating whether this output is currently
// connected to its target.
func (r *Resource) Connected() bool {
	out, err := r.mgr.GetOutput(r.name)
	if err != nil {
		r.log.Debugf("Failed to obtain output resource '%v': %v", r.name, err)
		r.mErrNotFound.Incr(1)
		return false
	}
	return out.Connected()
}

// CloseAsync shuts down the output and stops processing requests.
func (r *Resource) CloseAsync() {
	r.done()
}

// WaitForClose blocks until the output has closed down.
func (r *Resource) WaitForClose(timeout time.Duration) error {
	select {
	case <-r.ctx.Done():
	case <-time.After(timeout):
		return types.ErrTimeout
	}
	return nil
}

//------------------------------------------------------------------------------
<|endoftext|>


<commit_before>package worker

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/bitly/go-simplejson"
	"github.com/cenk/backoff"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/context"
	"github.com/travis-ci/worker/metrics"

	gocontext "context"
)

var (
	httpJobQueueNoJobsErr = fmt.Errorf("no jobs available")
	mismatchedJobIDErr    = fmt.Errorf("returned job ID does not match expected")
)

// HTTPJobQueue is a JobQueue that uses http
type HTTPJobQueue struct {
	jobBoardURL  *url.URL
	site         string
	providerName string
	queue        string
	pollInterval time.Duration
	cb           *CancellationBroadcaster

	DefaultLanguage, DefaultDist, DefaultGroup, DefaultOS string
}

type httpFetchJobsRequest struct {
	Jobs []string `json:"jobs"`
}

type httpFetchJobsResponse struct {
	Jobs []string `json:"jobs"`
}

type jobBoardErrorResponse struct {
	Type          string `json:"@type"`
	Error         string `json:"error"`
	UpstreamError string `json:"upstream_error,omitempty"`
}

// NewHTTPJobQueue creates a new job-board job queue
func NewHTTPJobQueue(jobBoardURL *url.URL, site, providerName, queue string,
	cb *CancellationBroadcaster) (*HTTPJobQueue, error) {

	return &HTTPJobQueue{
		jobBoardURL:  jobBoardURL,
		site:         site,
		providerName: providerName,
		queue:        queue,
		pollInterval: time.Second,
		cb:           cb,
	}, nil
}

// Jobs consumes new jobs from job-board
func (q *HTTPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err error) {
	buildJobChan := make(chan Job)
	outChan = buildJobChan
	logger := context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"self": "http_job_queue",
		"inst": fmt.Sprintf("%p", q),
	})

	go func() {
		for {
			logger.Debug("polling for job tick")
			if !q.pollForJob(ctx, buildJobChan) {
				return
			}
			time.Sleep(q.pollInterval)
		}
	}()

	return outChan, nil
}

func (q *HTTPJobQueue) pollForJob(ctx gocontext.Context, buildJobChan chan Job) bool {
	logger := context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"self": "http_job_queue",
		"inst": fmt.Sprintf("%p", q),
	})

	logger.Debug("fetching job id")
	jobID, err := q.fetchJobID(ctx)
	if err != nil {
		logger.WithField("err", err).Debug("continuing after failing to get job id")
		return true
	}
	logger.WithField("job_id", jobID).Debug("fetching complete job")
	buildJob, err := q.fetchJob(ctx, jobID)
	if err != nil {
		logger.WithFields(logrus.Fields{
			"err": err,
			"id":  jobID,
		}).Warn("failed to get complete job")
		return true
	}

	logger.WithField("job_id", jobID).Debug("sending job to output channel")
	jobSendBegin := time.Now()
	select {
	case buildJobChan <- buildJob:
		metrics.TimeSince("travis.worker.job_queue.http.blocking_time", jobSendBegin)
		logger.WithFields(logrus.Fields{
			"source": "http",
			"dur":    time.Since(jobSendBegin),
		}).Info("sent job to output channel")
		return true
	case <-ctx.Done():
		logger.WithField("err", ctx.Err()).Warn("returning from jobs loop due to context done")
		return false
	}
}

func (q *HTTPJobQueue) fetchJobID(ctx gocontext.Context) (uint64, error) {
	return q.postJobs(ctx, []string{})
}

func (q *HTTPJobQueue) refreshJobClaim(ctx gocontext.Context, jobID uint64) error {
	fetchedID, err := q.postJobs(ctx, []string{strconv.Itoa(int(jobID))})
	if err != nil {
		return err
	}

	if fetchedID != jobID {
		return mismatchedJobIDErr
	}

	return nil
}

func (q *HTTPJobQueue) postJobs(ctx gocontext.Context, jobIDs []string) (uint64, error) {
	logger := context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"self": "http_job_queue",
		"inst": fmt.Sprintf("%p", q),
	})

	processorID, ok := context.ProcessorFromContext(ctx)
	if !ok {
		processorID = "unknown-processor"
	}
	jobIDsJSON, err := json.Marshal(&httpFetchJobsRequest{Jobs: jobIDs})
	if err != nil {
		return 0, errors.Wrap(err, "failed to marshal job-board jobs request payload")
	}

	u := *q.jobBoardURL

	query := u.Query()
	query.Add("queue", q.queue)

	u.Path = "/jobs"
	u.RawQuery = query.Encode()

	client := &http.Client{}

	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(jobIDsJSON))
	if err != nil {
		return 0, errors.Wrap(err, "failed to create job-board jobs request")
	}

	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", processorID)
	req = req.WithContext(ctx)

	resp, err := client.Do(req)
	if err != nil {
		return 0, errors.Wrap(err, "failed to make job-board jobs request")
	}

	defer resp.Body.Close()
	fetchResponsePayload := &httpFetchJobsResponse{}
	err = json.NewDecoder(resp.Body).Decode(&fetchResponsePayload)
	if err != nil {
		return 0, errors.Wrap(err, "failed to decode job-board jobs response")
	}

	logger.WithField("jobs", fetchResponsePayload.Jobs).Debug("fetched")
	if len(fetchResponsePayload.Jobs) == 0 {
		return 0, httpJobQueueNoJobsErr
	}

	jobID, err := strconv.ParseUint(fetchResponsePayload.Jobs[0], 10, 64)
	if err != nil {
		return 0, errors.Wrap(err, "failed to parse job ID")
	}

	return jobID, nil
}

func (q *HTTPJobQueue) fetchJob(ctx gocontext.Context, jobID uint64) (Job, error) {
	logger := context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"self": "http_job_queue",
		"inst": fmt.Sprintf("%p", q),
	})

	processorID, ok := context.ProcessorFromContext(ctx)
	if !ok {
		processorID = "unknown-processor"
	}

	buildJob := &httpJob{
		payload: &httpJobPayload{
			Data: &JobPayload{},
		},
		startAttributes: &backend.StartAttributes{},

		refreshClaim: q.generateJobRefreshClaimFunc(jobID),

		jobBoardURL: q.jobBoardURL,
		site:        q.site,
		processorID: processorID,
	}
	startAttrs := &httpJobPayloadStartAttrs{
		Data: &jobPayloadStartAttrs{
			Config: &backend.StartAttributes{},
		},
	}

	u := *q.jobBoardURL
	u.Path = fmt.Sprintf("/jobs/%d", jobID)

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, errors.Wrap(err, "couldn't make job-board job request")
	}

	// TODO: ensure infrastructure is not synonymous with providerName since
	// there's the possibility that a provider has multiple infrastructures, which
	// is expected to be the case with the future cloudbrain provider.
	req.Header.Add("Travis-Infrastructure", q.providerName)
	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", processorID)
	req = req.WithContext(ctx)

	bo := backoff.NewExponentialBackOff()
	bo.MaxInterval = 10 * time.Second
	bo.MaxElapsedTime = 1 * time.Minute

	var resp *http.Response
	err = backoff.Retry(func() (err error) {
		resp, err = (&http.Client{}).Do(req)
		if resp != nil && resp.StatusCode != http.StatusOK {
			logger.WithFields(logrus.Fields{
				"expected_status": http.StatusOK,
				"actual_status":   resp.StatusCode,
			}).Debug("job fetch failed")

			if resp.Body != nil {
				resp.Body.Close()
			}

			return errors.Errorf("expected %d but got %d", http.StatusOK, resp.StatusCode)
		}
		return
	}, bo)

	if err != nil {
		return nil, errors.Wrap(err, "error making job-board job request")
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, errors.Wrap(err, "error reading body from job-board job request")
	}

	err = json.Unmarshal(body, buildJob.payload)
	if err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal job-board payload")
	}

	err = json.Unmarshal(body, &startAttrs)
	if err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal start attributes from job-board")
	}

	rawPayload, err := simplejson.NewJson(body)
	if err != nil {
		return nil, errors.Wrap(err, "failed to parse raw payload with simplejson")
	}
	buildJob.rawPayload = rawPayload.Get("data")

	buildJob.startAttributes = startAttrs.Data.Config
	buildJob.startAttributes.VMType = buildJob.payload.Data.VMType
	buildJob.startAttributes.SetDefaults(q.DefaultLanguage, q.DefaultDist, q.DefaultGroup, q.DefaultOS, VMTypeDefault)

	return buildJob, nil
}

func (q *HTTPJobQueue) generateJobRefreshClaimFunc(jobID uint64) func(gocontext.Context) {
	return func(ctx gocontext.Context) {
		for {
			err := q.refreshJobClaim(ctx, jobID)
			if err == httpJobQueueNoJobsErr {
				context.LoggerFromContext(ctx).WithFields(logrus.Fields{
					"err":    err,
					"job_id": jobID,
				}).Error("failed to refresh claim; cancelling")
				q.cb.Broadcast(jobID)
				return
			}

			if err != nil {
				context.LoggerFromContext(ctx).WithFields(logrus.Fields{
					"err":    err,
					"job_id": jobID,
				}).Error("failed to refresh claim; continuing")
			}
			select {
			case <-ctx.Done():
				return
			case <-time.After(q.pollInterval):
			}
		}
	}
}

// Name returns the name of this queue type, wow!
func (q *HTTPJobQueue) Name() string {
	return "http"
}

// Cleanup does not do anything!
func (q *HTTPJobQueue) Cleanup() error {
	return nil
}
<commit_msg>Reintroduce the ready chan specifically via reclaim loop<commit_after>package worker

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/bitly/go-simplejson"
	"github.com/cenk/backoff"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/context"
	"github.com/travis-ci/worker/metrics"

	gocontext "context"
)

var (
	httpJobQueueNoJobsErr  = fmt.Errorf("no jobs available")
	httpJobRefreshClaimErr = fmt.Errorf("failed to refresh claim")
)

// HTTPJobQueue is a JobQueue that uses http
type HTTPJobQueue struct {
	jobBoardURL  *url.URL
	site         string
	providerName string
	queue        string
	pollInterval time.Duration
	cb           *CancellationBroadcaster

	DefaultLanguage, DefaultDist, DefaultGroup, DefaultOS string
}

type httpFetchJobsRequest struct {
	Jobs []string `json:"jobs"`
}

type httpFetchJobsResponse struct {
	Jobs []string `json:"jobs"`
}

type jobBoardErrorResponse struct {
	Type          string `json:"@type"`
	Error         string `json:"error"`
	UpstreamError string `json:"upstream_error,omitempty"`
}

// NewHTTPJobQueue creates a new job-board job queue
func NewHTTPJobQueue(jobBoardURL *url.URL, site, providerName, queue string,
	cb *CancellationBroadcaster) (*HTTPJobQueue, error) {

	return &HTTPJobQueue{
		jobBoardURL:  jobBoardURL,
		site:         site,
		providerName: providerName,
		queue:        queue,
		pollInterval: time.Second,
		cb:           cb,
	}, nil
}

// Jobs consumes new jobs from job-board
func (q *HTTPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err error) {
	buildJobChan := make(chan Job)
	outChan = buildJobChan
	logger := context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"self": "http_job_queue",
		"inst": fmt.Sprintf("%p", q),
	})

	go func() {
		defer close(buildJobChan)

		for {
			logger.Debug("polling for job tick")
			keepPolling, readyChan := q.pollForJob(ctx, buildJobChan)
			if readyChan != nil {
				logger.Debug("blocking on ready channel recv")
				<-readyChan
			}
			if !keepPolling {
				return
			}
			time.Sleep(q.pollInterval)
		}
	}()

	return outChan, nil
}

func (q *HTTPJobQueue) pollForJob(ctx gocontext.Context, buildJobChan chan Job) (bool, <-chan struct{}) {
	logger := context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"self": "http_job_queue",
		"inst": fmt.Sprintf("%p", q),
	})

	logger.Debug("fetching job id")
	jobID, err := q.fetchJobID(ctx)
	if err != nil {
		logger.WithField("err", err).Debug("continuing after failing to get job id")
		return true, nil
	}
	logger.WithField("job_id", jobID).Debug("fetching complete job")
	buildJob, readyChan, err := q.fetchJob(ctx, jobID)
	if err != nil {
		logger.WithFields(logrus.Fields{
			"err": err,
			"id":  jobID,
		}).Warn("failed to get complete job")
		return true, nil
	}

	logger.WithField("job_id", jobID).Debug("sending job to output channel")
	jobSendBegin := time.Now()
	select {
	case buildJobChan <- buildJob:
		metrics.TimeSince("travis.worker.job_queue.http.blocking_time", jobSendBegin)
		logger.WithFields(logrus.Fields{
			"source": "http",
			"dur":    time.Since(jobSendBegin),
		}).Info("sent job to output channel")
		return true, readyChan
	case <-ctx.Done():
		logger.WithField("err", ctx.Err()).Warn("returning from jobs loop due to context done")
		return false, nil
	}
}

func (q *HTTPJobQueue) fetchJobID(ctx gocontext.Context) (uint64, error) {
	logger := context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"self": "http_job_queue",
		"inst": fmt.Sprintf("%p", q),
	})

	processorID, ok := context.ProcessorFromContext(ctx)
	if !ok {
		processorID = "unknown-processor"
	}

	u := *q.jobBoardURL

	query := u.Query()
	query.Add("queue", q.queue)

	u.Path = "/jobs/pop"
	u.RawQuery = query.Encode()

	client := &http.Client{}

	req, err := http.NewRequest("POST", u.String(), nil)
	if err != nil {
		return 0, errors.Wrap(err, "failed to create job-board job pop request")
	}

	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", processorID)
	req = req.WithContext(ctx)

	resp, err := client.Do(req)
	if err != nil {
		return 0, errors.Wrap(err, "failed to make job-board job pop request")
	}

	defer resp.Body.Close()

	if resp.StatusCode == http.StatusNoContent {
		return 0, httpJobQueueNoJobsErr
	}

	fetchResponsePayload := map[string]string{"job_id": ""}
	err = json.NewDecoder(resp.Body).Decode(&fetchResponsePayload)
	if err != nil {
		return 0, errors.Wrap(err, "failed to decode job-board job pop response")
	}

	fetchedJobID, err := strconv.ParseUint(fetchResponsePayload["job_id"], 10, 64)
	if err != nil {
		return 0, errors.Wrap(err, "failed to parse job ID")
	}

	logger.WithField("job_id", fetchedJobID).Debug("fetched")
	return fetchedJobID, nil
}

func (q *HTTPJobQueue) refreshJobClaim(ctx gocontext.Context, jobID uint64) error {
	logger := context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"self":   "http_job_queue",
		"job_id": jobID,
		"inst":   fmt.Sprintf("%p", q),
	})

	processorID, ok := context.ProcessorFromContext(ctx)
	if !ok {
		processorID = "unknown-processor"
	}

	u := *q.jobBoardURL

	query := u.Query()
	query.Add("queue", q.queue)

	u.Path = fmt.Sprintf("/jobs/%v/claim", jobID)
	u.RawQuery = query.Encode()

	client := &http.Client{}

	req, err := http.NewRequest("POST", u.String(), nil)
	if err != nil {
		return errors.Wrap(err, "failed to create job-board job claim request")
	}

	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", processorID)
	req = req.WithContext(ctx)

	resp, err := client.Do(req)
	if err != nil {
		return errors.Wrap(err, "failed to make job-board job claim request")
	}

	resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		logger.WithField("response_code", resp.StatusCode).Debug("non-200 response code")
		return httpJobRefreshClaimErr
	}

	logger.Debug("refreshed claim")
	return nil
}

func (q *HTTPJobQueue) fetchJob(ctx gocontext.Context, jobID uint64) (Job, <-chan struct{}, error) {
	logger := context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"self": "http_job_queue",
		"inst": fmt.Sprintf("%p", q),
	})

	processorID, ok := context.ProcessorFromContext(ctx)
	if !ok {
		processorID = "unknown-processor"
	}

	refreshClaimFunc, readyChan := q.generateJobRefreshClaimFunc(jobID)

	buildJob := &httpJob{
		payload: &httpJobPayload{
			Data: &JobPayload{},
		},
		startAttributes: &backend.StartAttributes{},

		refreshClaim: refreshClaimFunc,

		jobBoardURL: q.jobBoardURL,
		site:        q.site,
		processorID: processorID,
	}
	startAttrs := &httpJobPayloadStartAttrs{
		Data: &jobPayloadStartAttrs{
			Config: &backend.StartAttributes{},
		},
	}

	u := *q.jobBoardURL
	u.Path = fmt.Sprintf("/jobs/%d", jobID)

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, nil, errors.Wrap(err, "couldn't make job-board job request")
	}

	// TODO: ensure infrastructure is not synonymous with providerName since
	// there's the possibility that a provider has multiple infrastructures, which
	// is expected to be the case with the future cloudbrain provider.
	req.Header.Add("Travis-Infrastructure", q.providerName)
	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", processorID)
	req = req.WithContext(ctx)

	bo := backoff.NewExponentialBackOff()
	bo.MaxInterval = 10 * time.Second
	bo.MaxElapsedTime = 1 * time.Minute

	var resp *http.Response
	err = backoff.Retry(func() (err error) {
		resp, err = (&http.Client{}).Do(req)
		if resp != nil && resp.StatusCode != http.StatusOK {
			logger.WithFields(logrus.Fields{
				"expected_status": http.StatusOK,
				"actual_status":   resp.StatusCode,
			}).Debug("job fetch failed")

			if resp.Body != nil {
				resp.Body.Close()
			}

			return errors.Errorf("expected %d but got %d", http.StatusOK, resp.StatusCode)
		}
		return
	}, bo)

	if err != nil {
		return nil, nil, errors.Wrap(err, "error making job-board job request")
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, nil, errors.Wrap(err, "error reading body from job-board job request")
	}

	err = json.Unmarshal(body, buildJob.payload)
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to unmarshal job-board payload")
	}

	err = json.Unmarshal(body, &startAttrs)
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to unmarshal start attributes from job-board")
	}

	rawPayload, err := simplejson.NewJson(body)
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to parse raw payload with simplejson")
	}
	buildJob.rawPayload = rawPayload.Get("data")

	buildJob.startAttributes = startAttrs.Data.Config
	buildJob.startAttributes.VMType = buildJob.payload.Data.VMType
	buildJob.startAttributes.SetDefaults(q.DefaultLanguage, q.DefaultDist, q.DefaultGroup, q.DefaultOS, VMTypeDefault)

	return buildJob, readyChan, nil
}

func (q *HTTPJobQueue) generateJobRefreshClaimFunc(jobID uint64) (func(gocontext.Context), <-chan struct{}) {
	readyChan := make(chan struct{})

	return func(ctx gocontext.Context) {
		defer func() {
			readyChan <- struct{}{}
			close(readyChan)
		}()

		for {
			err := q.refreshJobClaim(ctx, jobID)
			if err == httpJobRefreshClaimErr {
				context.LoggerFromContext(ctx).WithFields(logrus.Fields{
					"err":    err,
					"job_id": jobID,
				}).Error("failed to refresh claim; cancelling")
				q.cb.Broadcast(jobID)
				return
			}

			if err != nil {
				context.LoggerFromContext(ctx).WithFields(logrus.Fields{
					"err":    err,
					"job_id": jobID,
				}).Error("failed to refresh claim; continuing")
			}
			select {
			case <-ctx.Done():
				return
			case <-time.After(q.pollInterval):
			}
		}
	}, (<-chan struct{})(readyChan)
}

// Name returns the name of this queue type, wow!
func (q *HTTPJobQueue) Name() string {
	return "http"
}

// Cleanup does not do anything!
func (q *HTTPJobQueue) Cleanup() error {
	return nil
}
<|endoftext|>


<commit_before>package formbuilder

func newPostError(err error) *postError {
	return &postError{error: err}
}

type postError struct {
	error
}

func (e *postError) Unwrap() error {
	return e.error
}

func ErrPostFailed(err error) bool {
	_, ok := err.(*postError)
	return ok
}
<commit_msg>update<commit_after>package formbuilder

func newPostError(err error) *postError {
	return &postError{err: err}
}

type postError struct {
	err error
}

func (e *postError) Unwrap() error {
	return e.err
}

func ErrPostFailed(err error) bool {
	_, ok := err.(*postError)
	return ok
}
<|endoftext|>


<commit_before>package daemon

import (
	"errors"
	"fmt"
	"strings"

	"github.com/Sirupsen/logrus"
	dockercontainer "github.com/docker/docker/container"
	"github.com/docker/libnetwork"
)

// ContainerRename changes the name of a container, using the oldName
// to find the container. An error is returned if newName is already
// reserved.
func (daemon *Daemon) ContainerRename(oldName, newName string) error {
	var (
		sid string
		sb  libnetwork.Sandbox
	)

	if oldName == "" || newName == "" {
		return errors.New("Neither old nor new names may be empty")
	}

	if newName[0] != '/' {
		newName = "/" + newName
	}

	container, err := daemon.GetContainer(oldName)
	if err != nil {
		return err
	}

	oldName = container.Name
	oldIsAnonymousEndpoint := container.NetworkSettings.IsAnonymousEndpoint

	if oldName == newName {
		return errors.New("Renaming a container with the same name as its current name")
	}

	container.Lock()
	defer container.Unlock()

	links := map[string]*dockercontainer.Container{}
	for k, v := range daemon.linkIndex.children(container) {
		if !strings.HasPrefix(k, oldName) {
			return fmt.Errorf("Linked container %s does not match parent %s", k, oldName)
		}
		links[strings.TrimPrefix(k, oldName)] = v
	}

	if newName, err = daemon.reserveName(container.ID, newName); err != nil {
		return fmt.Errorf("Error when allocating new name: %v", err)
	}

	for k, v := range links {
		daemon.nameIndex.Reserve(newName+k, v.ID)
		daemon.linkIndex.link(container, v, newName+k)
	}

	container.Name = newName
	container.NetworkSettings.IsAnonymousEndpoint = false

	defer func() {
		if err != nil {
			container.Name = oldName
			container.NetworkSettings.IsAnonymousEndpoint = oldIsAnonymousEndpoint
			daemon.reserveName(container.ID, oldName)
			for k, v := range links {
				daemon.nameIndex.Reserve(oldName+k, v.ID)
				daemon.linkIndex.link(container, v, oldName+k)
				daemon.linkIndex.unlink(newName+k, v, container)
				daemon.nameIndex.Release(newName + k)
			}
			daemon.releaseName(newName)
		}
	}()

	for k, v := range links {
		daemon.linkIndex.unlink(oldName+k, v, container)
		daemon.nameIndex.Release(oldName + k)
	}
	daemon.releaseName(oldName)
	if err = container.ToDisk(); err != nil {
		return err
	}

	attributes := map[string]string{
		"oldName": oldName,
	}

	if !container.Running {
		daemon.LogContainerEventWithAttributes(container, "rename", attributes)
		return nil
	}

	defer func() {
		if err != nil {
			container.Name = oldName
			container.NetworkSettings.IsAnonymousEndpoint = oldIsAnonymousEndpoint
			if e := container.ToDisk(); e != nil {
				logrus.Errorf("%s: Failed in writing to Disk on rename failure: %v", container.ID, e)
			}
		}
	}()

	sid = container.NetworkSettings.SandboxID
	if daemon.netController != nil {
		sb, err = daemon.netController.SandboxByID(sid)
		if err != nil {
			return err
		}

		err = sb.Rename(strings.TrimPrefix(container.Name, "/"))
		if err != nil {
			return err
		}
	}

	daemon.LogContainerEventWithAttributes(container, "rename", attributes)
	return nil
}
<commit_msg>Fix rename error when sid is empty<commit_after>package daemon

import (
	"errors"
	"fmt"
	"strings"

	"github.com/Sirupsen/logrus"
	dockercontainer "github.com/docker/docker/container"
	"github.com/docker/libnetwork"
)

// ContainerRename changes the name of a container, using the oldName
// to find the container. An error is returned if newName is already
// reserved.
func (daemon *Daemon) ContainerRename(oldName, newName string) error {
	var (
		sid string
		sb  libnetwork.Sandbox
	)

	if oldName == "" || newName == "" {
		return errors.New("Neither old nor new names may be empty")
	}

	if newName[0] != '/' {
		newName = "/" + newName
	}

	container, err := daemon.GetContainer(oldName)
	if err != nil {
		return err
	}

	oldName = container.Name
	oldIsAnonymousEndpoint := container.NetworkSettings.IsAnonymousEndpoint

	if oldName == newName {
		return errors.New("Renaming a container with the same name as its current name")
	}

	container.Lock()
	defer container.Unlock()

	links := map[string]*dockercontainer.Container{}
	for k, v := range daemon.linkIndex.children(container) {
		if !strings.HasPrefix(k, oldName) {
			return fmt.Errorf("Linked container %s does not match parent %s", k, oldName)
		}
		links[strings.TrimPrefix(k, oldName)] = v
	}

	if newName, err = daemon.reserveName(container.ID, newName); err != nil {
		return fmt.Errorf("Error when allocating new name: %v", err)
	}

	for k, v := range links {
		daemon.nameIndex.Reserve(newName+k, v.ID)
		daemon.linkIndex.link(container, v, newName+k)
	}

	container.Name = newName
	container.NetworkSettings.IsAnonymousEndpoint = false

	defer func() {
		if err != nil {
			container.Name = oldName
			container.NetworkSettings.IsAnonymousEndpoint = oldIsAnonymousEndpoint
			daemon.reserveName(container.ID, oldName)
			for k, v := range links {
				daemon.nameIndex.Reserve(oldName+k, v.ID)
				daemon.linkIndex.link(container, v, oldName+k)
				daemon.linkIndex.unlink(newName+k, v, container)
				daemon.nameIndex.Release(newName + k)
			}
			daemon.releaseName(newName)
		}
	}()

	for k, v := range links {
		daemon.linkIndex.unlink(oldName+k, v, container)
		daemon.nameIndex.Release(oldName + k)
	}
	daemon.releaseName(oldName)
	if err = container.ToDisk(); err != nil {
		return err
	}

	attributes := map[string]string{
		"oldName": oldName,
	}

	if !container.Running {
		daemon.LogContainerEventWithAttributes(container, "rename", attributes)
		return nil
	}

	defer func() {
		if err != nil {
			container.Name = oldName
			container.NetworkSettings.IsAnonymousEndpoint = oldIsAnonymousEndpoint
			if e := container.ToDisk(); e != nil {
				logrus.Errorf("%s: Failed in writing to Disk on rename failure: %v", container.ID, e)
			}
		}
	}()

	sid = container.NetworkSettings.SandboxID
	if sid != "" && daemon.netController != nil {
		sb, err = daemon.netController.SandboxByID(sid)
		if err != nil {
			return err
		}

		err = sb.Rename(strings.TrimPrefix(container.Name, "/"))
		if err != nil {
			return err
		}
	}

	daemon.LogContainerEventWithAttributes(container, "rename", attributes)
	return nil
}
<|endoftext|>


<commit_before>package whale

import (
	"strings"
	"time"
	"strconv"
)

import (
	"butler/null"
	"gokogiri/xpath"
	"golog"
	"rubex"
	tp "tritium/proto"
	"steno"
	"go-cache"
	"go-cache/arc"
)

type Whale struct {
	Log *golog.Logger
	Debugger steno.Debugger
	RegexpCache       cache.Cache
	XPathCache        cache.Cache
}

type EngineContext struct {
	Functions                []*Function
	Types                    []string
	Exports                  [][]string
	Logs                     []string
	ImportedFiles            []string
	Env                      map[string]string
	MatchStack               []string
	MatchShouldContinueStack []bool
	Yields                   []*YieldBlock
	*Whale
	*tp.Transform
	Rrules []*tp.RewriteRule

	InnerReplacer     *rubex.Regexp
	HeaderContentType *rubex.Regexp

	// Debug info
	Filename          string
	HadError          bool
	Deadline time.Time
	Mobjects []MemoryObject
	MessagePath string
}

const OutputBufferSize = 500 * 1024 //500KB
const defaultMobjects = 4
const TimeoutError = "EngineTimeout"

func NewEngine(logger *golog.Logger, debugger steno.Debugger) *Whale {
	e := &Whale{
		Log: logger,
		Debugger: debugger,
		RegexpCache:              arc.NewARCache(1000),
		XPathCache:               arc.NewARCache(1000),
	}
	e.RegexpCache.SetCleanFunc(CleanRegexpObject)
	e.XPathCache.SetCleanFunc(CleanXpathExpObject)
	return e
}

func NewEngineCtx(eng *Whale, vars map[string]string, transform *tp.Transform, rrules []*tp.RewriteRule, deadline time.Time, messagePath string) (ctx *EngineContext) {
	ctx = &EngineContext{
		Whale:                    eng,
		Exports:                  make([][]string, 0),
		Logs:                     make([]string, 0),
		Env:                      vars,
		Transform:                transform,
		Rrules:                   rrules,
		MatchStack:               make([]string, 0),
		MatchShouldContinueStack: make([]bool, 0),
		Yields:                   make([]*YieldBlock, 0),
		HadError:                 false,

		Deadline:                 deadline,
		Mobjects:                 make([]MemoryObject, 0, defaultMobjects),
		MessagePath:              messagePath,
	}
	return
}

func (eng *Whale) Free() {
	eng.RegexpCache.Reset()
	eng.XPathCache.Reset()
}

func (eng *Whale) Run(transform *tp.Transform, rrules []*tp.RewriteRule, input interface{}, vars map[string]string, deadline time.Time, customer, project, messagePath string) (output string, exports [][]string, logs []string) {
	ctx := NewEngineCtx(eng, vars, transform, rrules, deadline, messagePath)
	defer ctx.Free()
	ctx.Yields = append(ctx.Yields, &YieldBlock{Vars: make(map[string]interface{})})
	ctx.UsePackage(transform.Pkg)
	scope := &Scope{Value: input.(string)}
	obj := transform.Objects[0]
	ctx.Filename = null.GetString(obj.Name)
	ctx.RunInstruction(scope, obj.Root)
	output = scope.Value.(string)
	exports = ctx.Exports
	logs = ctx.Logs
	return
}

func (eng *Whale) GetCacheStats() (int, int, int, int) {
	return eng.RegexpCache.GetHitRate(), eng.RegexpCache.GetUsageRate(), eng.XPathCache.GetHitRate(), eng.XPathCache.GetUsageRate()
}

func (ctx *EngineContext) Free() {
    for _, o := range ctx.Mobjects {
		if o != nil {
			o.Free()
		}
	}
}

func (ctx *EngineContext) RunInstruction(scope *Scope, ins *tp.Instruction) (returnValue interface{}) {
	defer func() {
		//TODO Stack traces seem to get truncated on syslog...
		if x := recover(); x != nil {
			err, ok := x.(error)
			errString := ""
			if ok {
				errString = err.Error()
			} else {
				errString = x.(string)
			}
			if errString != TimeoutError {
				if ctx.HadError == false {
					ctx.HadError = true
					errString = errString + "\n" + ins.Type.String() + " " + null.GetString(ins.Value) + "\n\n\nTritium Stack\n=========\n\n"
				}
				errString = errString + ctx.FileAndLine(ins) + "\n"
			}
			panic(errString)
		}
	}()

	ctx.Whale.Debugger.TrapInstruction(ctx.MessagePath, ctx.Filename, ctx.Env, ins, scope.Value, scope.Index)

	if time.Now().After(ctx.Deadline) {
		panic(TimeoutError)
	}

	// If our object is invalid, then skip it
	if null.GetBool(ins.IsValid) == false {
		panic("Invalid instruction. Should have stopped before linking!")
	}
	indent := ""
	for i := 0; i < len(ctx.Yields); i++ {
		indent += "\t"
	}

	returnValue = ""
	switch *ins.Type {
	case tp.Instruction_BLOCK:
		for _, child := range ins.Children {
			returnValue = ctx.RunInstruction(scope, child)
		}
	case tp.Instruction_TEXT:
		returnValue = null.GetString(ins.Value)
	case tp.Instruction_LOCAL_VAR:
		name := null.GetString(ins.Value)
		vars := ctx.Vars()
		if len(ins.Arguments) > 0 {
			vars[name] = ctx.RunInstruction(scope, ins.Arguments[0])
		}
		if len(ins.Children) > 0 {
			ts := &Scope{Value: ctx.Vars()[name]}
			for _, child := range ins.Children {
				ctx.RunInstruction(ts, child)
			}
			vars[name] = ts.Value
		}
		returnValue = vars[name]
	case tp.Instruction_IMPORT:
		obj := ctx.Objects[int(null.GetInt32(ins.ObjectId))]
		curFile := ctx.Filename
		ctx.Filename = null.GetString(obj.Name)
		start := time.Now()
		index := ctx.AddLog("__IMPORT__FILE__:" + ctx.Filename)
		for _, child := range obj.Root.Children {
			ctx.RunInstruction(scope, child)
		}
		timeSpent := time.Since(start).Seconds()
		ctx.UpdateLog(index, "__IMPORT__FILE__:" + strconv.Itoa(int(timeSpent*1000)) + ":" + ctx.Filename)
		ctx.Filename = curFile
	case tp.Instruction_FUNCTION_CALL:
		fun := ctx.Functions[int(null.GetInt32(ins.FunctionId))]
		curFile := ctx.Filename
		ctx.Filename = fun.GetDescription()
		args := make([]interface{}, len(ins.Arguments))
		for i, argIns := range ins.Arguments {
			args[i] = ctx.RunInstruction(scope, argIns)
		}

		if null.GetBool(fun.BuiltIn) {
			if f := builtInFunctions[fun.Name]; f != nil {
				returnValue = f(ctx, scope, ins, args)
				if returnValue == nil {
					returnValue = ""
				}
			} else {
				panic("missing function: " + fun.Name)
			}
		} else {
			// We are using a user-defined function
			//println("Resetting localvar")
			// Setup the new local var
			vars := make(map[string]interface{}, len(args))
			for i, arg := range fun.Args {
				vars[null.GetString(arg.Name)] = args[i]
			}
			yieldBlock := &YieldBlock{
				Ins:  ins,
				Vars: vars,
			}
			// PUSH!
			ctx.PushYieldBlock(yieldBlock)
			for _, child := range fun.Instruction.Children {
				returnValue = ctx.RunInstruction(scope, child)
			}
			// POP!
			ctx.PopYieldBlock()
		}
		ctx.Filename = curFile
	}

	return
}

func (ctx *EngineContext) ShouldContinue() (result bool) {
	if len(ctx.MatchShouldContinueStack) > 0 {
		result = ctx.MatchShouldContinueStack[len(ctx.MatchShouldContinueStack)-1]
	} else {
		result = false
	}
	return
}

func (ctx *EngineContext) MatchTarget() string {
	return ctx.MatchStack[len(ctx.MatchStack)-1]
}

func (ctx *EngineContext) PushYieldBlock(b *YieldBlock) {
	ctx.Yields = append(ctx.Yields, b)
}

func (ctx *EngineContext) PopYieldBlock() (b *YieldBlock) {
	num := len(ctx.Yields)
	if num > 0 {
		b = ctx.Yields[num-1]
		ctx.Yields = ctx.Yields[:num-1]
	}
	return
}

func (ctx *EngineContext) HasYieldBlock() bool {
	return len(ctx.Yields) > 0
}

func (ctx *EngineContext) TopYieldBlock() (b *YieldBlock) {
	num := len(ctx.Yields)
	if num > 0 {
		b = ctx.Yields[num-1]
	}
	return
}

func (ctx *EngineContext) Vars() map[string]interface{} {
	b := ctx.TopYieldBlock()
	if b != nil {
		return b.Vars
	}
	return nil
}

func (ctx *EngineContext) FileAndLine(ins *tp.Instruction) string {
	lineNum := strconv.Itoa(int(null.GetInt32(ins.LineNumber)))
	return (ctx.Filename + ":" + lineNum)
}

func (ctx *EngineContext) UsePackage(pkg *tp.Package) {
	ctx.Types = make([]string, len(pkg.Types))
	for i, t := range pkg.Types {
		ctx.Types[i] = null.GetString(t.Name)
	}

	ctx.Functions = make([]*Function, len(pkg.Functions))
	for i, f := range pkg.Functions {
		name := null.GetString(f.Name)
		for _, a := range f.Args {
			typeString := ctx.Types[int(null.GetInt32(a.TypeId))]
			name = name + "." + typeString
		}
		fun := &Function{
			Name:     name,
			Function: f,
		}
		ctx.Functions[i] = fun
	}
}

func (ctx *EngineContext) GetRegexp(pattern, options string) (r *rubex.Regexp) {
	sig := pattern + "/" + options
	object, err := ctx.RegexpCache.Get(sig)
	if err != nil {
		mode := rubex.ONIG_OPTION_DEFAULT
		if strings.Index(options, "i") >= 0 {
			mode = rubex.ONIG_OPTION_IGNORECASE
		}
		if strings.Index(options, "m") >= 0 {
			mode = rubex.ONIG_OPTION_MULTILINE
		}
		var err error
		r, err = rubex.NewRegexp(pattern, mode)
		if err == nil {
			//ctx.AddMemoryObject(r)
			ctx.RegexpCache.Set(sig, &RegexpObject{Regexp:r})
		}
		return r
	}
	return object.(*RegexpObject).Regexp
}

func (ctx *EngineContext) GetXpathExpr(p string) (e *xpath.Expression) {
	object, err := ctx.XPathCache.Get(p)
	if err != nil {
		e = xpath.Compile(p)
		if e != nil {
			//ctx.AddMemoryObject(e)
			ctx.XPathCache.Set(p, &XpathExpObject{Expression: e})
		} else {
			ctx.AddLog("Invalid XPath used: " + p)
		}
		return e
	}
	return object.(*XpathExpObject).Expression
}

func (ctx *EngineContext) AddExport(exports []string) {
	ctx.Exports = append(ctx.Exports, exports)
}

func (ctx *EngineContext) AddLog(log string) int {
	//ctx.Log.Info("TRITIUM: " + log)
	index := len(ctx.Logs)
	ctx.Logs = append(ctx.Logs, log)
	return index
}

func (ctx *EngineContext) UpdateLog(index int, log string) {
	//ctx.Log.Info("TRITIUM: " + log)
	if index >= 0 && index < len(ctx.Logs) {
		ctx.Logs[index] = log
	}
}

func (ctx *EngineContext) SetEnv(key, val string) {
	ctx.Env[key] = val
}

func (ctx *EngineContext) GetEnv(key string) (val string) {
	val = ctx.Env[key]
	return
}

func (ctx *EngineContext) SetVar(key string, val interface{}) {
	b := ctx.TopYieldBlock()
	if b != nil {
		b.Vars[key] = val
	}
}

func (ctx *EngineContext) GetVar(key string) (val interface{}) {
	b := ctx.TopYieldBlock()
	if b != nil {
		val = b.Vars[key]
	}
	return
}

func (ctx *EngineContext) GetInnerReplacer() (r *rubex.Regexp) {
	return ctx.GetRegexp(`[\\$](\d)`, "i")
}

func (ctx *EngineContext) GetHeaderContentTypeRegex() (r *rubex.Regexp) {
	return ctx.GetRegexp(`<meta\s+http-equiv="content-type"\s+content="(.*?)"`, "i")
}

func (ctx *EngineContext) GetOutputBuffer() (b []byte) {
	//b = ctx.OutputBuffer
	return
}

func (ctx *EngineContext) Logger() (logger *golog.Logger) {
	logger = ctx.Log
	return
}

func (ctx *EngineContext) PushMatchStack(match string) {
	ctx.MatchStack = append(ctx.MatchStack, match)
}

func (ctx *EngineContext) PopMatchStack() (match string) {
	if num := len(ctx.MatchStack); num > 0 {
		match = ctx.MatchStack[num-1]
		ctx.MatchStack = ctx.MatchStack[:num-1]
	}
	return
}

func (ctx *EngineContext) PushShouldContinueStack(cont bool) {
	ctx.MatchShouldContinueStack = append(ctx.MatchShouldContinueStack, cont)
}

func (ctx *EngineContext) PopShouldContinueStack() (cont bool) {
	if num := len(ctx.MatchShouldContinueStack); num > 0 {
		cont = ctx.MatchShouldContinueStack[num-1]
		ctx.MatchShouldContinueStack = ctx.MatchShouldContinueStack[:num-1]
	}
	return
}

func (ctx *EngineContext) SetShouldContinue(cont bool) {
	if num := len(ctx.MatchShouldContinueStack); num > 0 {
		ctx.MatchShouldContinueStack[num-1] = cont
	}
}

func (ctx *EngineContext) AddMemoryObject(o MemoryObject) {
	ctx.Mobjects = append(ctx.Mobjects, o)
}
<commit_msg>Keeping track of the file in which a function is defined (for stack traces).<commit_after>package whale

import (
	"strings"
	"time"
	"strconv"
)

import (
	"butler/null"
	"gokogiri/xpath"
	"golog"
	"rubex"
	tp "tritium/proto"
	"steno"
	"go-cache"
	"go-cache/arc"
)

type Whale struct {
	Log *golog.Logger
	Debugger steno.Debugger
	RegexpCache       cache.Cache
	XPathCache        cache.Cache
}

type EngineContext struct {
	Functions                []*Function
	Types                    []string
	Exports                  [][]string
	Logs                     []string
	ImportedFiles            []string
	Env                      map[string]string
	MatchStack               []string
	MatchShouldContinueStack []bool
	Yields                   []*YieldBlock
	*Whale
	*tp.Transform
	Rrules []*tp.RewriteRule

	InnerReplacer     *rubex.Regexp
	HeaderContentType *rubex.Regexp

	// Debug info
	Filename          string
	HadError          bool
	Deadline time.Time
	Mobjects []MemoryObject
	MessagePath string
}

const OutputBufferSize = 500 * 1024 //500KB
const defaultMobjects = 4
const TimeoutError = "EngineTimeout"

func NewEngine(logger *golog.Logger, debugger steno.Debugger) *Whale {
	e := &Whale{
		Log: logger,
		Debugger: debugger,
		RegexpCache:              arc.NewARCache(1000),
		XPathCache:               arc.NewARCache(1000),
	}
	e.RegexpCache.SetCleanFunc(CleanRegexpObject)
	e.XPathCache.SetCleanFunc(CleanXpathExpObject)
	return e
}

func NewEngineCtx(eng *Whale, vars map[string]string, transform *tp.Transform, rrules []*tp.RewriteRule, deadline time.Time, messagePath string) (ctx *EngineContext) {
	ctx = &EngineContext{
		Whale:                    eng,
		Exports:                  make([][]string, 0),
		Logs:                     make([]string, 0),
		Env:                      vars,
		Transform:                transform,
		Rrules:                   rrules,
		MatchStack:               make([]string, 0),
		MatchShouldContinueStack: make([]bool, 0),
		Yields:                   make([]*YieldBlock, 0),
		HadError:                 false,

		Deadline:                 deadline,
		Mobjects:                 make([]MemoryObject, 0, defaultMobjects),
		MessagePath:              messagePath,
	}
	return
}

func (eng *Whale) Free() {
	eng.RegexpCache.Reset()
	eng.XPathCache.Reset()
}

func (eng *Whale) Run(transform *tp.Transform, rrules []*tp.RewriteRule, input interface{}, vars map[string]string, deadline time.Time, customer, project, messagePath string) (output string, exports [][]string, logs []string) {
	ctx := NewEngineCtx(eng, vars, transform, rrules, deadline, messagePath)
	defer ctx.Free()
	ctx.Yields = append(ctx.Yields, &YieldBlock{Vars: make(map[string]interface{})})
	ctx.UsePackage(transform.Pkg)
	scope := &Scope{Value: input.(string)}
	obj := transform.Objects[0]
	ctx.Filename = null.GetString(obj.Name)
	ctx.RunInstruction(scope, obj.Root)
	output = scope.Value.(string)
	exports = ctx.Exports
	logs = ctx.Logs
	return
}

func (eng *Whale) GetCacheStats() (int, int, int, int) {
	return eng.RegexpCache.GetHitRate(), eng.RegexpCache.GetUsageRate(), eng.XPathCache.GetHitRate(), eng.XPathCache.GetUsageRate()
}

func (ctx *EngineContext) Free() {
    for _, o := range ctx.Mobjects {
		if o != nil {
			o.Free()
		}
	}
}

func (ctx *EngineContext) RunInstruction(scope *Scope, ins *tp.Instruction) (returnValue interface{}) {
	defer func() {
		//TODO Stack traces seem to get truncated on syslog...
		if x := recover(); x != nil {
			err, ok := x.(error)
			errString := ""
			if ok {
				errString = err.Error()
			} else {
				errString = x.(string)
			}
			if errString != TimeoutError {
				if ctx.HadError == false {
					ctx.HadError = true
					errString = errString + "\n" + ins.Type.String() + " " + null.GetString(ins.Value) + "\n\n\nTritium Stack\n=========\n\n"
				}
				errString = errString + ctx.FileAndLine(ins) + "\n"
			}
			panic(errString)
		}
	}()

	ctx.Whale.Debugger.TrapInstruction(ctx.MessagePath, ctx.Filename, ctx.Env, ins, scope.Value, scope.Index)

	if time.Now().After(ctx.Deadline) {
		panic(TimeoutError)
	}

	// If our object is invalid, then skip it
	if null.GetBool(ins.IsValid) == false {
		panic("Invalid instruction. Should have stopped before linking!")
	}
	indent := ""
	for i := 0; i < len(ctx.Yields); i++ {
		indent += "\t"
	}

	returnValue = ""
	switch *ins.Type {
	case tp.Instruction_BLOCK:
		for _, child := range ins.Children {
			returnValue = ctx.RunInstruction(scope, child)
		}
	case tp.Instruction_TEXT:
		returnValue = null.GetString(ins.Value)
	case tp.Instruction_LOCAL_VAR:
		name := null.GetString(ins.Value)
		vars := ctx.Vars()
		if len(ins.Arguments) > 0 {
			vars[name] = ctx.RunInstruction(scope, ins.Arguments[0])
		}
		if len(ins.Children) > 0 {
			ts := &Scope{Value: ctx.Vars()[name]}
			for _, child := range ins.Children {
				ctx.RunInstruction(ts, child)
			}
			vars[name] = ts.Value
		}
		returnValue = vars[name]
	case tp.Instruction_IMPORT:
		obj := ctx.Objects[int(null.GetInt32(ins.ObjectId))]
		curFile := ctx.Filename
		ctx.Filename = null.GetString(obj.Name)
		start := time.Now()
		index := ctx.AddLog("__IMPORT__FILE__:" + ctx.Filename)
		for _, child := range obj.Root.Children {
			ctx.RunInstruction(scope, child)
		}
		timeSpent := time.Since(start).Seconds()
		ctx.UpdateLog(index, "__IMPORT__FILE__:" + strconv.Itoa(int(timeSpent*1000)) + ":" + ctx.Filename)
		ctx.Filename = curFile
	case tp.Instruction_FUNCTION_CALL:
		fun := ctx.Functions[int(null.GetInt32(ins.FunctionId))]
		args := make([]interface{}, len(ins.Arguments))
		for i, argIns := range ins.Arguments {
			args[i] = ctx.RunInstruction(scope, argIns)
		}

		if null.GetBool(fun.BuiltIn) {
			if f := builtInFunctions[fun.Name]; f != nil {
				returnValue = f(ctx, scope, ins, args)
				if returnValue == nil {
					returnValue = ""
				}
			} else {
				panic("missing function: " + fun.Name)
			}
		} else {
			// We are using a user-defined function
			//println("Resetting localvar")
			// Setup the new local var
			curFile := ctx.Filename
			ctx.Filename = fun.GetDescription()
			vars := make(map[string]interface{}, len(args))
			for i, arg := range fun.Args {
				vars[null.GetString(arg.Name)] = args[i]
			}
			yieldBlock := &YieldBlock{
				Ins:  ins,
				Vars: vars,
			}
			// PUSH!
			ctx.PushYieldBlock(yieldBlock)
			for _, child := range fun.Instruction.Children {
				returnValue = ctx.RunInstruction(scope, child)
			}
			// POP!
			ctx.PopYieldBlock()
			ctx.Filename = curFile
		}
	}

	return
}

func (ctx *EngineContext) ShouldContinue() (result bool) {
	if len(ctx.MatchShouldContinueStack) > 0 {
		result = ctx.MatchShouldContinueStack[len(ctx.MatchShouldContinueStack)-1]
	} else {
		result = false
	}
	return
}

func (ctx *EngineContext) MatchTarget() string {
	return ctx.MatchStack[len(ctx.MatchStack)-1]
}

func (ctx *EngineContext) PushYieldBlock(b *YieldBlock) {
	ctx.Yields = append(ctx.Yields, b)
}

func (ctx *EngineContext) PopYieldBlock() (b *YieldBlock) {
	num := len(ctx.Yields)
	if num > 0 {
		b = ctx.Yields[num-1]
		ctx.Yields = ctx.Yields[:num-1]
	}
	return
}

func (ctx *EngineContext) HasYieldBlock() bool {
	return len(ctx.Yields) > 0
}

func (ctx *EngineContext) TopYieldBlock() (b *YieldBlock) {
	num := len(ctx.Yields)
	if num > 0 {
		b = ctx.Yields[num-1]
	}
	return
}

func (ctx *EngineContext) Vars() map[string]interface{} {
	b := ctx.TopYieldBlock()
	if b != nil {
		return b.Vars
	}
	return nil
}

func (ctx *EngineContext) FileAndLine(ins *tp.Instruction) string {
	lineNum := strconv.Itoa(int(null.GetInt32(ins.LineNumber)))
	return (ctx.Filename + ":" + lineNum)
}

func (ctx *EngineContext) UsePackage(pkg *tp.Package) {
	ctx.Types = make([]string, len(pkg.Types))
	for i, t := range pkg.Types {
		ctx.Types[i] = null.GetString(t.Name)
	}

	ctx.Functions = make([]*Function, len(pkg.Functions))
	for i, f := range pkg.Functions {
		name := null.GetString(f.Name)
		for _, a := range f.Args {
			typeString := ctx.Types[int(null.GetInt32(a.TypeId))]
			name = name + "." + typeString
		}
		fun := &Function{
			Name:     name,
			Function: f,
		}
		ctx.Functions[i] = fun
	}
}

func (ctx *EngineContext) GetRegexp(pattern, options string) (r *rubex.Regexp) {
	sig := pattern + "/" + options
	object, err := ctx.RegexpCache.Get(sig)
	if err != nil {
		mode := rubex.ONIG_OPTION_DEFAULT
		if strings.Index(options, "i") >= 0 {
			mode = rubex.ONIG_OPTION_IGNORECASE
		}
		if strings.Index(options, "m") >= 0 {
			mode = rubex.ONIG_OPTION_MULTILINE
		}
		var err error
		r, err = rubex.NewRegexp(pattern, mode)
		if err == nil {
			//ctx.AddMemoryObject(r)
			ctx.RegexpCache.Set(sig, &RegexpObject{Regexp:r})
		}
		return r
	}
	return object.(*RegexpObject).Regexp
}

func (ctx *EngineContext) GetXpathExpr(p string) (e *xpath.Expression) {
	object, err := ctx.XPathCache.Get(p)
	if err != nil {
		e = xpath.Compile(p)
		if e != nil {
			//ctx.AddMemoryObject(e)
			ctx.XPathCache.Set(p, &XpathExpObject{Expression: e})
		} else {
			ctx.AddLog("Invalid XPath used: " + p)
		}
		return e
	}
	return object.(*XpathExpObject).Expression
}

func (ctx *EngineContext) AddExport(exports []string) {
	ctx.Exports = append(ctx.Exports, exports)
}

func (ctx *EngineContext) AddLog(log string) int {
	//ctx.Log.Info("TRITIUM: " + log)
	index := len(ctx.Logs)
	ctx.Logs = append(ctx.Logs, log)
	return index
}

func (ctx *EngineContext) UpdateLog(index int, log string) {
	//ctx.Log.Info("TRITIUM: " + log)
	if index >= 0 && index < len(ctx.Logs) {
		ctx.Logs[index] = log
	}
}

func (ctx *EngineContext) SetEnv(key, val string) {
	ctx.Env[key] = val
}

func (ctx *EngineContext) GetEnv(key string) (val string) {
	val = ctx.Env[key]
	return
}

func (ctx *EngineContext) SetVar(key string, val interface{}) {
	b := ctx.TopYieldBlock()
	if b != nil {
		b.Vars[key] = val
	}
}

func (ctx *EngineContext) GetVar(key string) (val interface{}) {
	b := ctx.TopYieldBlock()
	if b != nil {
		val = b.Vars[key]
	}
	return
}

func (ctx *EngineContext) GetInnerReplacer() (r *rubex.Regexp) {
	return ctx.GetRegexp(`[\\$](\d)`, "i")
}

func (ctx *EngineContext) GetHeaderContentTypeRegex() (r *rubex.Regexp) {
	return ctx.GetRegexp(`<meta\s+http-equiv="content-type"\s+content="(.*?)"`, "i")
}

func (ctx *EngineContext) GetOutputBuffer() (b []byte) {
	//b = ctx.OutputBuffer
	return
}

func (ctx *EngineContext) Logger() (logger *golog.Logger) {
	logger = ctx.Log
	return
}

func (ctx *EngineContext) PushMatchStack(match string) {
	ctx.MatchStack = append(ctx.MatchStack, match)
}

func (ctx *EngineContext) PopMatchStack() (match string) {
	if num := len(ctx.MatchStack); num > 0 {
		match = ctx.MatchStack[num-1]
		ctx.MatchStack = ctx.MatchStack[:num-1]
	}
	return
}

func (ctx *EngineContext) PushShouldContinueStack(cont bool) {
	ctx.MatchShouldContinueStack = append(ctx.MatchShouldContinueStack, cont)
}

func (ctx *EngineContext) PopShouldContinueStack() (cont bool) {
	if num := len(ctx.MatchShouldContinueStack); num > 0 {
		cont = ctx.MatchShouldContinueStack[num-1]
		ctx.MatchShouldContinueStack = ctx.MatchShouldContinueStack[:num-1]
	}
	return
}

func (ctx *EngineContext) SetShouldContinue(cont bool) {
	if num := len(ctx.MatchShouldContinueStack); num > 0 {
		ctx.MatchShouldContinueStack[num-1] = cont
	}
}

func (ctx *EngineContext) AddMemoryObject(o MemoryObject) {
	ctx.Mobjects = append(ctx.Mobjects, o)
}
<|endoftext|>


<commit_before>package svg

import (
	"log"
	"testing"

	"github.com/stretchr/testify/require"
)

type PathTest struct {
	Description string
	Svg         string
	Kinds       []InstructionType
	XCoords     []float64
	YCoords     []float64
}

var tests = []PathTest{
	{
		"absolute lines",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 L100.000 0.000 100.000 100.000 L0.000 100.000 Z" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, LineInstruction, CloseInstruction, PaintInstruction},
		[]float64{0, 100, 100, 0, 0},
		[]float64{0, 0, 100, 100, 0},
	},
	{
		"relative lines",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 l100.000 0.000 100.000 100.000 l0.000 100.000 Z" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, LineInstruction, CloseInstruction, PaintInstruction},
		[]float64{0, 100, 200, 200, 0},
		[]float64{0, 0, 100, 200, 0},
	},
	{
		"relative h-line test",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 h100.000 50.000" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, PaintInstruction},
		[]float64{0, 100, 150, 0},
		[]float64{0, 0, 0, 0},
	},
	{
		"absolute h-line test",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 H100.000 50.000" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, PaintInstruction},
		[]float64{0, 100, 50, 0},
		[]float64{0, 0, 0, 0},
	},
	{
		"relative v-line test",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 v100.000 50.000" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, PaintInstruction},
		[]float64{0, 0, 0, 0},
		[]float64{0, 100, 150, 0},
	},
	{
		"absolute v-line test",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 V100.000 50.000" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, PaintInstruction},
		[]float64{0, 0, 0, 0},
		[]float64{0, 100, 50, 0},
	},
}

func TestParsePathList(t *testing.T) {
	for _, test := range tests {
		svg, err := ParseSvg(test.Svg, "test", 0)
		require.NoError(t, err)

		dis, errChan := svg.ParseDrawingInstructions()
		for err := range errChan {
			require.NoError(t, err)
		}

		strux := []*DrawingInstruction{}
		for di := range dis {
			strux = append(strux, di)
			log.Printf("di: %+v, di.M: %+v", di, di.M)
		}

		if len(strux) != len(test.Kinds) {
			t.Fatalf("expected %d instructions for test %s, but received %d", len(test.Kinds), test.Description, len(strux))
		}

		for i, kind := range test.Kinds {
			if strux[i].Kind != kind {
				t.Fatalf("expected instruction %d for test %s to be %d, but was %d", i, test.Description, kind, strux[i].Kind)
			}
		}

		for i, x := range test.XCoords {
			if strux[i].M == nil {
				continue
			}

			if strux[i].M[0] != x {
				t.Fatalf("expected X coordinate %d for test %s to be %f, but was %f", i, test.Description, x, strux[i].M[0])
			}
		}

		for i, y := range test.YCoords {
			if strux[i].M == nil {
				continue
			}

			if strux[i].M[1] != y {
				t.Fatalf("expected Y coordinate %d for test %s to be %f, but was %f", i, test.Description, y, strux[i].M[1])
			}
		}
	}
}
<commit_msg>t.Logf instead of log.Printf<commit_after>package svg

import (
	"testing"

	"github.com/stretchr/testify/require"
)

type PathTest struct {
	Description string
	Svg         string
	Kinds       []InstructionType
	XCoords     []float64
	YCoords     []float64
}

var tests = []PathTest{
	{
		"absolute lines",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 L100.000 0.000 100.000 100.000 L0.000 100.000 Z" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, LineInstruction, CloseInstruction, PaintInstruction},
		[]float64{0, 100, 100, 0, 0},
		[]float64{0, 0, 100, 100, 0},
	},
	{
		"relative lines",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 l100.000 0.000 100.000 100.000 l0.000 100.000 Z" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, LineInstruction, CloseInstruction, PaintInstruction},
		[]float64{0, 100, 200, 200, 0},
		[]float64{0, 0, 100, 200, 0},
	},
	{
		"relative h-line test",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 h100.000 50.000" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, PaintInstruction},
		[]float64{0, 100, 150, 0},
		[]float64{0, 0, 0, 0},
	},
	{
		"absolute h-line test",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 H100.000 50.000" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, PaintInstruction},
		[]float64{0, 100, 50, 0},
		[]float64{0, 0, 0, 0},
	},
	{
		"relative v-line test",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 v100.000 50.000" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, PaintInstruction},
		[]float64{0, 0, 0, 0},
		[]float64{0, 100, 150, 0},
	},
	{
		"absolute v-line test",
		`<svg viewBox="0 0 100 100"><path d="M0.000 0.000 V100.000 50.000" fill="#000000" stroke="#000000" stroke-width="2"/></svg>`,
		[]InstructionType{MoveInstruction, LineInstruction, LineInstruction, PaintInstruction},
		[]float64{0, 0, 0, 0},
		[]float64{0, 100, 50, 0},
	},
}

func TestParsePathList(t *testing.T) {
	for _, test := range tests {
		svg, err := ParseSvg(test.Svg, "test", 0)
		require.NoError(t, err)

		dis, errChan := svg.ParseDrawingInstructions()
		for err := range errChan {
			require.NoError(t, err)
		}

		strux := []*DrawingInstruction{}
		for di := range dis {
			strux = append(strux, di)
			t.Logf("di: %+v, di.M: %+v", di, di.M)
		}

		if len(strux) != len(test.Kinds) {
			t.Fatalf("expected %d instructions for test %s, but received %d", len(test.Kinds), test.Description, len(strux))
		}

		for i, kind := range test.Kinds {
			if strux[i].Kind != kind {
				t.Fatalf("expected instruction %d for test %s to be %d, but was %d", i, test.Description, kind, strux[i].Kind)
			}
		}

		for i, x := range test.XCoords {
			if strux[i].M == nil {
				continue
			}

			if strux[i].M[0] != x {
				t.Fatalf("expected X coordinate %d for test %s to be %f, but was %f", i, test.Description, x, strux[i].M[0])
			}
		}

		for i, y := range test.YCoords {
			if strux[i].M == nil {
				continue
			}

			if strux[i].M[1] != y {
				t.Fatalf("expected Y coordinate %d for test %s to be %f, but was %f", i, test.Description, y, strux[i].M[1])
			}
		}
	}
}
<|endoftext|>


<commit_before>package openstack

import (
	"log"

	"github.com/haklop/gophercloud-extensions/network"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/racker/perigee"
	"github.com/rackspace/gophercloud"
)

func resourceFirewallPolicy() *schema.Resource {
	return &schema.Resource{
		Create: resourceFirewallPolicyCreate,
		Read:   resourceFirewallPolicyRead,
		Delete: resourceFirewallPolicyDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"audited": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"shared": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
		},
	}
}

func resourceFirewallPolicyCreate(d *schema.ResourceData, meta interface{}) error {

	p := meta.(*Config)
	networksApi, err := p.getNetworkApi()
	if err != nil {
		return err
	}

	access := p.AccessProvider.(*gophercloud.Access)

	policyConfiguration := network.NewFirewallPolicy{
		Name:        d.Get("name").(string),
		Description: d.Get("description").(string),
		Audited:     d.Get("audited").(bool),
		Shared:      d.Get("shared").(bool),
		TenantId:    access.Token.Tenant.Id,
	}

	log.Printf("[DEBUG] Create firewall policy: %#v", policyConfiguration)

	firewall, err := networksApi.CreateFirewallPolicy(policyConfiguration)
	if err != nil {
		return err
	}

	d.SetId(firewall.Id)

	return nil
}

func resourceFirewallPolicyRead(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] Retrieve information about firewall policy: %s", d.Id())

	p := meta.(*Config)
	networksApi, err := p.getNetworkApi()
	if err != nil {
		return err
	}

	n, err := networksApi.GetFirewallPolicy(d.Id())
	if err != nil {
		httpError, ok := err.(*perigee.UnexpectedResponseCodeError)
		if !ok {
			return err
		}

		if httpError.Actual == 404 {
			d.SetId("")
			return nil
		}

		return err
	}

	d.Set("name", n.Name)

	return nil
}

func resourceFirewallPolicyDelete(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] Destroy firewall policy: %s", d.Id())

	p := meta.(*Config)
	networksApi, err := p.getNetworkApi()
	if err != nil {
		return err
	}

	return networksApi.DeleteFirewallPolicy(d.Id())
}
<commit_msg>Add rules in firewall policy<commit_after>package openstack

import (
	"log"

	"github.com/haklop/gophercloud-extensions/network"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/racker/perigee"
	"github.com/rackspace/gophercloud"
)

func resourceFirewallPolicy() *schema.Resource {
	return &schema.Resource{
		Create: resourceFirewallPolicyCreate,
		Read:   resourceFirewallPolicyRead,
		Delete: resourceFirewallPolicyDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"audited": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"shared": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"rules": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},
		},
	}
}

func resourceFirewallPolicyCreate(d *schema.ResourceData, meta interface{}) error {

	p := meta.(*Config)
	networksApi, err := p.getNetworkApi()
	if err != nil {
		return err
	}

	access := p.AccessProvider.(*gophercloud.Access)

	v := d.Get("rules").(*schema.Set)
	rules := make([]string, v.Len())
	for i, v := range v.List() {
		rules[i] = v.(string)
	}

	policyConfiguration := network.NewFirewallPolicy{
		Name:        d.Get("name").(string),
		Description: d.Get("description").(string),
		Audited:     d.Get("audited").(bool),
		Shared:      d.Get("shared").(bool),
		TenantId:    access.Token.Tenant.Id,
		Rules:       rules,
	}

	log.Printf("[DEBUG] Create firewall policy: %#v", policyConfiguration)

	firewall, err := networksApi.CreateFirewallPolicy(policyConfiguration)
	if err != nil {
		return err
	}

	d.SetId(firewall.Id)

	return nil
}

func resourceFirewallPolicyRead(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] Retrieve information about firewall policy: %s", d.Id())

	p := meta.(*Config)
	networksApi, err := p.getNetworkApi()
	if err != nil {
		return err
	}

	n, err := networksApi.GetFirewallPolicy(d.Id())
	if err != nil {
		httpError, ok := err.(*perigee.UnexpectedResponseCodeError)
		if !ok {
			return err
		}

		if httpError.Actual == 404 {
			d.SetId("")
			return nil
		}

		return err
	}

	d.Set("name", n.Name)

	return nil
}

func resourceFirewallPolicyDelete(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] Destroy firewall policy: %s", d.Id())

	p := meta.(*Config)
	networksApi, err := p.getNetworkApi()
	if err != nil {
		return err
	}

	return networksApi.DeleteFirewallPolicy(d.Id())
}
<|endoftext|>


<commit_before>package cmdlogger

import (
	"flag"
	"github.com/Symantec/Dominator/lib/log/debuglogger"
)

var (
	logDatestamps = flag.Bool("logDatestamps", false,
		"if true, prefix logs with datestamps")
	logDebugLevel = flag.Int("logDebugLevel", -1, "debug log level")
)

// New will create a debuglogger.Logger which writes to the standard error.
// The following command-line flags are registered:
//   -logDatestamps: if true, prefix logs with datestamps
//   -logDebugLevel: debug log level
func New() *debuglogger.Logger {
	return newLogger()
}

// SetDatestampsDefault will change the default for the -logDatestamps command
// line flag. This should be called before flag.Parse().
func SetDatestampsDefault(defaultValue bool) {
	*logDatestamps = defaultValue
}
<commit_msg>Minor documentation fix for lib/log/cmdlogger.<commit_after>package cmdlogger

import (
	"flag"
	"github.com/Symantec/Dominator/lib/log/debuglogger"
)

var (
	logDatestamps = flag.Bool("logDatestamps", false,
		"If true, prefix logs with datestamps")
	logDebugLevel = flag.Int("logDebugLevel", -1, "Debug log level")
)

// New will create a debuglogger.Logger which writes to the standard error.
// The following command-line flags are registered:
//   -logDatestamps: if true, prefix logs with datestamps
//   -logDebugLevel: debug log level
func New() *debuglogger.Logger {
	return newLogger()
}

// SetDatestampsDefault will change the default for the -logDatestamps command
// line flag. This should be called before flag.Parse().
func SetDatestampsDefault(defaultValue bool) {
	*logDatestamps = defaultValue
}
<|endoftext|>