<commit_before>// Package api has type definitions for webdav
package api

import (
	"encoding/xml"
	"regexp"
	"strconv"
	"time"
)

const (
	// Wed, 27 Sep 2017 14:28:34 GMT
	timeFormat = time.RFC1123
)

// Multistatus contains responses returned from an HTTP 207 return code
type Multistatus struct {
	Responses []Response `xml:"response"`
}

// Response contains an Href the response it about and its properties
type Response struct {
	Href  string `xml:"href"`
	Props Prop   `xml:"propstat"`
}

// Prop is the properties of a response
//
// This is a lazy way of decoding the multiple <s:propstat> in the
// response.
//
// The response might look like this
//
// <d:response>
//   <d:href>/remote.php/webdav/Nextcloud%20Manual.pdf</d:href>
//   <d:propstat>
//     <d:prop>
//       <d:getlastmodified>Tue, 19 Dec 2017 22:02:36 GMT</d:getlastmodified>
//       <d:getcontentlength>4143665</d:getcontentlength>
//       <d:resourcetype/>
//       <d:getetag>"048d7be4437ff7deeae94db50ff3e209"</d:getetag>
//       <d:getcontenttype>application/pdf</d:getcontenttype>
//     </d:prop>
//     <d:status>HTTP/1.1 200 OK</d:status>
//   </d:propstat>
//   <d:propstat>
//     <d:prop>
//       <d:quota-used-bytes/>
//       <d:quota-available-bytes/>
//     </d:prop>
//     <d:status>HTTP/1.1 404 Not Found</d:status>
//   </d:propstat>
// </d:response>
//
// So we elide the array of <d:propstat> and within that the array of
// <d:prop> into one struct.
//
// Note that status collects all the status values for which we just
// check the first is OK.
type Prop struct {
	Status   []string  `xml:"DAV: status"`
	Name     string    `xml:"DAV: prop>displayname,omitempty"`
	Type     *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
	Size     int64     `xml:"DAV: prop>getcontentlength,omitempty"`
	Modified Time      `xml:"DAV: prop>getlastmodified,omitempty"`
}

// Parse a status of the form "HTTP/1.1 200 OK",
var parseStatus = regexp.MustCompile(`^HTTP/[0-9.]+\s+(\d+)\s+(.*)$`)

// StatusOK examines the Status and returns an OK flag
func (p *Prop) StatusOK() bool {
	// Assume OK if no statuses received
	if len(p.Status) == 0 {
		return true
	}
	match := parseStatus.FindStringSubmatch(p.Status[0])
	if len(match) < 3 {
		return false
	}
	code, err := strconv.Atoi(match[1])
	if err != nil {
		return false
	}
	if code >= 200 && code < 300 {
		return true
	}
	return false
}

// PropValue is a tagged name and value
type PropValue struct {
	XMLName xml.Name `xml:""`
	Value   string   `xml:",chardata"`
}

// Error is used to desribe webdav errors
//
// <d:error xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns">
//   <s:exception>Sabre\DAV\Exception\NotFound</s:exception>
//   <s:message>File with name Photo could not be located</s:message>
// </d:error>
type Error struct {
	Exception  string `xml:"exception,omitempty"`
	Message    string `xml:"message,omitempty"`
	Status     string
	StatusCode int
}

// Error returns a string for the error and statistifes the error interface
func (e *Error) Error() string {
	if e.Message != "" {
		return e.Message
	}
	if e.Exception != "" {
		return e.Exception
	}
	if e.Status != "" {
		return e.Status
	}
	return "Webdav Error"
}

// Time represents represents date and time information for the
// webdav API marshalling to and from timeFormat
type Time time.Time

// MarshalXML turns a Time into XML
func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	timeString := (*time.Time)(t).Format(timeFormat)
	return e.EncodeElement(timeString, start)
}

// UnmarshalXML turns XML into a Time
func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	var v string
	err := d.DecodeElement(&v, &start)
	if err != nil {
		return err
	}
	newT, err := time.Parse(timeFormat, v)
	if err != nil {
		return err
	}
	*t = Time(newT)
	return nil
}
<commit_msg>webdav: parse time in alternate format for mydrive.ch - fixes #1952<commit_after>// Package api has type definitions for webdav
package api

import (
	"encoding/xml"
	"regexp"
	"strconv"
	"time"
)

const (
	// Wed, 27 Sep 2017 14:28:34 GMT
	timeFormat = time.RFC1123
	// Fri, 05 Jan 2018 14:14:38 +0000 (as used by mydrive.ch)
	timeFormatZ = time.RFC1123Z
)

// Multistatus contains responses returned from an HTTP 207 return code
type Multistatus struct {
	Responses []Response `xml:"response"`
}

// Response contains an Href the response it about and its properties
type Response struct {
	Href  string `xml:"href"`
	Props Prop   `xml:"propstat"`
}

// Prop is the properties of a response
//
// This is a lazy way of decoding the multiple <s:propstat> in the
// response.
//
// The response might look like this
//
// <d:response>
//   <d:href>/remote.php/webdav/Nextcloud%20Manual.pdf</d:href>
//   <d:propstat>
//     <d:prop>
//       <d:getlastmodified>Tue, 19 Dec 2017 22:02:36 GMT</d:getlastmodified>
//       <d:getcontentlength>4143665</d:getcontentlength>
//       <d:resourcetype/>
//       <d:getetag>"048d7be4437ff7deeae94db50ff3e209"</d:getetag>
//       <d:getcontenttype>application/pdf</d:getcontenttype>
//     </d:prop>
//     <d:status>HTTP/1.1 200 OK</d:status>
//   </d:propstat>
//   <d:propstat>
//     <d:prop>
//       <d:quota-used-bytes/>
//       <d:quota-available-bytes/>
//     </d:prop>
//     <d:status>HTTP/1.1 404 Not Found</d:status>
//   </d:propstat>
// </d:response>
//
// So we elide the array of <d:propstat> and within that the array of
// <d:prop> into one struct.
//
// Note that status collects all the status values for which we just
// check the first is OK.
type Prop struct {
	Status   []string  `xml:"DAV: status"`
	Name     string    `xml:"DAV: prop>displayname,omitempty"`
	Type     *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
	Size     int64     `xml:"DAV: prop>getcontentlength,omitempty"`
	Modified Time      `xml:"DAV: prop>getlastmodified,omitempty"`
}

// Parse a status of the form "HTTP/1.1 200 OK",
var parseStatus = regexp.MustCompile(`^HTTP/[0-9.]+\s+(\d+)\s+(.*)$`)

// StatusOK examines the Status and returns an OK flag
func (p *Prop) StatusOK() bool {
	// Assume OK if no statuses received
	if len(p.Status) == 0 {
		return true
	}
	match := parseStatus.FindStringSubmatch(p.Status[0])
	if len(match) < 3 {
		return false
	}
	code, err := strconv.Atoi(match[1])
	if err != nil {
		return false
	}
	if code >= 200 && code < 300 {
		return true
	}
	return false
}

// PropValue is a tagged name and value
type PropValue struct {
	XMLName xml.Name `xml:""`
	Value   string   `xml:",chardata"`
}

// Error is used to desribe webdav errors
//
// <d:error xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns">
//   <s:exception>Sabre\DAV\Exception\NotFound</s:exception>
//   <s:message>File with name Photo could not be located</s:message>
// </d:error>
type Error struct {
	Exception  string `xml:"exception,omitempty"`
	Message    string `xml:"message,omitempty"`
	Status     string
	StatusCode int
}

// Error returns a string for the error and statistifes the error interface
func (e *Error) Error() string {
	if e.Message != "" {
		return e.Message
	}
	if e.Exception != "" {
		return e.Exception
	}
	if e.Status != "" {
		return e.Status
	}
	return "Webdav Error"
}

// Time represents represents date and time information for the
// webdav API marshalling to and from timeFormat
type Time time.Time

// MarshalXML turns a Time into XML
func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	timeString := (*time.Time)(t).Format(timeFormat)
	return e.EncodeElement(timeString, start)
}

// UnmarshalXML turns XML into a Time
func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	var v string
	err := d.DecodeElement(&v, &start)
	if err != nil {
		return err
	}
	newT, err := time.Parse(timeFormat, v)
	if err != nil {
		newT, err = time.Parse(timeFormatZ, v)
		if err != nil {
			return err
		}
	}
	*t = Time(newT)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"context"
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var _ = Describe("K8sBandwidthTest", func() {
	const (
		testDS10       = "run=netperf-10"
		testDS25       = "run=netperf-25"
		testClientPod  = "run=netperf-client-pod"
		testClientHost = "run=netperf-client-host"

		maxRateDeviation = 5
	)

	var (
		kubectl        *helpers.Kubectl
		ciliumFilename string

		backgroundCancel       context.CancelFunc = func() {}
		backgroundError        error
		enableBackgroundReport = true

		podLabels = []string{
			testDS10,
			testDS25,
		}
	)

	BeforeAll(func() {
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
		DeployCiliumAndDNS(kubectl, ciliumFilename)
	})

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.CiliumNamespace,
			"cilium bpf bandwidth list",
			"cilium endpoint list")
	})

	JustBeforeEach(func() {
		if enableBackgroundReport {
			backgroundCancel, backgroundError = kubectl.BackgroundReport("uptime")
			Expect(backgroundError).To(BeNil(), "Cannot start background report process")
		}
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
		backgroundCancel()
	})

	AfterEach(func() {
		ExpectAllPodsTerminated(kubectl)
	})

	AfterAll(func() {
		UninstallCiliumFromManifest(kubectl, ciliumFilename)
		kubectl.CloseSSHClient()
	})

	SkipContextIf(func() bool {
		return !helpers.RunsOnNetNextKernel()
	}, "Checks Bandwidth Rate-Limiting", func() {
		var demoYAML string

		BeforeAll(func() {
			demoYAML = helpers.ManifestGet(kubectl.BasePath(), "demo_bw.yaml")

			res := kubectl.ApplyDefault(demoYAML)
			res.ExpectSuccess("unable to apply %s", demoYAML)

			podLabels := []string{
				testDS10,
				testDS25,
				testClientPod,
				testClientHost,
			}
			for _, label := range podLabels {
				err := kubectl.WaitforPods(helpers.DefaultNamespace,
					fmt.Sprintf("-l %s", label), helpers.HelperTimeout)
				Expect(err).Should(BeNil())
			}
		})

		AfterAll(func() {
			_ = kubectl.Delete(demoYAML)
		})

		testNetperfFromPods := func(clientPodLabel, targetIP string, maxSessions, rate int) {
			pods, err := kubectl.GetPodNames(helpers.DefaultNamespace, clientPodLabel)
			ExpectWithOffset(1, err).Should(BeNil(), "cannot retrieve pod names by filter %q",
				clientPodLabel)
			for i := 1; i <= maxSessions; i++ {
				cmd := helpers.SuperNetperf(i, targetIP, helpers.TCP_MAERTS, "")
				for _, pod := range pods {
					By("Running %d netperf session from %s pod to pod with IP %s (expected rate: %d)",
						i, pod, targetIP, rate)
					res := kubectl.ExecPodCmd(helpers.DefaultNamespace, pod, cmd)
					ExpectWithOffset(1, res).Should(helpers.CMDSuccess(),
						"Request from %s pod to pod with IP %s failed", pod, targetIP)
					By("Session test completed, netperf result raw: %s", res.SingleOut())
					if rate > 0 {
						ExpectWithOffset(1, res.InRange(rate, maxRateDeviation)).To(BeNil(),
							"Rate mismatch")
					}
				}
			}
		}

		testNetperf := func(podLabels []string, fromLabel string) {
			for _, label := range podLabels {
				podIPs, err := kubectl.GetPodsIPs(helpers.DefaultNamespace, label)
				ExpectWithOffset(1, err).Should(BeNil(), "Cannot retrieve pod IPs for %s", label)
				ExpectWithOffset(1, len(podIPs)).To(Equal(int(1)), "Expected pod IPs mismatch")
				rate := 0
				fmt.Sscanf(label, "run=netperf-%d", &rate)
				for _, podIP := range podIPs {
					testNetperfFromPods(fromLabel, podIP, 1, rate)
				}
			}
		}

		It("Checks Pod to Pod bandwidth, vxlan tunneling", func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"global.tunnel": "vxlan",
			})
			testNetperf(podLabels, testClientPod)
			testNetperf(podLabels, testClientHost)
		})
		It("Checks Pod to Pod bandwidth, geneve tunneling", func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"global.tunnel": "geneve",
			})
			testNetperf(podLabels, testClientPod)
			testNetperf(podLabels, testClientHost)
		})
		It("Checks Pod to Pod bandwidth, direct routing", func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"global.tunnel":               "disabled",
				"global.autoDirectNodeRoutes": "true",
			})
			testNetperf(podLabels, testClientPod)
			testNetperf(podLabels, testClientHost)
		})
	})
})
<commit_msg>test: Increase range of accepted values for bandwidth test<commit_after>// Copyright 2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"context"
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var _ = Describe("K8sBandwidthTest", func() {
	const (
		testDS10       = "run=netperf-10"
		testDS25       = "run=netperf-25"
		testClientPod  = "run=netperf-client-pod"
		testClientHost = "run=netperf-client-host"

		maxRateDeviation = 7
	)

	var (
		kubectl        *helpers.Kubectl
		ciliumFilename string

		backgroundCancel       context.CancelFunc = func() {}
		backgroundError        error
		enableBackgroundReport = true

		podLabels = []string{
			testDS10,
			testDS25,
		}
	)

	BeforeAll(func() {
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
		DeployCiliumAndDNS(kubectl, ciliumFilename)
	})

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.CiliumNamespace,
			"cilium bpf bandwidth list",
			"cilium endpoint list")
	})

	JustBeforeEach(func() {
		if enableBackgroundReport {
			backgroundCancel, backgroundError = kubectl.BackgroundReport("uptime")
			Expect(backgroundError).To(BeNil(), "Cannot start background report process")
		}
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
		backgroundCancel()
	})

	AfterEach(func() {
		ExpectAllPodsTerminated(kubectl)
	})

	AfterAll(func() {
		UninstallCiliumFromManifest(kubectl, ciliumFilename)
		kubectl.CloseSSHClient()
	})

	SkipContextIf(func() bool {
		return !helpers.RunsOnNetNextKernel()
	}, "Checks Bandwidth Rate-Limiting", func() {
		var demoYAML string

		BeforeAll(func() {
			demoYAML = helpers.ManifestGet(kubectl.BasePath(), "demo_bw.yaml")

			res := kubectl.ApplyDefault(demoYAML)
			res.ExpectSuccess("unable to apply %s", demoYAML)

			podLabels := []string{
				testDS10,
				testDS25,
				testClientPod,
				testClientHost,
			}
			for _, label := range podLabels {
				err := kubectl.WaitforPods(helpers.DefaultNamespace,
					fmt.Sprintf("-l %s", label), helpers.HelperTimeout)
				Expect(err).Should(BeNil())
			}
		})

		AfterAll(func() {
			_ = kubectl.Delete(demoYAML)
		})

		testNetperfFromPods := func(clientPodLabel, targetIP string, maxSessions, rate int) {
			pods, err := kubectl.GetPodNames(helpers.DefaultNamespace, clientPodLabel)
			ExpectWithOffset(1, err).Should(BeNil(), "cannot retrieve pod names by filter %q",
				clientPodLabel)
			for i := 1; i <= maxSessions; i++ {
				cmd := helpers.SuperNetperf(i, targetIP, helpers.TCP_MAERTS, "")
				for _, pod := range pods {
					By("Running %d netperf session from %s pod to pod with IP %s (expected rate: %d)",
						i, pod, targetIP, rate)
					res := kubectl.ExecPodCmd(helpers.DefaultNamespace, pod, cmd)
					ExpectWithOffset(1, res).Should(helpers.CMDSuccess(),
						"Request from %s pod to pod with IP %s failed", pod, targetIP)
					By("Session test completed, netperf result raw: %s", res.SingleOut())
					if rate > 0 {
						ExpectWithOffset(1, res.InRange(rate, maxRateDeviation)).To(BeNil(),
							"Rate mismatch")
					}
				}
			}
		}

		testNetperf := func(podLabels []string, fromLabel string) {
			for _, label := range podLabels {
				podIPs, err := kubectl.GetPodsIPs(helpers.DefaultNamespace, label)
				ExpectWithOffset(1, err).Should(BeNil(), "Cannot retrieve pod IPs for %s", label)
				ExpectWithOffset(1, len(podIPs)).To(Equal(int(1)), "Expected pod IPs mismatch")
				rate := 0
				fmt.Sscanf(label, "run=netperf-%d", &rate)
				for _, podIP := range podIPs {
					testNetperfFromPods(fromLabel, podIP, 1, rate)
				}
			}
		}

		It("Checks Pod to Pod bandwidth, vxlan tunneling", func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"global.tunnel": "vxlan",
			})
			testNetperf(podLabels, testClientPod)
			testNetperf(podLabels, testClientHost)
		})
		It("Checks Pod to Pod bandwidth, geneve tunneling", func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"global.tunnel": "geneve",
			})
			testNetperf(podLabels, testClientPod)
			testNetperf(podLabels, testClientHost)
		})
		It("Checks Pod to Pod bandwidth, direct routing", func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"global.tunnel":               "disabled",
				"global.autoDirectNodeRoutes": "true",
			})
			testNetperf(podLabels, testClientPod)
			testNetperf(podLabels, testClientHost)
		})
	})
})
<|endoftext|>


<commit_before>package weibo

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"strings"
	"testing"
)

var (
	// mux is the HTTP request multiplexer used with the test server.
	mux *http.ServeMux

	// client is the Weibo client being tested.
	client *Client

	// server is a test HTTP server used to provide mock API responses.
	server *httptest.Server
)

// setup sets up a test HTTP server along with a github.Client that is
// configured to talk to that test server.  Tests should register handlers on
// mux which provide mock responses for the API method being testd.
func setup() {
	// test server
	mux = http.NewServeMux()
	server = httptest.NewServer(mux)

	// weibo client configured to use test server
	client = NewClient("123")
	url, _ := url.Parse(server.URL)
	client.BaseURL = url
}

// teardown closes the test HTTP server.
func teardown() {
	server.Close()
}

func testMethod(t *testing.T, r *http.Request, want string) {
	if want != r.Method {
		t.Errorf("Request method = %v, want %v", r.Method, want)
	}
}

type values map[string]string

func testFormValues(t *testing.T, r *http.Request, values values) {
	want := url.Values{}
	for k, v := range values {
		want.Add(k, v)
	}

	r.ParseForm()
	if !reflect.DeepEqual(want, r.Form) {
		t.Errorf("Request parameters = %v, want %v", r.Form, want)
	}
}

func TestNewClient(t *testing.T) {
	c := NewClient("123")

	if c.BaseURL.String() != defaultBaseURL {
		t.Errorf("NewClient BaseURL = %v, want %v", c.BaseURL.String(), defaultBaseURL)
	}
}

func TestNewRequest(t *testing.T) {
	c := NewClient("123")

	inURL, outURL := "/foo", defaultBaseURL+"2/foo"
	req, _ := c.NewRequest("GET", inURL, nil)

	// test that relative URL was expanded
	if req.URL.String() != outURL {
		t.Errorf("NewRequest(%v) URL = %v, want %v", inURL, req.URL, outURL)
	}
}

func TestNewRequest_invalidJSON(t *testing.T) {
	c := NewClient("123")

	type T struct {
		A map[int]interface{}
	}
	_, err := c.NewRequest("GET", "/", &T{})

	if err == nil {
		t.Error("Expected error to be returned.")
	}
	if err, ok := err.(*json.UnsupportedTypeError); !ok {
		t.Errorf("Expected a JSON error; got %#v", err)
	}
}

func TestDo(t *testing.T) {
	setup()
	defer teardown()

	type foo struct {
		A string
	}

	route := "/" + weiboApiVersion + "/"
	mux.HandleFunc(route, func(w http.ResponseWriter, r *http.Request) {
		if m := "GET"; m != r.Method {
			t.Errorf("Request method = %v, want %v", r.Method, m)
		}
		fmt.Fprint(w, `{"A":"a"}`)
	})

	req, _ := client.NewRequest("GET", "/", nil)
	body := new(foo)
	client.Do(req, body)

	want := &foo{"a"}
	if !reflect.DeepEqual(body, want) {
		t.Errorf("Response body = %v, want %v", body, want)
	}
}

func TestDo_httpError(t *testing.T) {
	setup()
	defer teardown()

	route := "/" + weiboApiVersion + "/"
	mux.HandleFunc(route, func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "BadRequest", 400)
	})

	req, _ := client.NewRequest("GET", "/", nil)
	_, err := client.Do(req, nil)

	if err == nil {
		t.Error("Expected HTTP 400 error.")
	}
}

// Testing handling of an error caused by the internal http client's Do()
// function.  A redirect loop is pretty unlikely to occur within the Weibo
// API, but does allows us to exercise the right code path.
func TestDo_redirectLoop(t *testing.T) {
	setup()
	defer teardown()

	route := "/" + weiboApiVersion + "/"
	mux.HandleFunc(route, func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/", http.StatusFound)
	})

	req, _ := client.NewRequest("GET", "/", nil)
	_, err := client.Do(req, nil)

	if err == nil {
		t.Error("Expected error to be returned.")
	}
	if err, ok := err.(*url.Error); ok {
		t.Errorf("Expected a URL error; got %#v", err)
	}
}

func TestCheckResponse(t *testing.T) {
	res := &http.Response{
		Request:    &http.Request{},
		StatusCode: http.StatusNotFound,
		Body: ioutil.NopCloser(strings.NewReader(
			`{"request": "r", "error_code": 400, "error": "e"}`,
		)),
	}
	err := CheckResponse(res)

	if err == nil {
		t.Error("Expected error response.")
	}

	want := &ErrorResponse{
		Response:   res,
		RequestURL: "r",
		ErrorCode:  400,
		Message:    "e",
	}

	if !reflect.DeepEqual(err, want) {
		t.Errorf("Error = %#v, want %#v", err, want)
	}
}

func TestCheckResponse_noBody(t *testing.T) {
	res := &http.Response{
		Request:    &http.Request{},
		StatusCode: http.StatusNotFound,
		Body:       ioutil.NopCloser(strings.NewReader("")),
	}
	err := CheckResponse(res).(*ErrorResponse)

	if err == nil {
		t.Error("Expected error response.")
	}

	want := &ErrorResponse{
		Response: res,
	}
	if !reflect.DeepEqual(err, want) {
		t.Error("Error = %#v, want %#v", err, want)
	}
}

func TestErrorResponse_Error(t *testing.T) {
	res := &http.Response{Request: &http.Request{}}
	err := ErrorResponse{Message: "m", Response: res}
	if err.Error() == "" {
		t.Error("Expected non-empty ErrorResponse.Error()")
	}
}
<commit_msg>Test url with prefix slash.<commit_after>package weibo

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"strings"
	"testing"
)

var (
	// mux is the HTTP request multiplexer used with the test server.
	mux *http.ServeMux

	// client is the Weibo client being tested.
	client *Client

	// server is a test HTTP server used to provide mock API responses.
	server *httptest.Server
)

// setup sets up a test HTTP server along with a github.Client that is
// configured to talk to that test server.  Tests should register handlers on
// mux which provide mock responses for the API method being testd.
func setup() {
	// test server
	mux = http.NewServeMux()
	server = httptest.NewServer(mux)

	// weibo client configured to use test server
	client = NewClient("123")
	url, _ := url.Parse(server.URL)
	client.BaseURL = url
}

// teardown closes the test HTTP server.
func teardown() {
	server.Close()
}

func testMethod(t *testing.T, r *http.Request, want string) {
	if want != r.Method {
		t.Errorf("Request method = %v, want %v", r.Method, want)
	}
}

type values map[string]string

func testFormValues(t *testing.T, r *http.Request, values values) {
	want := url.Values{}
	for k, v := range values {
		want.Add(k, v)
	}

	r.ParseForm()
	if !reflect.DeepEqual(want, r.Form) {
		t.Errorf("Request parameters = %v, want %v", r.Form, want)
	}
}

func TestNewClient(t *testing.T) {
	c := NewClient("123")

	if c.BaseURL.String() != defaultBaseURL {
		t.Errorf("NewClient BaseURL = %v, want %v", c.BaseURL.String(), defaultBaseURL)
	}
}

func TestNewRequest(t *testing.T) {
	c := NewClient("123")

	inURL, outURL := "foo", defaultBaseURL+"2/foo"
	req, _ := c.NewRequest("GET", inURL, nil)

	// test that relative URL was expanded
	if req.URL.String() != outURL {
		t.Errorf("NewRequest(%v) URL = %v, want %v", inURL, req.URL, outURL)
	}
}

func TestNewRequest_hasSlashPrefix(t *testing.T) {
	c := NewClient("123")

	inURL, outURL := "/foo", defaultBaseURL+"2/foo"
	req, _ := c.NewRequest("GET", inURL, nil)

	// test that relative URL was expanded
	if req.URL.String() != outURL {
		t.Errorf("NewRequest(%v) URL = %v, want %v", inURL, req.URL, outURL)
	}
}

func TestNewRequest_invalidJSON(t *testing.T) {
	c := NewClient("123")

	type T struct {
		A map[int]interface{}
	}
	_, err := c.NewRequest("GET", "/", &T{})

	if err == nil {
		t.Error("Expected error to be returned.")
	}
	if err, ok := err.(*json.UnsupportedTypeError); !ok {
		t.Errorf("Expected a JSON error; got %#v", err)
	}
}

func TestDo(t *testing.T) {
	setup()
	defer teardown()

	type foo struct {
		A string
	}

	route := "/" + weiboApiVersion + "/"
	mux.HandleFunc(route, func(w http.ResponseWriter, r *http.Request) {
		if m := "GET"; m != r.Method {
			t.Errorf("Request method = %v, want %v", r.Method, m)
		}
		fmt.Fprint(w, `{"A":"a"}`)
	})

	req, _ := client.NewRequest("GET", "/", nil)
	body := new(foo)
	client.Do(req, body)

	want := &foo{"a"}
	if !reflect.DeepEqual(body, want) {
		t.Errorf("Response body = %v, want %v", body, want)
	}
}

func TestDo_httpError(t *testing.T) {
	setup()
	defer teardown()

	route := "/" + weiboApiVersion + "/"
	mux.HandleFunc(route, func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "BadRequest", 400)
	})

	req, _ := client.NewRequest("GET", "/", nil)
	_, err := client.Do(req, nil)

	if err == nil {
		t.Error("Expected HTTP 400 error.")
	}
}

// Testing handling of an error caused by the internal http client's Do()
// function.  A redirect loop is pretty unlikely to occur within the Weibo
// API, but does allows us to exercise the right code path.
func TestDo_redirectLoop(t *testing.T) {
	setup()
	defer teardown()

	route := "/" + weiboApiVersion + "/"
	mux.HandleFunc(route, func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/", http.StatusFound)
	})

	req, _ := client.NewRequest("GET", "/", nil)
	_, err := client.Do(req, nil)

	if err == nil {
		t.Error("Expected error to be returned.")
	}
	if err, ok := err.(*url.Error); ok {
		t.Errorf("Expected a URL error; got %#v", err)
	}
}

func TestCheckResponse(t *testing.T) {
	res := &http.Response{
		Request:    &http.Request{},
		StatusCode: http.StatusNotFound,
		Body: ioutil.NopCloser(strings.NewReader(
			`{"request": "r", "error_code": 400, "error": "e"}`,
		)),
	}
	err := CheckResponse(res)

	if err == nil {
		t.Error("Expected error response.")
	}

	want := &ErrorResponse{
		Response:   res,
		RequestURL: "r",
		ErrorCode:  400,
		Message:    "e",
	}

	if !reflect.DeepEqual(err, want) {
		t.Errorf("Error = %#v, want %#v", err, want)
	}
}

func TestCheckResponse_noBody(t *testing.T) {
	res := &http.Response{
		Request:    &http.Request{},
		StatusCode: http.StatusNotFound,
		Body:       ioutil.NopCloser(strings.NewReader("")),
	}
	err := CheckResponse(res).(*ErrorResponse)

	if err == nil {
		t.Error("Expected error response.")
	}

	want := &ErrorResponse{
		Response: res,
	}
	if !reflect.DeepEqual(err, want) {
		t.Error("Error = %#v, want %#v", err, want)
	}
}

func TestErrorResponse_Error(t *testing.T) {
	res := &http.Response{Request: &http.Request{}}
	err := ErrorResponse{Message: "m", Response: res}
	if err.Error() == "" {
		t.Error("Expected non-empty ErrorResponse.Error()")
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repo

import (
	"strings"

	"github.com/codegangsta/martini"

	"github.com/gogits/git"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/base"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/middleware"
)

func Branches(ctx *middleware.Context, params martini.Params) {
	if !ctx.Repo.IsValid {
		return
	}

	brs, err := models.GetBranches(params["username"], params["reponame"])
	if err != nil {
		ctx.Handle(200, "repo.Branches", err)
		return
	} else if len(brs) == 0 {
		ctx.Error(404)
		return
	}

	ctx.Data["Username"] = params["username"]
	ctx.Data["Reponame"] = params["reponame"]

	ctx.Data["Branchname"] = brs[0]
	ctx.Data["Branches"] = brs
	ctx.Data["IsRepoToolbarBranches"] = true

	ctx.HTML(200, "repo/branches", ctx.Data)
}

func Single(ctx *middleware.Context, params martini.Params) {
	if !ctx.Repo.IsValid {
		return
	}

	if params["branchname"] == "" {
		params["branchname"] = "master"
	}

	// Get tree path
	treename := params["_1"]

	// Branches.
	brs, err := models.GetBranches(params["username"], params["reponame"])
	if err != nil {
		log.Error("repo.Single(GetBranches): %v", err)
		ctx.Error(404)
		return
	} else if len(brs) == 0 {
		ctx.Data["IsBareRepo"] = true
		ctx.HTML(200, "repo/single", ctx.Data)
		return
	}

	ctx.Data["Branches"] = brs

	// Directory and file list.
	files, err := models.GetReposFiles(params["username"], params["reponame"],
		params["branchname"], params["commitid"], treename)
	if err != nil {
		log.Error("repo.Single(GetReposFiles): %v", err)
		ctx.Error(404)
		return
	}
	ctx.Data["Username"] = params["username"]
	ctx.Data["Reponame"] = params["reponame"]
	ctx.Data["Branchname"] = params["branchname"]

	var treenames []string
	Paths := make([]string, 0)

	if len(treename) > 0 {
		treenames = strings.Split(treename, "/")
		for i, _ := range treenames {
			Paths = append(Paths, strings.Join(treenames[0:i+1], "/"))
		}

		ctx.Data["HasParentPath"] = true
		if len(Paths)-2 >= 0 {
			ctx.Data["ParentPath"] = "/" + Paths[len(Paths)-2]
		}
	}

	// Get latest commit according username and repo name
	commit, err := models.GetCommit(params["username"], params["reponame"],
		params["branchname"], params["commitid"])
	if err != nil {
		log.Error("repo.Single(GetCommit): %v", err)
		ctx.Error(404)
		return
	}
	ctx.Data["LastCommit"] = commit

	var readmeFile *models.RepoFile

	for _, f := range files {
		if !f.IsFile() || len(f.Name) < 6 {
			continue
		} else if strings.ToLower(f.Name[:6]) == "readme" {
			readmeFile = f
			break
		}
	}

	if readmeFile != nil {
		ctx.Data["ReadmeExist"] = true
		// if file large than 1M not show it
		if readmeFile.Size > 1024*1024 || readmeFile.Filemode != git.FileModeBlob {
			ctx.Data["FileIsLarge"] = true
		} else if blob, err := readmeFile.LookupBlob(); err != nil {
			ctx.Data["ReadmeExist"] = false
		} else {
			// current repo branch link
			urlPrefix := "http://" + base.Domain + "/" + ctx.Repo.Owner.LowerName + "/" +
				ctx.Repo.Repository.Name + "/blob/" + params["branchname"]

			ctx.Data["ReadmeContent"] = string(base.RenderMarkdown(blob.Contents(), urlPrefix))
		}
	}

	ctx.Data["Paths"] = Paths
	ctx.Data["Treenames"] = treenames
	ctx.Data["IsRepoToolbarSource"] = true
	ctx.Data["Files"] = files
	ctx.HTML(200, "repo/single", ctx.Data)
}

func Setting(ctx *middleware.Context, params martini.Params) {
	if !ctx.Repo.IsOwner {
		ctx.Error(404)
		return
	}

	// Branches.
	brs, err := models.GetBranches(params["username"], params["reponame"])
	if err != nil {
		log.Error("repo.Setting(GetBranches): %v", err)
		ctx.Error(404)
		return
	} else if len(brs) == 0 {
		ctx.Data["IsBareRepo"] = true
		ctx.HTML(200, "repo/setting", ctx.Data)
		return
	}

	var title string
	if t, ok := ctx.Data["Title"].(string); ok {
		title = t
	}

	ctx.Data["Title"] = title + " - settings"
	ctx.Data["IsRepoToolbarSetting"] = true
	ctx.HTML(200, "repo/setting", ctx.Data)
}

func Commits(ctx *middleware.Context, params martini.Params) {
	brs, err := models.GetBranches(params["username"], params["reponame"])
	if err != nil {
		ctx.Handle(200, "repo.Commits", err)
		return
	} else if len(brs) == 0 {
		ctx.Error(404)
		return
	}

	ctx.Data["IsRepoToolbarCommits"] = true
	commits, err := models.GetCommits(params["username"],
		params["reponame"], params["branchname"])
	if err != nil {
		ctx.Error(404)
		return
	}
	ctx.Data["Commits"] = commits
	ctx.HTML(200, "repo/commits", ctx.Data)
}

func Issues(ctx *middleware.Context) {
	ctx.Data["IsRepoToolbarIssues"] = true
	ctx.HTML(200, "repo/issues", ctx.Data)
}

func Pulls(ctx *middleware.Context) {
	ctx.Data["IsRepoToolbarPulls"] = true
	ctx.HTML(200, "repo/pulls", ctx.Data)
}
<commit_msg>directory redirect<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repo

import (
	"strings"

	"github.com/codegangsta/martini"

	"github.com/gogits/git"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/base"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/middleware"
)

func Branches(ctx *middleware.Context, params martini.Params) {
	if !ctx.Repo.IsValid {
		return
	}

	brs, err := models.GetBranches(params["username"], params["reponame"])
	if err != nil {
		ctx.Handle(200, "repo.Branches", err)
		return
	} else if len(brs) == 0 {
		ctx.Error(404)
		return
	}

	ctx.Data["Username"] = params["username"]
	ctx.Data["Reponame"] = params["reponame"]

	ctx.Data["Branchname"] = brs[0]
	ctx.Data["Branches"] = brs
	ctx.Data["IsRepoToolbarBranches"] = true

	ctx.HTML(200, "repo/branches", ctx.Data)
}

func Single(ctx *middleware.Context, params martini.Params) {
	if !ctx.Repo.IsValid {
		return
	}

	if params["branchname"] == "" {
		params["branchname"] = "master"
	}

	// Get tree path
	treename := params["_1"]

	if len(treename) > 0 && treename[len(treename)-1] == '/' {
		ctx.Redirect("/"+ctx.Repo.Owner.LowerName+"/"+
			ctx.Repo.Repository.Name+"/tree/"+params["branchname"]+"/"+treename[:len(treename)-1], 302)
		return
	}

	// Branches.
	brs, err := models.GetBranches(params["username"], params["reponame"])
	if err != nil {
		log.Error("repo.Single(GetBranches): %v", err)
		ctx.Error(404)
		return
	} else if len(brs) == 0 {
		ctx.Data["IsBareRepo"] = true
		ctx.HTML(200, "repo/single", ctx.Data)
		return
	}

	ctx.Data["Branches"] = brs

	// Directory and file list.
	files, err := models.GetReposFiles(params["username"], params["reponame"],
		params["branchname"], params["commitid"], treename)
	if err != nil {
		log.Error("repo.Single(GetReposFiles): %v", err)
		ctx.Error(404)
		return
	}
	ctx.Data["Username"] = params["username"]
	ctx.Data["Reponame"] = params["reponame"]
	ctx.Data["Branchname"] = params["branchname"]

	var treenames []string
	Paths := make([]string, 0)

	if len(treename) > 0 {
		treenames = strings.Split(treename, "/")
		for i, _ := range treenames {
			Paths = append(Paths, strings.Join(treenames[0:i+1], "/"))
		}

		ctx.Data["HasParentPath"] = true
		if len(Paths)-2 >= 0 {
			ctx.Data["ParentPath"] = "/" + Paths[len(Paths)-2]
		}
	}

	// Get latest commit according username and repo name
	commit, err := models.GetCommit(params["username"], params["reponame"],
		params["branchname"], params["commitid"])
	if err != nil {
		log.Error("repo.Single(GetCommit): %v", err)
		ctx.Error(404)
		return
	}
	ctx.Data["LastCommit"] = commit

	var readmeFile *models.RepoFile

	for _, f := range files {
		if !f.IsFile() || len(f.Name) < 6 {
			continue
		} else if strings.ToLower(f.Name[:6]) == "readme" {
			readmeFile = f
			break
		}
	}

	if readmeFile != nil {
		ctx.Data["ReadmeExist"] = true
		// if file large than 1M not show it
		if readmeFile.Size > 1024*1024 || readmeFile.Filemode != git.FileModeBlob {
			ctx.Data["FileIsLarge"] = true
		} else if blob, err := readmeFile.LookupBlob(); err != nil {
			ctx.Data["ReadmeExist"] = false
		} else {
			// current repo branch link
			urlPrefix := "http://" + base.Domain + "/" + ctx.Repo.Owner.LowerName + "/" +
				ctx.Repo.Repository.Name + "/tree/" + params["branchname"]

			ctx.Data["ReadmeContent"] = string(base.RenderMarkdown(blob.Contents(), urlPrefix))
		}
	}

	ctx.Data["Paths"] = Paths
	ctx.Data["Treenames"] = treenames
	ctx.Data["IsRepoToolbarSource"] = true
	ctx.Data["Files"] = files
	ctx.HTML(200, "repo/single", ctx.Data)
}

func Setting(ctx *middleware.Context, params martini.Params) {
	if !ctx.Repo.IsOwner {
		ctx.Error(404)
		return
	}

	// Branches.
	brs, err := models.GetBranches(params["username"], params["reponame"])
	if err != nil {
		log.Error("repo.Setting(GetBranches): %v", err)
		ctx.Error(404)
		return
	} else if len(brs) == 0 {
		ctx.Data["IsBareRepo"] = true
		ctx.HTML(200, "repo/setting", ctx.Data)
		return
	}

	var title string
	if t, ok := ctx.Data["Title"].(string); ok {
		title = t
	}

	ctx.Data["Title"] = title + " - settings"
	ctx.Data["IsRepoToolbarSetting"] = true
	ctx.HTML(200, "repo/setting", ctx.Data)
}

func Commits(ctx *middleware.Context, params martini.Params) {
	brs, err := models.GetBranches(params["username"], params["reponame"])
	if err != nil {
		ctx.Handle(200, "repo.Commits", err)
		return
	} else if len(brs) == 0 {
		ctx.Error(404)
		return
	}

	ctx.Data["IsRepoToolbarCommits"] = true
	commits, err := models.GetCommits(params["username"],
		params["reponame"], params["branchname"])
	if err != nil {
		ctx.Error(404)
		return
	}
	ctx.Data["Commits"] = commits
	ctx.HTML(200, "repo/commits", ctx.Data)
}

func Issues(ctx *middleware.Context) {
	ctx.Data["IsRepoToolbarIssues"] = true
	ctx.HTML(200, "repo/issues", ctx.Data)
}

func Pulls(ctx *middleware.Context) {
	ctx.Data["IsRepoToolbarPulls"] = true
	ctx.HTML(200, "repo/pulls", ctx.Data)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"strconv"
	"time"

	"os"

	"runtime"

	termbox "github.com/nsf/termbox-go"
	"github.com/nsf/tulib"
	tuikit "github.com/sgeb/go-tuikit"
	db "github.com/sgeb/go-tuikit/databinding"
)

func main() {
	repaint := make(chan struct{}, 1)
	quit := make(chan struct{}, 1)

	if err := tuikit.Init(); err != nil {
		panic(err)
	}
	defer tuikit.Close()

	fmt.Fprintln(os.Stderr, "-----\nStarting")
	w := newWindow()
	w.SetPaintSubscriber(func() { repaint <- struct{}{} })
	tuikit.SetPainter(w)
	repaint <- struct{}{}

	go func() {
		for _ = range time.Tick(time.Second) {
			fmt.Fprintf(os.Stderr, "Nbr of goroutines: %v\n", runtime.NumGoroutine())
		}
	}()

	for {
		select {
		case ev := <-tuikit.Events:
			if ev.Handled || ev.Type != termbox.EventKey {
				continue
			}
			if ev.Ch == 'q' {
				quit <- struct{}{}
			}
		case <-repaint:
			tuikit.Paint()
		case <-quit:
			return
		}
	}
}

//----------------------------------------------------------------------------
// window
//----------------------------------------------------------------------------

type window struct {
	*tuikit.BaseView
	lastPaintRect tulib.Rect
	views         []*tuikit.TextView
	randomStrings []*randomString
}

func newWindow() *window {
	return &window{
		BaseView: tuikit.NewBaseView(),
	}
}

func (w *window) PaintTo(buffer *tulib.Buffer, rect tulib.Rect) error {
	if w.lastPaintRect.Width != rect.Width ||
		w.lastPaintRect.Height != rect.Height {
		for _, v := range w.views {
			w.DetachChild(v)
		}

		ns := rect.Width * rect.Height
		diff := ns - len(w.views)
		if diff > 0 {
			for i := 0; i < diff; i++ {
				rs := newRandomString()
				tv := tuikit.NewTextView()

				go func() {
					for _ = range rs.Subscribe() {
						tv.SetText(rs.Get())
					}
				}()
				rs.startRandomness()

				w.randomStrings = append(w.randomStrings, rs)
				w.views = append(w.views, tv)
			}
		} else {
			for _, rs := range w.randomStrings[ns:] {
				rs.Dispose()
			}
			w.randomStrings = w.randomStrings[:ns]
			w.views = w.views[:ns]
		}

		for i, v := range w.views {
			dx := int(i % rect.Width)
			dy := int(i / rect.Width)
			w.AttachChild(v, tulib.Rect{dx, dy, 1, 1})
		}

		w.lastPaintRect = rect
	}

	return w.BaseView.PaintTo(buffer, rect)
}

//----------------------------------------------------------------------------
// randomString
//----------------------------------------------------------------------------

type randomString struct {
	db.StringProperty
	stopRandom chan struct{}
}

func newRandomString() *randomString {
	return &randomString{
		db.NewStringProperty(),
		make(chan struct{}),
	}
}

func (rs *randomString) startRandomness() {
	go func() {
		tick := time.Tick(time.Duration(rand.Float64() * 2.0 * 1e9))
		i := uint64(1)

		for {
			select {
			case <-rs.stopRandom:
				return
			case <-tick:
				rs.Set(strconv.Itoa(int(i % 10)))
				i++
			}
		}
	}()
}

func (rs *randomString) Dispose() {
	rs.stopRandom <- struct{}{}
	rs.StringProperty.Dispose()
}
<commit_msg>Add remote profiling<commit_after>package main

import (
	"fmt"
	"math/rand"
	"strconv"
	"time"

	"os"

	"runtime"

	termbox "github.com/nsf/termbox-go"
	"github.com/nsf/tulib"
	tuikit "github.com/sgeb/go-tuikit"
	db "github.com/sgeb/go-tuikit/databinding"

	"net/http"
	_ "net/http/pprof"
)

func main() {
	go func() {
		fmt.Fprintln(os.Stderr, http.ListenAndServe("0.0.0.0:6060", nil))
	}()

	repaint := make(chan struct{}, 1)
	quit := make(chan struct{}, 1)

	if err := tuikit.Init(); err != nil {
		panic(err)
	}
	defer tuikit.Close()

	fmt.Fprintln(os.Stderr, "-----\nStarting")
	w := newWindow()
	w.SetPaintSubscriber(func() { repaint <- struct{}{} })
	tuikit.SetPainter(w)
	repaint <- struct{}{}

	go func() {
		for _ = range time.Tick(time.Second) {
			fmt.Fprintf(os.Stderr, "Nbr of goroutines: %v\n", runtime.NumGoroutine())
		}
	}()

	for {
		select {
		case ev := <-tuikit.Events:
			if ev.Handled || ev.Type != termbox.EventKey {
				continue
			}
			if ev.Ch == 'q' {
				quit <- struct{}{}
			}
		case <-repaint:
			tuikit.Paint()
		case <-quit:
			return
		}
	}
}

//----------------------------------------------------------------------------
// window
//----------------------------------------------------------------------------

type window struct {
	*tuikit.BaseView
	lastPaintRect tulib.Rect
	views         []*tuikit.TextView
	randomStrings []*randomString
}

func newWindow() *window {
	return &window{
		BaseView: tuikit.NewBaseView(),
	}
}

func (w *window) PaintTo(buffer *tulib.Buffer, rect tulib.Rect) error {
	if w.lastPaintRect.Width != rect.Width ||
		w.lastPaintRect.Height != rect.Height {
		for _, v := range w.views {
			w.DetachChild(v)
		}

		ns := rect.Width * rect.Height
		diff := ns - len(w.views)
		if diff > 0 {
			for i := 0; i < diff; i++ {
				rs := newRandomString()
				tv := tuikit.NewTextView()

				go func() {
					for _ = range rs.Subscribe() {
						tv.SetText(rs.Get())
					}
				}()
				rs.startRandomness()

				w.randomStrings = append(w.randomStrings, rs)
				w.views = append(w.views, tv)
			}
		} else {
			for _, rs := range w.randomStrings[ns:] {
				rs.Dispose()
			}
			w.randomStrings = w.randomStrings[:ns]
			w.views = w.views[:ns]
		}

		for i, v := range w.views {
			dx := int(i % rect.Width)
			dy := int(i / rect.Width)
			w.AttachChild(v, tulib.Rect{dx, dy, 1, 1})
		}

		w.lastPaintRect = rect
	}

	return w.BaseView.PaintTo(buffer, rect)
}

//----------------------------------------------------------------------------
// randomString
//----------------------------------------------------------------------------

type randomString struct {
	db.StringProperty
	stopRandom chan struct{}
}

func newRandomString() *randomString {
	return &randomString{
		StringProperty: db.NewStringProperty(),
		stopRandom:     make(chan struct{}, 1),
	}
}

func (rs *randomString) startRandomness() {
	go func() {
		tick := time.Tick(time.Duration(rand.Float64() * 2.0 * 1e9))
		i := uint64(1)

		for {
			select {
			case <-rs.stopRandom:
				return
			case <-tick:
				rs.Set(strconv.Itoa(int(i % 10)))
				i++
			}
		}
	}()
}

func (rs *randomString) Dispose() {
	rs.stopRandom <- struct{}{}
	rs.StringProperty.Dispose()
}
<|endoftext|>


<commit_before>package errorsx

import (
	"net/http"
)

type logger interface {
	Warn(message string, args ...interface{})
	Error(message string, args ...interface{})
}

func HTTPError(w http.ResponseWriter, log logger, err Error, statusCode int) {
	w.WriteHeader(statusCode)
	if statusCode < 500 {
		log.Warn("%s. Stack trace:\n%s", err.Error(), err.Stack())
	} else {
		log.Error("%s. Stack trace:\n%s", err.Error(), err.Stack())
	}

	w.Write([]byte(err.Error()))
}
<commit_msg>json errors<commit_after>package errorsx

import (
	"encoding/json"
	"net/http"
)

type logger interface {
	Warn(message string, args ...interface{})
	Error(message string, args ...interface{})
}

func HTTPError(w http.ResponseWriter, log logger, err Error, statusCode int) {
	w.WriteHeader(statusCode)
	if statusCode < 500 {
		log.Warn("%s. Stack trace:\n%s", err.Error(), err.Stack())
	} else {
		log.Error("%s. Stack trace:\n%s", err.Error(), err.Stack())
	}

	w.Write([]byte(err.Error()))
}

type jsonErrorMessageType struct {
	Message string `json:"message"`
}

func HTTPJSONError(w http.ResponseWriter, log logger, err Error, statusCode int) {
	w.WriteHeader(statusCode)
	if statusCode < 500 {
		log.Warn("%s. Stack trace:\n%s", err.Error(), err.Stack())
	} else {
		log.Error("%s. Stack trace:\n%s", err.Error(), err.Stack())
	}

	json.NewEncoder(w).Encode(jsonErrorMessageType{err.Error()})
}
<|endoftext|>


<commit_before>package hood

type Dialect interface {
	Name() string          // dialect name
	Pk() string            // primary key
	Quote(s string) string // quote string
	MarkerStartPos() int   // index for first marker
	Marker(pos int) string // marker for a prepared statement, e.g. $0 or ?
}
<commit_msg>fixed comment<commit_after>package hood

type Dialect interface {
	Name() string          // dialect name
	Pk() string            // primary key
	Quote(s string) string // quote string
	MarkerStartPos() int   // index for first marker
	Marker(pos int) string // marker for a prepared statement, e.g. $1 or ?
}
<|endoftext|>


<commit_before>package nvd_search

import (
	"io"
	"os"
	"log"
	"fmt"
	"time"
	"path"
	"crypto/sha256"

	"github.com/levigross/grequests"
)

const (
	NVDFeedBaseUrl = "https://static.nvd.nist.gov/feeds/"
	NVDJsonFeedUrl = "json/cve/%.1[1]f/nvdcve-%.1[1]f-"
	NVDFeedVersion = 1.0
)

var NVDUrl string = fmt.Sprintf("%v%v", NVDFeedBaseUrl, fmt.Sprintf(NVDJsonFeedUrl, NVDFeedVersion))

type meta map[string]interface{}
}

func checkFatal(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func checkError(e error) bool {
	if e != nil {
		log.Println(e)
		return false
	}
	return true
}

func downloadFile(uri, filename string) bool {
	log.Println("Downloading license file from", uri)
	response, err := grequests.Get(uri, nil)
	if !checkError(err) {
		log.Println("Couldn't download file, maybe it's not valid URL?")
		return false
	}
	err = response.DownloadToFile(filename)
	checkFatal(err)
	log.Println("Saved content from", uri, "to", filename)
	return true
}

func generateFileList() []string {
	fileList := []string{"modified"}
	for year := 2002; year <= time.Now().Year(); year++ {
		fileList = append(fileList, fmt.Sprintf("%v", year))
	}
	return fileList
}

func getMeta(variety string) {
	url := fmt.Sprintf("%v%v.meta", NVDUrl, variety)
	response, err := grequests.Get(url, nil)
	checkFatal(err)
	fmt.Println(response.String())
}

func getJsonGz(variety, filepath string) {
	filename := fmt.Sprintf("%v.json.gz", variety)
	url := fmt.Sprintf("%v%v", NVDUrl, filename)
	if !downloadFile(url, path.Join(filepath, filename)) {
		log.Fatal("oops")
	}
}

func calculateSHA(r io.Reader) []byte {
	hasher := sha256.New()
	_, err := io.Copy(hasher, r)
	checkFatal(err)
	return hasher.Sum(nil)
}

func loadNVD(dbPath string) {
	os.MkdirAll(dbPath, 0755)
	file, err := os.Open(path.Join(dbPath, "db.json"))
	if ! checkError(err) {
		log.Print("Concatenated database does not exist, creating from scratch")
		Update(dbPath, true)
		os.Exit(2)
	}
	log.Printf("%x", calculateSHA(file))
	Update(dbPath, false)
}

func Update(dbPath string, all bool) {
	os.MkdirAll(dbPath, 0755)
	fileList := []string{"modified"}
	if all {
		fileList = generateFileList()
	}
	for _, f := range fileList {
		getMeta(f)
	}
}

func Search(cve, key, vendor, product, dbPath string) {
	if cve != "" && key != "" {
		log.Fatal("CVE and keyword search are mutually exclusive, please give only either or.")
	} else if ! (cve == "" || key == "" || vendor == "" || product == "") {
		log.Fatal("Give at least one search parameter")
	}
	loadNVD(dbPath)
}
<commit_msg>Fix typo in downloadFile log message<commit_after>package nvd_search

import (
	"io"
	"os"
	"log"
	"fmt"
	"time"
	"path"
	"crypto/sha256"

	"github.com/levigross/grequests"
)

const (
	NVDFeedBaseUrl = "https://static.nvd.nist.gov/feeds/"
	NVDJsonFeedUrl = "json/cve/%.1[1]f/nvdcve-%.1[1]f-"
	NVDFeedVersion = 1.0
)

var NVDUrl string = fmt.Sprintf("%v%v", NVDFeedBaseUrl, fmt.Sprintf(NVDJsonFeedUrl, NVDFeedVersion))

type meta map[string]interface{}
}

func checkFatal(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func checkError(e error) bool {
	if e != nil {
		log.Println(e)
		return false
	}
	return true
}

func downloadFile(uri, filename string) bool {
	log.Println("Downloading file from", uri)
	response, err := grequests.Get(uri, nil)
	if !checkError(err) {
		log.Println("Couldn't download file, maybe it's not valid URL?")
		return false
	}
	err = response.DownloadToFile(filename)
	checkFatal(err)
	log.Println("Saved content from", uri, "to", filename)
	return true
}

func generateFileList() []string {
	fileList := []string{"modified"}
	for year := 2002; year <= time.Now().Year(); year++ {
		fileList = append(fileList, fmt.Sprintf("%v", year))
	}
	return fileList
}

func getMeta(variety string) {
	url := fmt.Sprintf("%v%v.meta", NVDUrl, variety)
	response, err := grequests.Get(url, nil)
	checkFatal(err)
	fmt.Println(response.String())
}

func getJsonGz(variety, filepath string) {
	filename := fmt.Sprintf("%v.json.gz", variety)
	url := fmt.Sprintf("%v%v", NVDUrl, filename)
	if !downloadFile(url, path.Join(filepath, filename)) {
		log.Fatal("oops")
	}
}

func calculateSHA(r io.Reader) []byte {
	hasher := sha256.New()
	_, err := io.Copy(hasher, r)
	checkFatal(err)
	return hasher.Sum(nil)
}

func loadNVD(dbPath string) {
	os.MkdirAll(dbPath, 0755)
	file, err := os.Open(path.Join(dbPath, "db.json"))
	if ! checkError(err) {
		log.Print("Concatenated database does not exist, creating from scratch")
		Update(dbPath, true)
		os.Exit(2)
	}
	log.Printf("%x", calculateSHA(file))
	Update(dbPath, false)
}

func Update(dbPath string, all bool) {
	os.MkdirAll(dbPath, 0755)
	fileList := []string{"modified"}
	if all {
		fileList = generateFileList()
	}
	for _, f := range fileList {
		getMeta(f)
	}
}

func Search(cve, key, vendor, product, dbPath string) {
	if cve != "" && key != "" {
		log.Fatal("CVE and keyword search are mutually exclusive, please give only either or.")
	} else if ! (cve == "" || key == "" || vendor == "" || product == "") {
		log.Fatal("Give at least one search parameter")
	}
	loadNVD(dbPath)
}
<|endoftext|>


<commit_before>package aphgrpc

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/dictyBase/apihelpers/aphcollection"
	"github.com/dictyBase/go-genproto/dictybaseapis/api/jsonapi"
	"google.golang.org/grpc/metadata"
)

var re = regexp.MustCompile(`(\w+)(\=\=|\!\=|\=\@|\!\@)(\w+)(\,|\;)?`)

// JSONAPIParams is a container for various JSON API query parameters
type JSONAPIParams struct {
	// contain include query paramters
	Includes []string
	// contain fields query paramters
	Fields []string
	// check for presence of fields parameters
	HasFields bool
	// check for presence of include parameters
	HasIncludes bool
	// check for presence of filter parameters
	HasFilter bool
	// slice of filters
	Filter []*APIFilter
}

// APIFilter is a container for filter parameters
type APIFilter struct {
	// Attribute of the resource on which the filter will be applied
	Attribute string
	// Type of filter for matching or exclusion
	Operator string
	// The value to match or exclude
	Expression string
	//
	Logic string
}

func hasInclude(r *jsonapi.GetRequest) bool {
	if len(r.Include) > 0 {
		return true
	}
	return false
}

func hasFields(r *jsonapi.GetRequest) bool {
	if len(r.Fields) > 0 {
		return true
	}
	return false
}

func hasListInclude(r *jsonapi.ListRequest) bool {
	if len(r.Include) > 0 {
		return true
	}
	return false
}

func hasListFields(r *jsonapi.ListRequest) bool {
	if len(r.Fields) > 0 {
		return true
	}
	return false
}

func hasFilter(r *jsonapi.ListRequest) bool {
	if len(r.Filter) > 0 {
		return true
	}
	return false
}

// ValidateAndParseListParams validate and parse the JSON API include, fields, filter parameters
func ValidateAndParseListParams(jsapi JSONAPIAllowedParams, r *jsonapi.ListRequest) (*JSONAPIParams, metadata.MD, error) {
	params := &JSONAPIParams{
		HasFields:   false,
		HasIncludes: false,
		HasFilter:   false,
	}
	if hasListInclude(r) {
		if strings.Contains(r.Include, ",") {
			params.Includes = strings.Split(r.Include, ",")
		} else {
			params.Includes = []string{r.Include}
		}
		for _, v := range params.Includes {
			if !aphcollection.Contains(jsapi.AllowedInclude(), v) {
				return params, ErrIncludeParam, fmt.Errorf("include %s relationship is not allowed", v)
			}
		}
		params.HasIncludes = true
	}

	if hasListFields(r) {
		if strings.Contains(r.Fields, ",") {
			params.Fields = strings.Split(r.Fields, ",")
		} else {
			params.Fields = []string{r.Fields}
		}
		for _, v := range params.Fields {
			if !aphcollection.Contains(jsapi.AllowedFields(), v) {
				return params, ErrFields, fmt.Errorf("%s fields attribute is not allowed", v)
			}
		}
		params.HasFields = true
	}
	if HasFilter(r) {
		m := re.FindAllStringSubmatch(r.Filter)
		if len(m) > 0 {
			var filters []*APIFilter
			for _, n := range m {
				if !aphcollection.Contains(jsapi.AllowedFilter(), n[1]) {
					return params, ErrFilterParam, fmt.Errorf("%s filter attribute is not allowed", n[1])
				}
				f := &APIFilter{
					Attribute:  n[1],
					Operator:   n[2],
					Expression: n[3],
					Logic:      n[4],
				}
				filters = append(filters, f)
			}
			params.HasFilter = true
			params.Filter = filters
		}
	}
	return params, metadata.Pairs("errors", "none"), nil
}

// ValidateAndParseGetParams validate and parse the JSON API include and fields parameters
// that are used for singular resources
func ValidateAndParseGetParams(jsapi JSONAPIAllowedParams, r *jsonapi.GetRequest) (*JSONAPIParams, metadata.MD, error) {
	params := &JSONAPIParams{}
	if hasInclude(r) {
		if strings.Contains(r.Include, ",") {
			params.Includes = strings.Split(r.Include, ",")
		} else {
			params.Includes = []string{r.Include}
		}
		for _, v := range params.Includes {
			if !aphcollection.Contains(jsapi.AllowedInclude(), v) {
				return params, ErrIncludeParam, fmt.Errorf("include %s relationship is not allowed", v)
			}
		}
	} else {
		params.HasIncludes = false
	}

	if hasFields(r) {
		if strings.Contains(r.Fields, ",") {
			params.Fields = strings.Split(r.Fields, ",")
		} else {
			params.Fields = []string{r.Fields}
		}
		for _, v := range params.Fields {
			if !aphcollection.Contains(jsapi.AllowedFields(), v) {
				return params, ErrFilterParam, fmt.Errorf("%s value in fields is not allowed", v)
			}
		}
	} else {
		params.HasFields = true
	}
	return params, metadata.Pairs("errors", "none"), nil
}

func HasPagination(r *jsonapi.ListRequest) bool {
	if r.Pagenum != 0 && r.Pagesize != 0 {
		return true
	}
	return false
}
<commit_msg>Added two methods for generating postgresql compatible query clause and expression<commit_after>package aphgrpc

import (
	"bytes"
	"fmt"
	"regexp"
	"strings"

	"github.com/dictyBase/apihelpers/aphcollection"
	"github.com/dictyBase/go-genproto/dictybaseapis/api/jsonapi"
	"google.golang.org/grpc/metadata"
)

var re = regexp.MustCompile(`(\w+)(\=\=|\!\=|\=\@|\!\@)(\w+)(\,|\;)?`)

// JSONAPIParams is a container for various JSON API query parameters
type JSONAPIParams struct {
	// contain include query paramters
	Includes []string
	// contain fields query paramters
	Fields []string
	// check for presence of fields parameters
	HasFields bool
	// check for presence of include parameters
	HasIncludes bool
	// check for presence of filter parameters
	HasFilter bool
	// slice of filters
	Filter []*APIFilter
}

// APIFilter is a container for filter parameters
type APIFilter struct {
	// Attribute of the resource on which the filter will be applied
	Attribute string
	// Type of filter for matching or exclusion
	Operator string
	// The value to match or exclude
	Expression string
	//
	Logic string
}

// FilterToBindValue generates a postgresql compatible query expression from
// the given filters
func FilterToBindValue(filter []*APIFilter) []string {
	values := make([]string, len(filters))
	for i, f := range filters {
		expr := f.Expression
		if strings.Contains(f.Operator, "@") {
			expr = fmt.Sprintf(".*%s.*", expr)
		}
		values[i] = expr
	}
	return values
}

// FilterToWhereClause generates a postgresql compatible where clause from the
// provided filters
func FilterToWhereClause(s JSONAPIParamsInfo, filters []*APIFilter) string {
	lmap := map[string]string{",": "OR", ";": "AND"}
	fmap := s.FilterToColumns()
	omap := getOperatorMap()
	var clause bytes.Buffer
	for i, f := range filters {
		clause.WriteString(
			fmt.Sprintf(
				"%s %s $%d",
				fmap[f.Attribute],
				omap[f.Operator],
				i+1,
			),
		)
		if len(f.Logic) != 0 {
			clause.WriteString(fmt.Sprintf(" %s", lmap[f.Logic]))
		}
	}
	return clause.String()
}

func getOperatorMap() map[string]string {
	return map[string]string{
		"==": "==",
		"!=": "!=",
		"=@": "SIMILAR TO",
		"!@": "NOT SIMILAR TO",
	}
}

func hasInclude(r *jsonapi.GetRequest) bool {
	if len(r.Include) > 0 {
		return true
	}
	return false
}

func hasFields(r *jsonapi.GetRequest) bool {
	if len(r.Fields) > 0 {
		return true
	}
	return false
}

func hasListInclude(r *jsonapi.ListRequest) bool {
	if len(r.Include) > 0 {
		return true
	}
	return false
}

func hasListFields(r *jsonapi.ListRequest) bool {
	if len(r.Fields) > 0 {
		return true
	}
	return false
}

func hasFilter(r *jsonapi.ListRequest) bool {
	if len(r.Filter) > 0 {
		return true
	}
	return false
}

// ValidateAndParseListParams validate and parse the JSON API include, fields, filter parameters
func ValidateAndParseListParams(jsapi JSONAPIParamsInfo, r *jsonapi.ListRequest) (*JSONAPIParams, metadata.MD, error) {
	params := &JSONAPIParams{
		HasFields:   false,
		HasIncludes: false,
		HasFilter:   false,
	}
	if hasListInclude(r) {
		if strings.Contains(r.Include, ",") {
			params.Includes = strings.Split(r.Include, ",")
		} else {
			params.Includes = []string{r.Include}
		}
		for _, v := range params.Includes {
			if !aphcollection.Contains(jsapi.AllowedInclude(), v) {
				return params, ErrIncludeParam, fmt.Errorf("include %s relationship is not allowed", v)
			}
		}
		params.HasIncludes = true
	}

	if hasListFields(r) {
		if strings.Contains(r.Fields, ",") {
			params.Fields = strings.Split(r.Fields, ",")
		} else {
			params.Fields = []string{r.Fields}
		}
		for _, v := range params.Fields {
			if !aphcollection.Contains(jsapi.AllowedFields(), v) {
				return params, ErrFields, fmt.Errorf("%s fields attribute is not allowed", v)
			}
		}
		params.HasFields = true
	}
	if HasFilter(r) {
		m := re.FindAllStringSubmatch(r.Filter)
		if len(m) > 0 {
			var filters []*APIFilter
			for _, n := range m {
				if !aphcollection.Contains(jsapi.AllowedFilter(), n[1]) {
					return params, ErrFilterParam, fmt.Errorf("%s filter attribute is not allowed", n[1])
				}
				f := &APIFilter{
					Attribute:  n[1],
					Operator:   n[2],
					Expression: n[3],
				}
				if len(n) == 5 {
					f.Logic = n[4]
				}
				filters = append(filters, f)
			}
			params.HasFilter = true
			params.Filter = filters
		}
	}
	return params, metadata.Pairs("errors", "none"), nil
}

// ValidateAndParseGetParams validate and parse the JSON API include and fields parameters
// that are used for singular resources
func ValidateAndParseGetParams(jsapi JSONAPIParamsInfo, r *jsonapi.GetRequest) (*JSONAPIParams, metadata.MD, error) {
	params := &JSONAPIParams{}
	if hasInclude(r) {
		if strings.Contains(r.Include, ",") {
			params.Includes = strings.Split(r.Include, ",")
		} else {
			params.Includes = []string{r.Include}
		}
		for _, v := range params.Includes {
			if !aphcollection.Contains(jsapi.AllowedInclude(), v) {
				return params, ErrIncludeParam, fmt.Errorf("include %s relationship is not allowed", v)
			}
		}
	} else {
		params.HasIncludes = false
	}

	if hasFields(r) {
		if strings.Contains(r.Fields, ",") {
			params.Fields = strings.Split(r.Fields, ",")
		} else {
			params.Fields = []string{r.Fields}
		}
		for _, v := range params.Fields {
			if !aphcollection.Contains(jsapi.AllowedFields(), v) {
				return params, ErrFilterParam, fmt.Errorf("%s value in fields is not allowed", v)
			}
		}
	} else {
		params.HasFields = true
	}
	return params, metadata.Pairs("errors", "none"), nil
}

func HasPagination(r *jsonapi.ListRequest) bool {
	if r.Pagenum != 0 && r.Pagesize != 0 {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package worker

import (
	"math"

	schematypes "github.com/taskcluster/go-schematypes"
	tcclient "github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-worker/engines"
	"github.com/taskcluster/taskcluster-worker/plugins"
	"github.com/taskcluster/taskcluster-worker/runtime/monitoring"
	"github.com/taskcluster/taskcluster-worker/runtime/util"
	"github.com/taskcluster/taskcluster-worker/runtime/webhookserver"
)

type options struct {
	ProvisionerID       string `json:"provisionerId"`
	WorkerType          string `json:"workerType"`
	WorkerGroup         string `json:"workerGroup"`
	WorkerID            string `json:"workerId"`
	PollingInterval     int    `json:"pollingInterval"`
	ReclaimOffset       int    `json:"reclaimOffset"`
	MinimumReclaimDelay int    `json:"minimumReclaimDelay"`
	Concurrency         int    `json:"concurrency"`
	EnableSuperseding   bool   `json:"enableSuperseding"`
}

type configType struct {
	Engine           string                 `json:"engine"`
	EngineConfig     map[string]interface{} `json:"engines"`
	Plugins          interface{}            `json:"plugins"`
	WebHookServer    interface{}            `json:"webHookServer"`
	TemporaryFolder  string                 `json:"temporaryFolder"`
	MinimumDiskSpace int64                  `json:"minimumDiskSpace"`
	MinimumMemory    int64                  `json:"minimumMemory"`
	Monitor          interface{}            `json:"monitor"`
	Credentials      tcclient.Credentials   `json:"credentials"`
	QueueBaseURL     string                 `json:"queueBaseUrl"`
	AuthBaseURL      string                 `json:"authBaseUrl"`
	WorkerOptions    options                `json:"worker"`
}

// optionsSchema must be satisfied by Options used to construct a Worker
var optionsSchema schematypes.Schema = schematypes.Object{
	Title:       "Worker Config",
	Description: "Configuration for the worker",
	Properties: schematypes.Properties{
		"provisionerId": schematypes.String{
			Title: "ProvisionerId",
			Description: util.Markdown(`
				ProvisionerId for workerType that tasks should be claimed
				from. Note, a 'workerType' is only unique given the 'provisionerId'.
			`),
			Pattern: `^[a-zA-Z0-9_-]{1,22}$`,
		},
		"workerType": schematypes.String{
			Title: "WorkerType",
			Description: util.Markdown(`
				WorkerType to claim tasks for, combined with 'provisionerId' this
				identifies the pool of workers the machine belongs to.
			`),
			Pattern: `^[a-zA-Z0-9_-]{1,22}$`,
		},
		"workerGroup": schematypes.String{
			Title: "WorkerGroup",
			Description: util.Markdown(`
				Group of workers this machine belongs to. This is any identifier such
				that workerGroup and workerId uniquely identifies this machine.
			`),
			Pattern: `^[a-zA-Z0-9_-]{1,22}$`,
		},
		"workerId": schematypes.String{
			Title: "WorkerId",
			Description: util.Markdown(`
				Identifier for this machine. This is any identifier such
				that workerGroup and workerId uniquely identifies this machine.
			`),
			Pattern: `^[a-zA-Z0-9_-]{1,22}$`,
		},
		"pollingInterval": schematypes.Integer{
			Title: "Task Polling Interval",
			Description: util.Markdown(`
				The amount of time to wait between task polling
				iterations in seconds.
			`),
			Minimum: 0,
			Maximum: 10 * 60,
		},
		"reclaimOffset": schematypes.Integer{
			Title: "Reclaim Offset",
			Description: util.Markdown(`
				The number of seconds prior to task claim expiration the
				claim should be reclamed.
			`),
			Minimum: 0,
			Maximum: 10 * 60,
		},
		"minimumReclaimDelay": schematypes.Integer{
			Title: "Minimum Reclaim Delay",
			Description: util.Markdown(`
				Minimum number of seconds to wait before reclaiming a task.
				it is important that this is some reasonable non-zero minimum to avoid
				overloading servers if there is some error.
			`),
			Minimum: 0,
			Maximum: 10 * 60,
		},
		"concurrency": schematypes.Integer{
			Title:       "Concurrency",
			Description: "The number of tasks that this worker supports running in parallel.",
			Minimum:     1,
			Maximum:     1000,
		},
		"enableSuperseding": schematypes.Boolean{
			Title: "Enable Superseding",
			Description: util.Markdown(`
				If superseding is enabled, tasks can specify a URL that returns a list
				of taskIds that supersedes the given task.

				For details see [superseding documentation](https://docs.taskcluster.net` +
				`/reference/platform/taskcluster-queue/docs/superseding).
			`),
		},
	},
	Required: []string{
		"provisionerId",
		"workerType",
		"workerGroup",
		"workerId",
		"pollingInterval",
		"reclaimOffset",
		"minimumReclaimDelay",
		"concurrency",
	},
}

var credentialsSchema schematypes.Schema = schematypes.Object{
	Title: "TaskCluster Credentials",
	Description: util.Markdown(`
		The set of credentials that should be used by the worker
		when authenticating against taskcluster endpoints. This needs scopes
		for claiming tasks for the given workerType.
	`),
	Properties: schematypes.Properties{
		"clientId": schematypes.String{
			Title:       "ClientId",
			Description: `ClientId for credentials`,
			Pattern:     `^[A-Za-z0-9@/:._|-]+$`,
		},
		"accessToken": schematypes.String{
			Title:       "AccessToken",
			Description: `The security-sensitive access token for the client.`,
			Pattern:     `^[a-zA-Z0-9_-]{22,66}$`,
		},
		"certificate": schematypes.String{
			Title: "Certificate",
			Description: util.Markdown(`
				The certificate for the client, if using temporary credentials.
			`),
		},
		"authorizedScopes": schematypes.Array{
			Items: schematypes.String{},
		},
	},
	Required: []string{"clientId", "accessToken"},
}

// ConfigSchema returns the schema for configuration.
func ConfigSchema() schematypes.Object {
	engineConfig := schematypes.Properties{}
	engineNames := []string{}
	for name, provider := range engines.Engines() {
		engineNames = append(engineNames, name)
		engineConfig[name] = provider.ConfigSchema()
	}
	return schematypes.Object{
		Properties: schematypes.Properties{
			"engine": schematypes.StringEnum{
				Title: "Worker Engine",
				Description: util.Markdown(`
					Selected worker engine to use, notice that the
					configuration for this engine **must** be present under the
					'engines.<engine>' configuration key.
				`),
				Options: engineNames,
			},
			"engines": schematypes.Object{
				Title: "Engine Configuration",
				Description: util.Markdown(`
					Mapping from engine name to engine configuration.
					Even-though the worker will only use one engine at any given time,
					the configuration file can hold configuration for all engines.
					Hence, you need only update the 'engine' key to change which engine
					should be used.
				`),
				Properties: engineConfig,
			},
			"plugins":       plugins.PluginManagerConfigSchema(),
			"webHookServer": webhookserver.ConfigSchema,
			"temporaryFolder": schematypes.String{
				Title: "Temporary Folder",
				Description: util.Markdown(`
					Path to folder that can be used for temporary files and
					folders, if folder doesn't exist it will be created, otherwise it
					will be overwritten.
				`),
			},
			"minimumDiskSpace": schematypes.Integer{
				Title: "Minimum Disk Space",
				Description: util.Markdown(`
					The minimum amount of disk space in bytes to have available
					before starting on the next task. Garbage collector will do a
					best-effort attempt at releasing resources to satisfy this limit.
				`),
				Minimum: 0,
				Maximum: math.MaxInt64,
			},
			"minimumMemory": schematypes.Integer{
				Title: "Minimum Memory",
				Description: util.Markdown(`
					The minimum amount of memory in bytes to have available
					before starting on the next task. Garbage collector will do a
					best-effort attempt at releasing resources to satisfy this limit.
				`),
				Minimum: 0,
				Maximum: math.MaxInt64,
			},
			"monitor":      monitoring.ConfigSchema,
			"credentials":  credentialsSchema,
			"queueBaseUrl": schematypes.String{},
			"authBaseUrl":  schematypes.String{},
			"worker":       optionsSchema,
		},
		Required: []string{
			"engine",
			"engines",
			"plugins",
			"temporaryFolder",
			"minimumDiskSpace",
			"minimumMemory",
			"monitor",
			"credentials",
			"worker",
		},
	}
}
<commit_msg>Updated clientId pattern from tc-auth<commit_after>package worker

import (
	"math"

	schematypes "github.com/taskcluster/go-schematypes"
	tcclient "github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-worker/engines"
	"github.com/taskcluster/taskcluster-worker/plugins"
	"github.com/taskcluster/taskcluster-worker/runtime/monitoring"
	"github.com/taskcluster/taskcluster-worker/runtime/util"
	"github.com/taskcluster/taskcluster-worker/runtime/webhookserver"
)

type options struct {
	ProvisionerID       string `json:"provisionerId"`
	WorkerType          string `json:"workerType"`
	WorkerGroup         string `json:"workerGroup"`
	WorkerID            string `json:"workerId"`
	PollingInterval     int    `json:"pollingInterval"`
	ReclaimOffset       int    `json:"reclaimOffset"`
	MinimumReclaimDelay int    `json:"minimumReclaimDelay"`
	Concurrency         int    `json:"concurrency"`
	EnableSuperseding   bool   `json:"enableSuperseding"`
}

type configType struct {
	Engine           string                 `json:"engine"`
	EngineConfig     map[string]interface{} `json:"engines"`
	Plugins          interface{}            `json:"plugins"`
	WebHookServer    interface{}            `json:"webHookServer"`
	TemporaryFolder  string                 `json:"temporaryFolder"`
	MinimumDiskSpace int64                  `json:"minimumDiskSpace"`
	MinimumMemory    int64                  `json:"minimumMemory"`
	Monitor          interface{}            `json:"monitor"`
	Credentials      tcclient.Credentials   `json:"credentials"`
	QueueBaseURL     string                 `json:"queueBaseUrl"`
	AuthBaseURL      string                 `json:"authBaseUrl"`
	WorkerOptions    options                `json:"worker"`
}

// optionsSchema must be satisfied by Options used to construct a Worker
var optionsSchema schematypes.Schema = schematypes.Object{
	Title:       "Worker Config",
	Description: "Configuration for the worker",
	Properties: schematypes.Properties{
		"provisionerId": schematypes.String{
			Title: "ProvisionerId",
			Description: util.Markdown(`
				ProvisionerId for workerType that tasks should be claimed
				from. Note, a 'workerType' is only unique given the 'provisionerId'.
			`),
			Pattern: `^[a-zA-Z0-9_-]{1,22}$`,
		},
		"workerType": schematypes.String{
			Title: "WorkerType",
			Description: util.Markdown(`
				WorkerType to claim tasks for, combined with 'provisionerId' this
				identifies the pool of workers the machine belongs to.
			`),
			Pattern: `^[a-zA-Z0-9_-]{1,22}$`,
		},
		"workerGroup": schematypes.String{
			Title: "WorkerGroup",
			Description: util.Markdown(`
				Group of workers this machine belongs to. This is any identifier such
				that workerGroup and workerId uniquely identifies this machine.
			`),
			Pattern: `^[a-zA-Z0-9_-]{1,22}$`,
		},
		"workerId": schematypes.String{
			Title: "WorkerId",
			Description: util.Markdown(`
				Identifier for this machine. This is any identifier such
				that workerGroup and workerId uniquely identifies this machine.
			`),
			Pattern: `^[a-zA-Z0-9_-]{1,22}$`,
		},
		"pollingInterval": schematypes.Integer{
			Title: "Task Polling Interval",
			Description: util.Markdown(`
				The amount of time to wait between task polling
				iterations in seconds.
			`),
			Minimum: 0,
			Maximum: 10 * 60,
		},
		"reclaimOffset": schematypes.Integer{
			Title: "Reclaim Offset",
			Description: util.Markdown(`
				The number of seconds prior to task claim expiration the
				claim should be reclamed.
			`),
			Minimum: 0,
			Maximum: 10 * 60,
		},
		"minimumReclaimDelay": schematypes.Integer{
			Title: "Minimum Reclaim Delay",
			Description: util.Markdown(`
				Minimum number of seconds to wait before reclaiming a task.
				it is important that this is some reasonable non-zero minimum to avoid
				overloading servers if there is some error.
			`),
			Minimum: 0,
			Maximum: 10 * 60,
		},
		"concurrency": schematypes.Integer{
			Title:       "Concurrency",
			Description: "The number of tasks that this worker supports running in parallel.",
			Minimum:     1,
			Maximum:     1000,
		},
		"enableSuperseding": schematypes.Boolean{
			Title: "Enable Superseding",
			Description: util.Markdown(`
				If superseding is enabled, tasks can specify a URL that returns a list
				of taskIds that supersedes the given task.

				For details see [superseding documentation](https://docs.taskcluster.net` +
				`/reference/platform/taskcluster-queue/docs/superseding).
			`),
		},
	},
	Required: []string{
		"provisionerId",
		"workerType",
		"workerGroup",
		"workerId",
		"pollingInterval",
		"reclaimOffset",
		"minimumReclaimDelay",
		"concurrency",
	},
}

var credentialsSchema schematypes.Schema = schematypes.Object{
	Title: "TaskCluster Credentials",
	Description: util.Markdown(`
		The set of credentials that should be used by the worker
		when authenticating against taskcluster endpoints. This needs scopes
		for claiming tasks for the given workerType.
	`),
	Properties: schematypes.Properties{
		"clientId": schematypes.String{
			Title:       "ClientId",
			Description: `ClientId for credentials`,
			Pattern:     `^[A-Za-z0-9!@/:.+|_-]+$`,
		},
		"accessToken": schematypes.String{
			Title:       "AccessToken",
			Description: `The security-sensitive access token for the client.`,
			Pattern:     `^[a-zA-Z0-9_-]{22,66}$`,
		},
		"certificate": schematypes.String{
			Title: "Certificate",
			Description: util.Markdown(`
				The certificate for the client, if using temporary credentials.
			`),
		},
		"authorizedScopes": schematypes.Array{
			Items: schematypes.String{},
		},
	},
	Required: []string{"clientId", "accessToken"},
}

// ConfigSchema returns the schema for configuration.
func ConfigSchema() schematypes.Object {
	engineConfig := schematypes.Properties{}
	engineNames := []string{}
	for name, provider := range engines.Engines() {
		engineNames = append(engineNames, name)
		engineConfig[name] = provider.ConfigSchema()
	}
	return schematypes.Object{
		Properties: schematypes.Properties{
			"engine": schematypes.StringEnum{
				Title: "Worker Engine",
				Description: util.Markdown(`
					Selected worker engine to use, notice that the
					configuration for this engine **must** be present under the
					'engines.<engine>' configuration key.
				`),
				Options: engineNames,
			},
			"engines": schematypes.Object{
				Title: "Engine Configuration",
				Description: util.Markdown(`
					Mapping from engine name to engine configuration.
					Even-though the worker will only use one engine at any given time,
					the configuration file can hold configuration for all engines.
					Hence, you need only update the 'engine' key to change which engine
					should be used.
				`),
				Properties: engineConfig,
			},
			"plugins":       plugins.PluginManagerConfigSchema(),
			"webHookServer": webhookserver.ConfigSchema,
			"temporaryFolder": schematypes.String{
				Title: "Temporary Folder",
				Description: util.Markdown(`
					Path to folder that can be used for temporary files and
					folders, if folder doesn't exist it will be created, otherwise it
					will be overwritten.
				`),
			},
			"minimumDiskSpace": schematypes.Integer{
				Title: "Minimum Disk Space",
				Description: util.Markdown(`
					The minimum amount of disk space in bytes to have available
					before starting on the next task. Garbage collector will do a
					best-effort attempt at releasing resources to satisfy this limit.
				`),
				Minimum: 0,
				Maximum: math.MaxInt64,
			},
			"minimumMemory": schematypes.Integer{
				Title: "Minimum Memory",
				Description: util.Markdown(`
					The minimum amount of memory in bytes to have available
					before starting on the next task. Garbage collector will do a
					best-effort attempt at releasing resources to satisfy this limit.
				`),
				Minimum: 0,
				Maximum: math.MaxInt64,
			},
			"monitor":      monitoring.ConfigSchema,
			"credentials":  credentialsSchema,
			"queueBaseUrl": schematypes.String{},
			"authBaseUrl":  schematypes.String{},
			"worker":       optionsSchema,
		},
		Required: []string{
			"engine",
			"engines",
			"plugins",
			"temporaryFolder",
			"minimumDiskSpace",
			"minimumMemory",
			"monitor",
			"credentials",
			"worker",
		},
	}
}
<|endoftext|>


<commit_before>/*
* @Author: souravray
* @Date:   2014-11-02 22:19:25
* @Last Modified by:   souravray
* @Last Modified time: 2015-02-07 23:34:47
 */

package worker

import (
	"net/url"
	"time"
)

type Interface interface {
	Perform(payload url.Values) error
	SetTimeout(duration string)
	SetRetryLimit(limit int32)
	GetRetryLimit() int32
	SetAgeLimit(duration string)
	GetAgeLimit() time.Duration
	SetMinBackoff(duration string)
	SetMaxBackoff(duration string)
	SetMaxDoubling(attempts int32)
	GetInterval(retryAttempts int32) time.Duration
}

type Config struct {
	// Maximum time allocated to a worker to complete a job
	// If Timeout is not set expilicitly, then Timeout will same as DefaultWorkerTimeout.
	Timeout time.Duration

	// Number of tries after which the task fails permanently and is deleted.
	// If AgeLimit is also set, both limits must be exceeded for the task to fail permanently.
	RetryLimit int32

	// Maximum time allowed since the task's first try before the task fails permanently and is deleted
	// If RetryLimit is also set, both limits must be exceeded for the task to fail permanently.
	AgeLimit time.Duration

	// Minimum time between successive tries
	MinBackoff time.Duration

	// Maximum time between successive tries
	MaxBackoff time.Duration

	// Maximum number of times to double the interval between successive tries before the intervals increase linearly
	MaxDoubling int32
}

func (wc Config) SetTimeout(duration string) {
	timeduration, err := time.ParseDuration(duration)
	if err == nil {
		wc.AgeLimit = timeduration
	}
}

func (wc Config) SetRetryLimit(limit int32) {
	if limit > 0 {
		wc.RetryLimit = limit
	} else {
		wc.RetryLimit = 0
	}
}

func (wc Config) GetRetryLimit() int32 {
	return wc.RetryLimit
}

func (wc Config) SetAgeLimit(duration string) {
	timeduration, err := time.ParseDuration(duration)
	if err == nil {
		wc.AgeLimit = timeduration
	}
}

func (wc Config) GetAgeLimit() time.Duration {
	return wc.AgeLimit
}

func (wc Config) SetMinBackoff(duration string) {
	timeduration, err := time.ParseDuration(duration)
	if err == nil {
		wc.MinBackoff = timeduration
	}
}

func (wc Config) SetMaxBackoff(duration string) {
	timeduration, err := time.ParseDuration(duration)
	if err == nil {
		wc.MaxBackoff = timeduration
	}
}

func (wc Config) SetMaxDoubling(attempts int32) {
	wc.MaxDoubling = attempts
}

func (wc Config) GetInterval(retryAttempts int32) time.Duration {
	var interval time.Duration
	if wc.MaxDoubling <= 0 {
		interval = wc.MinBackoff
	}

	if wc.MaxDoubling+1 > retryAttempts {
		interval = time.Duration(retryAttempts) * wc.MinBackoff
	} else {
		interval = time.Duration(wc.MaxDoubling+1) * wc.MinBackoff
	}
	return wc.intervalPassFilter(interval)
}

func (wc Config) intervalPassFilter(interval time.Duration) time.Duration {

	if wc.MaxBackoff == time.Duration(0) ||
		wc.MinBackoff > wc.MaxBackoff {
		return interval
	} else if wc.MaxBackoff > interval {
		return interval
	} else {
		return wc.MaxBackoff
	}

	return time.Duration(0)
}
<commit_msg>fix bug in SetTimeout and change non-pointer receiver of config methods to pointer receiver<commit_after>/*
* @Author: souravray
* @Date:   2014-11-02 22:19:25
* @Last Modified by:   souravray
* @Last Modified time: 2015-02-10 01:17:01
 */

package worker

import (
	"net/url"
	"time"
)

type Interface interface {
	Perform(payload url.Values) error
	SetTimeout(duration string)
	SetRetryLimit(limit int32)
	GetRetryLimit() int32
	SetAgeLimit(duration string)
	GetAgeLimit() time.Duration
	SetMinBackoff(duration string)
	SetMaxBackoff(duration string)
	SetMaxDoubling(attempts int32)
	GetInterval(retryAttempts int32) time.Duration
}

type Config struct {
	// Maximum time allocated to a worker to complete a job
	// If Timeout is not set expilicitly, then Timeout will same as DefaultWorkerTimeout.
	Timeout time.Duration

	// Number of tries after which the task fails permanently and is deleted.
	// If AgeLimit is also set, both limits must be exceeded for the task to fail permanently.
	RetryLimit int32

	// Maximum time allowed since the task's first try before the task fails permanently and is deleted
	// If RetryLimit is also set, both limits must be exceeded for the task to fail permanently.
	AgeLimit time.Duration

	// Minimum time between successive tries
	MinBackoff time.Duration

	// Maximum time between successive tries
	MaxBackoff time.Duration

	// Maximum number of times to double the interval between successive tries before the intervals increase linearly
	MaxDoubling int32
}

func (wc *Config) SetTimeout(duration string) {
	timeduration, err := time.ParseDuration(duration)
	if err == nil {
		wc.Timeout = timeduration
	}
}

func (wc *Config) SetRetryLimit(limit int32) {
	if limit > 0 {
		wc.RetryLimit = limit
	} else {
		wc.RetryLimit = 0
	}
}

func (wc *Config) GetRetryLimit() int32 {
	return wc.RetryLimit
}

func (wc *Config) SetAgeLimit(duration string) {
	timeduration, err := time.ParseDuration(duration)
	if err == nil {
		wc.AgeLimit = timeduration
	}
}

func (wc *Config) GetAgeLimit() time.Duration {
	return wc.AgeLimit
}

func (wc *Config) SetMinBackoff(duration string) {
	timeduration, err := time.ParseDuration(duration)
	if err == nil {
		wc.MinBackoff = timeduration
	}
}

func (wc *Config) SetMaxBackoff(duration string) {
	timeduration, err := time.ParseDuration(duration)
	if err == nil {
		wc.MaxBackoff = timeduration
	}
}

func (wc *Config) SetMaxDoubling(attempts int32) {
	wc.MaxDoubling = attempts
}

func (wc *Config) GetInterval(retryAttempts int32) time.Duration {
	var interval time.Duration
	if wc.MaxDoubling <= 0 {
		interval = wc.MinBackoff
	}

	if wc.MaxDoubling+1 > retryAttempts {
		interval = time.Duration(retryAttempts) * wc.MinBackoff
	} else {
		interval = time.Duration(wc.MaxDoubling+1) * wc.MinBackoff
	}
	return wc.intervalPassFilter(interval)
}

func (wc *Config) intervalPassFilter(interval time.Duration) time.Duration {
	if wc.MaxBackoff == time.Duration(0) ||
		wc.MinBackoff > wc.MaxBackoff {
		return interval
	} else if wc.MaxBackoff > interval {
		return interval
	} else {
		return wc.MaxBackoff
	}

	return time.Duration(0)
}
<|endoftext|>


<commit_before>package worker

import (
	"errors"
	"github.com/Sirupsen/logrus"
	"sync"
)

type pooledWorker struct {
	ID int
}

/*
Pool holds desired number of worker instances, dispatch jobs to them, and handle their lifecycle.
*/
type Pool struct {
	workers     []*pooledWorker
	isRunning   bool
	mutex       *sync.Mutex
	jobReceiver chan func()
	stop        chan bool
}

/*
NewPool is a helper function that construct and return new Pool instance.
*/
func NewPool(workerNum int) *Pool {
	var workers = make([]*pooledWorker, workerNum)
	for i := range workers {
		workers[i] = &pooledWorker{ID: i + 1}
	}
	return &Pool{
		workers:     workers,
		isRunning:   false,
		mutex:       &sync.Mutex{},
		stop:        make(chan bool),
		jobReceiver: make(chan func(), 100),
	}
}

/*
Run prepares all underlying workers to receive jobs for execution.
Once Run is called, this Pool receives job until Sop is called.
*/
func (pool *Pool) Run() error {
	logrus.Infof("start workers")
	pool.mutex.Lock()
	defer pool.mutex.Unlock()

	if pool.isRunning == true {
		return errors.New("workers are already running")
	}

	for _, worker := range pool.workers {
		go pool.runWorker(worker)
	}
	pool.isRunning = true

	return nil
}

func (pool *Pool) runWorker(worker *pooledWorker) {
	logrus.Infof("start worker id: %d.", worker.ID)
	for {
		select {
		case <-pool.stop:
			logrus.Infof("stopping worker id: %d", worker.ID)
			return
		case job := <-pool.jobReceiver:
			logrus.Infof("receiving job on worker: %d", worker.ID)
			// To avoid given job's panic affect later jobs, wrap them with recover.
			func() {
				defer func() {
					if r := recover(); r != nil {
						logrus.Warnf("panic in given job. recovered: %+v", r)
					}
				}()
				job()
			}()
		}
	}
}

/*
Stop lets underlying workers stop receiving jobs
*/
func (pool *Pool) Stop() error {
	logrus.Infof("stop workers")
	pool.mutex.Lock()
	defer pool.mutex.Unlock()

	if pool.isRunning != true {
		return errors.New("workers are already stopped")
	}
	close(pool.stop)
	pool.isRunning = false

	return nil
}

/*
EnqueueJob appends new job to be executed.
*/
func (pool *Pool) EnqueueJob(job func()) {
	pool.jobReceiver <- job
}
<commit_msg>fix warning message format<commit_after>package worker

import (
	"errors"
	"github.com/Sirupsen/logrus"
	"sync"
)

type pooledWorker struct {
	ID int
}

/*
Pool holds desired number of worker instances, dispatch jobs to them, and handle their lifecycle.
*/
type Pool struct {
	workers     []*pooledWorker
	isRunning   bool
	mutex       *sync.Mutex
	jobReceiver chan func()
	stop        chan bool
}

/*
NewPool is a helper function that construct and return new Pool instance.
*/
func NewPool(workerNum int) *Pool {
	var workers = make([]*pooledWorker, workerNum)
	for i := range workers {
		workers[i] = &pooledWorker{ID: i + 1}
	}
	return &Pool{
		workers:     workers,
		isRunning:   false,
		mutex:       &sync.Mutex{},
		stop:        make(chan bool),
		jobReceiver: make(chan func(), 100),
	}
}

/*
Run prepares all underlying workers to receive jobs for execution.
Once Run is called, this Pool receives job until Sop is called.
*/
func (pool *Pool) Run() error {
	logrus.Infof("start workers")
	pool.mutex.Lock()
	defer pool.mutex.Unlock()

	if pool.isRunning == true {
		return errors.New("workers are already running")
	}

	for _, worker := range pool.workers {
		go pool.runWorker(worker)
	}
	pool.isRunning = true

	return nil
}

func (pool *Pool) runWorker(worker *pooledWorker) {
	logrus.Infof("start worker id: %d.", worker.ID)
	for {
		select {
		case <-pool.stop:
			logrus.Infof("stopping worker id: %d", worker.ID)
			return
		case job := <-pool.jobReceiver:
			logrus.Infof("receiving job on worker: %d", worker.ID)
			// To avoid given job's panic affect later jobs, wrap them with recover.
			func() {
				defer func() {
					if r := recover(); r != nil {
						logrus.Warnf("panic in given job. recovered: %#v", r)
					}
				}()
				job()
			}()
		}
	}
}

/*
Stop lets underlying workers stop receiving jobs
*/
func (pool *Pool) Stop() error {
	logrus.Infof("stop workers")
	pool.mutex.Lock()
	defer pool.mutex.Unlock()

	if pool.isRunning != true {
		return errors.New("workers are already stopped")
	}
	close(pool.stop)
	pool.isRunning = false

	return nil
}

/*
EnqueueJob appends new job to be executed.
*/
func (pool *Pool) EnqueueJob(job func()) {
	pool.jobReceiver <- job
}
<|endoftext|>


<commit_before>package client

import (
	"encoding/json"
	"fmt"
	"io"
	"net/url"
	"sort"
	"strings"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/docker/docker/api/types"
	flag "github.com/docker/docker/pkg/mflag"
	"github.com/docker/docker/pkg/units"
)

type containerStats struct {
	Name             string
	CPUPercentage    float64
	Memory           float64
	MemoryLimit      float64
	MemoryPercentage float64
	NetworkRx        float64
	NetworkTx        float64
	mu               sync.RWMutex
	err              error
}

func (s *containerStats) Collect(cli *DockerCli, streamStats bool) {
	v := url.Values{}
	if streamStats {
		v.Set("stream", "1")
	} else {
		v.Set("stream", "0")
	}
	stream, _, _, err := cli.call("GET", "/containers/"+s.Name+"/stats?"+v.Encode(), nil, nil)
	if err != nil {
		s.mu.Lock()
		s.err = err
		s.mu.Unlock()
		return
	}
	defer stream.Close()
	var (
		previousCPU    uint64
		previousSystem uint64
		dec            = json.NewDecoder(stream)
		u              = make(chan error, 1)
	)
	go func() {
		for {
			var v *types.Stats
			if err := dec.Decode(&v); err != nil {
				u <- err
				return
			}
			var (
				memPercent = float64(v.MemoryStats.Usage) / float64(v.MemoryStats.Limit) * 100.0
				cpuPercent = 0.0
			)
			previousCPU = v.PreCpuStats.CpuUsage.TotalUsage
			previousSystem = v.PreCpuStats.SystemUsage
			cpuPercent = calculateCPUPercent(previousCPU, previousSystem, v)
			s.mu.Lock()
			s.CPUPercentage = cpuPercent
			s.Memory = float64(v.MemoryStats.Usage)
			s.MemoryLimit = float64(v.MemoryStats.Limit)
			s.MemoryPercentage = memPercent
			s.NetworkRx = float64(v.Network.RxBytes)
			s.NetworkTx = float64(v.Network.TxBytes)
			s.mu.Unlock()
			u <- nil
			if !streamStats {
				return
			}
		}
	}()
	for {
		select {
		case <-time.After(2 * time.Second):
			// zero out the values if we have not received an update within
			// the specified duration.
			s.mu.Lock()
			s.CPUPercentage = 0
			s.Memory = 0
			s.MemoryPercentage = 0
			s.mu.Unlock()
		case err := <-u:
			if err != nil {
				s.mu.Lock()
				s.err = err
				s.mu.Unlock()
				return
			}
		}
		if !streamStats {
			return
		}
	}
}

func (s *containerStats) Display(w io.Writer) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	if s.err != nil {
		return s.err
	}
	fmt.Fprintf(w, "%s\t%.2f%%\t%s/%s\t%.2f%%\t%s/%s\n",
		s.Name,
		s.CPUPercentage,
		units.HumanSize(s.Memory), units.HumanSize(s.MemoryLimit),
		s.MemoryPercentage,
		units.HumanSize(s.NetworkRx), units.HumanSize(s.NetworkTx))
	return nil
}

// CmdStats displays a live stream of resource usage statistics for one or more containers.
//
// This shows real-time information on CPU usage, memory usage, and network I/O.
//
// Usage: docker stats CONTAINER [CONTAINER...]
func (cli *DockerCli) CmdStats(args ...string) error {
	cmd := cli.Subcmd("stats", []string{"CONTAINER [CONTAINER...]"}, "Display a live stream of one or more containers' resource usage statistics", true)
	noStream := cmd.Bool([]string{"-no-stream"}, false, "Disable streaming stats and only pull the first result")
	cmd.Require(flag.Min, 1)
	cmd.ParseFlags(args, true)

	names := cmd.Args()
	sort.Strings(names)
	var (
		cStats []*containerStats
		w      = tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
	)
	printHeader := func() {
		if !*noStream {
			fmt.Fprint(cli.out, "\033[2J")
			fmt.Fprint(cli.out, "\033[H")
		}
		io.WriteString(w, "CONTAINER\tCPU %\tMEM USAGE/LIMIT\tMEM %\tNET I/O\n")
	}
	for _, n := range names {
		s := &containerStats{Name: n}
		cStats = append(cStats, s)
		go s.Collect(cli, !*noStream)
	}
	// do a quick pause so that any failed connections for containers that do not exist are able to be
	// evicted before we display the initial or default values.
	time.Sleep(1500 * time.Millisecond)
	var errs []string
	for _, c := range cStats {
		c.mu.Lock()
		if c.err != nil {
			errs = append(errs, fmt.Sprintf("%s: %v", c.Name, c.err))
		}
		c.mu.Unlock()
	}
	if len(errs) > 0 {
		return fmt.Errorf("%s", strings.Join(errs, ", "))
	}
	for range time.Tick(500 * time.Millisecond) {
		printHeader()
		toRemove := []int{}
		for i, s := range cStats {
			if err := s.Display(w); err != nil && !*noStream {
				toRemove = append(toRemove, i)
			}
		}
		for j := len(toRemove) - 1; j >= 0; j-- {
			i := toRemove[j]
			cStats = append(cStats[:i], cStats[i+1:]...)
		}
		if len(cStats) == 0 {
			return nil
		}
		w.Flush()
		if *noStream {
			break
		}
	}
	return nil
}

func calculateCPUPercent(previousCPU, previousSystem uint64, v *types.Stats) float64 {
	var (
		cpuPercent = 0.0
		// calculate the change for the cpu usage of the container in between readings
		cpuDelta = float64(v.CpuStats.CpuUsage.TotalUsage - previousCPU)
		// calculate the change for the entire system between readings
		systemDelta = float64(v.CpuStats.SystemUsage - previousSystem)
	)

	if systemDelta > 0.0 && cpuDelta > 0.0 {
		cpuPercent = (cpuDelta / systemDelta) * float64(len(v.CpuStats.CpuUsage.PercpuUsage)) * 100.0
	}
	return cpuPercent
}
<commit_msg>Docker stats displays all running containers<commit_after>package client

import (
	"encoding/json"
	"fmt"
	"io"
	"net/url"
	"sort"
	"strings"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/docker/docker/api/types"
	flag "github.com/docker/docker/pkg/mflag"
	"github.com/docker/docker/pkg/units"
)

type containerStats struct {
	Name             string
	CPUPercentage    float64
	Memory           float64
	MemoryLimit      float64
	MemoryPercentage float64
	NetworkRx        float64
	NetworkTx        float64
	mu               sync.RWMutex
	err              error
}

func (s *containerStats) Collect(cli *DockerCli, streamStats bool) {
	v := url.Values{}
	if streamStats {
		v.Set("stream", "1")
	} else {
		v.Set("stream", "0")
	}
	stream, _, _, err := cli.call("GET", "/containers/"+s.Name+"/stats?"+v.Encode(), nil, nil)
	if err != nil {
		s.mu.Lock()
		s.err = err
		s.mu.Unlock()
		return
	}
	defer stream.Close()
	var (
		previousCPU    uint64
		previousSystem uint64
		dec            = json.NewDecoder(stream)
		u              = make(chan error, 1)
	)
	go func() {
		for {
			var v *types.Stats
			if err := dec.Decode(&v); err != nil {
				u <- err
				return
			}
			var (
				memPercent = float64(v.MemoryStats.Usage) / float64(v.MemoryStats.Limit) * 100.0
				cpuPercent = 0.0
			)
			previousCPU = v.PreCpuStats.CpuUsage.TotalUsage
			previousSystem = v.PreCpuStats.SystemUsage
			cpuPercent = calculateCPUPercent(previousCPU, previousSystem, v)
			s.mu.Lock()
			s.CPUPercentage = cpuPercent
			s.Memory = float64(v.MemoryStats.Usage)
			s.MemoryLimit = float64(v.MemoryStats.Limit)
			s.MemoryPercentage = memPercent
			s.NetworkRx = float64(v.Network.RxBytes)
			s.NetworkTx = float64(v.Network.TxBytes)
			s.mu.Unlock()
			u <- nil
			if !streamStats {
				return
			}
		}
	}()
	for {
		select {
		case <-time.After(2 * time.Second):
			// zero out the values if we have not received an update within
			// the specified duration.
			s.mu.Lock()
			s.CPUPercentage = 0
			s.Memory = 0
			s.MemoryPercentage = 0
			s.mu.Unlock()
		case err := <-u:
			if err != nil {
				s.mu.Lock()
				s.err = err
				s.mu.Unlock()
				return
			}
		}
		if !streamStats {
			return
		}
	}
}

func (s *containerStats) Display(w io.Writer) error {
	s.mu.RLock()
	defer s.mu.RUnlock()
	if s.err != nil {
		return s.err
	}
	fmt.Fprintf(w, "%s\t%.2f%%\t%s/%s\t%.2f%%\t%s/%s\n",
		s.Name,
		s.CPUPercentage,
		units.HumanSize(s.Memory), units.HumanSize(s.MemoryLimit),
		s.MemoryPercentage,
		units.HumanSize(s.NetworkRx), units.HumanSize(s.NetworkTx))
	return nil
}

// CmdStats displays a live stream of resource usage statistics for one or more containers.
//
// This shows real-time information on CPU usage, memory usage, and network I/O.
//
// Usage: docker stats CONTAINER [CONTAINER...]
func (cli *DockerCli) CmdStats(args ...string) error {
	cmd := cli.Subcmd("stats", []string{"CONTAINER [CONTAINER...]"}, "Display a live stream of one or more containers' resource usage statistics", true)
	noStream := cmd.Bool([]string{"-no-stream"}, false, "Disable streaming stats and only pull the first result")
	cmd.Require(flag.Min, 0)
	cmd.ParseFlags(args, true)

	var names []string
	if len(args) > 0 {
		names = cmd.Args()
	} else {
		body, _, err := readBody(cli.call("GET", "/containers/json", nil, nil))
		if err != nil {
			return err
		}
		var containers []struct{ ID string }
		if err := json.Unmarshal(body, &containers); err != nil {
			return err
		}
		names = make([]string, len(containers))
		for i, item := range containers {
			names[i] = item.ID[:12]
		}
	}
	sort.Strings(names)
	var (
		cStats []*containerStats
		w      = tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
	)
	printHeader := func() {
		if !*noStream {
			fmt.Fprint(cli.out, "\033[2J")
			fmt.Fprint(cli.out, "\033[H")
		}
		io.WriteString(w, "CONTAINER\tCPU %\tMEM USAGE/LIMIT\tMEM %\tNET I/O\n")
	}
	for _, n := range names {
		s := &containerStats{Name: n}
		cStats = append(cStats, s)
		go s.Collect(cli, !*noStream)
	}
	// do a quick pause so that any failed connections for containers that do not exist are able to be
	// evicted before we display the initial or default values.
	time.Sleep(1500 * time.Millisecond)
	var errs []string
	for _, c := range cStats {
		c.mu.Lock()
		if c.err != nil {
			errs = append(errs, fmt.Sprintf("%s: %v", c.Name, c.err))
		}
		c.mu.Unlock()
	}
	if len(errs) > 0 {
		return fmt.Errorf("%s", strings.Join(errs, ", "))
	}
	for range time.Tick(500 * time.Millisecond) {
		printHeader()
		toRemove := []int{}
		for i, s := range cStats {
			if err := s.Display(w); err != nil && !*noStream {
				toRemove = append(toRemove, i)
			}
		}
		for j := len(toRemove) - 1; j >= 0; j-- {
			i := toRemove[j]
			cStats = append(cStats[:i], cStats[i+1:]...)
		}
		if len(cStats) == 0 {
			return nil
		}
		w.Flush()
		if *noStream {
			break
		}
	}
	return nil
}

func calculateCPUPercent(previousCPU, previousSystem uint64, v *types.Stats) float64 {
	var (
		cpuPercent = 0.0
		// calculate the change for the cpu usage of the container in between readings
		cpuDelta = float64(v.CpuStats.CpuUsage.TotalUsage - previousCPU)
		// calculate the change for the entire system between readings
		systemDelta = float64(v.CpuStats.SystemUsage - previousSystem)
	)

	if systemDelta > 0.0 && cpuDelta > 0.0 {
		cpuPercent = (cpuDelta / systemDelta) * float64(len(v.CpuStats.CpuUsage.PercpuUsage)) * 100.0
	}
	return cpuPercent
}
<|endoftext|>