<commit_before>package api

import (
	"koding/db/mongodb/modelhelper"
	"math/rand"
	"net/http"
	"socialapi/config"
	"socialapi/models"
	"socialapi/workers/common/mux"
	"socialapi/workers/integration/webhook"
	"socialapi/workers/integration/webhook/services"
	"testing"
	"time"

	"github.com/koding/logging"
	"github.com/koding/runner"
	"github.com/rcrowley/go-tigertonic/mocking"
	. "github.com/smartystreets/goconvey/convey"
)

var (
	r *runner.Runner
	h *Handler
	m *mux.Mux
)

func newRequest(body string, channelId int64, groupName string) *WebhookRequest {
	return &WebhookRequest{
		Message: &webhook.Message{
			Body:      body,
			ChannelId: channelId,
		},
		GroupName: groupName,
	}
}

func newPrepareRequest(data *services.ServiceInput) *PrepareRequest {
	return &PrepareRequest{
		Data: data,
	}
}

func newBotChannelRequest(groupName string) *BotChannelRequest {
	return &BotChannelRequest{
		GroupName: groupName,
	}
}

func init() {
	var err error
	r = runner.New("test")
	if err := r.Init(); err != nil {
		panic(err)
	}
	r.Log.SetLevel(logging.CRITICAL)

	defer r.Close()
	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)

	mc := mux.NewConfig("testing", "", "")
	m = mux.New(mc, r.Log)

	h, err = NewHandler(r.Log)
	if err != nil {
		panic(err)
	}

	h.AddHandlers(m)

	rand.Seed(time.Now().UTC().UnixNano())
}

func TestWebhookListen(t *testing.T) {

	account, err := models.CreateAccountInBothDbsWithNick("sinan")
	if err != nil {
		t.Fatalf("could not create test account: %s", err)
	}

	channelIntegration := webhook.CreateTestChannelIntegration(t)

	Convey("while testing incoming webhook", t, func() {

		groupName := models.RandomName()
		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_TOPIC, groupName)

		Convey("users should not be able to send any message when they don't have valid token", func() {
			token := "123123"
			s, _, _, err := h.Push(
				mocking.URL(m, "POST", "/webhook/push/"+token),
				mocking.Header(nil),
				newRequest("hey", channel.Id, "koding"),
			)
			So(err, ShouldNotBeNil)
			So(s, ShouldEqual, http.StatusNotFound)

			token = ""
			s, _, _, err = h.Push(
				mocking.URL(m, "POST", "/webhook/push/"+token),
				mocking.Header(nil),
				newRequest("hey", channel.Id, "koding"),
			)
			So(err.Error(), ShouldEqual, ErrTokenNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)
		})

		Convey("users should not be able to send any message when their request does not include body or channel name", func() {

			token := "123123"
			s, _, _, err := h.Push(
				mocking.URL(m, "POST", "/webhook/push/"+token),
				mocking.Header(nil),
				newRequest("", channel.Id, "koding"),
			)
			So(err.Error(), ShouldEqual, ErrBodyNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			s, _, _, err = h.Push(
				mocking.URL(m, "POST", "/webhook/push/"+token),
				mocking.Header(nil),
				newRequest("hey", 0, "koding"),
			)
			So(err.Error(), ShouldEqual, ErrChannelNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			s, _, _, err = h.Push(
				mocking.URL(m, "POST", "/webhook/push/"+token),
				mocking.Header(nil),
				newRequest("hey", channel.Id, ""),
			)
			So(err.Error(), ShouldEqual, ErrGroupNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)
		})

		Convey("users should be able to send message when token is valid", func() {

			Convey("related integrations must be created", func() {

				token := channelIntegration.Token
				s, _, _, err := h.Push(
					mocking.URL(m, "POST", "/webhook/push/"+token),
					mocking.Header(nil),
					newRequest("hey", channel.Id, "koding"),
				)

				So(err, ShouldBeNil)
				So(s, ShouldEqual, http.StatusOK)
			})
		})

	})
}

func TestWebhookPrepare(t *testing.T) {

	webhook.CreateIterableIntegration(t)

	Convey("while testing incoming webhook", t, func() {

		Convey("users should not be able to send any message when they don't have valid token", func() {
			token := ""
			integrationName := "testing"
			s, _, _, err := h.Prepare(
				mocking.URL(m, "POST", "/webhook/"+integrationName+"/"+token),
				mocking.Header(nil),
				newPrepareRequest(&services.ServiceInput{}),
			)
			So(err.Error(), ShouldEqual, ErrTokenNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			token = "123123123"
			integrationName = ""
			s, _, _, err = h.Prepare(
				mocking.URL(m, "POST", "/webhook/"+integrationName+"/"+token),
				mocking.Header(nil),
				newPrepareRequest(&services.ServiceInput{}),
			)
			So(err.Error(), ShouldEqual, ErrNameNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			integrationName = "testing"
			s, _, _, err = h.Prepare(
				mocking.URL(m, "POST", "/webhook/"+integrationName+"/"+token),
				mocking.Header(nil),
				newPrepareRequest(&services.ServiceInput{}),
			)
			So(err, ShouldNotBeNil)
			So(s, ShouldEqual, http.StatusNotFound)

			push = func(endPoint string, request map[string]string) error {
				return nil
			}
			s, _, _, err = h.Prepare(
				mocking.URL(m, "POST", "/webhook/iterable/"+token),
				mocking.Header(nil),
				newPrepareRequest(&services.ServiceInput{}),
			)
			// TODO temporary assertion
			So(err, ShouldBeNil)
			So(s, ShouldEqual, http.StatusOK)
		})
	})
}

func TestWebhookFetchBotChannel(t *testing.T) {

	Convey("while testing bot channel fetcher", t, func() {

		Convey("users should not be able to fetch bot channel when they don't have valid nick or group name", func() {
			nick := ""
			s, _, _, err := h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest("hi"),
			)
			So(err.Error(), ShouldEqual, ErrUsernameNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			nick = "canthefason"
			s, _, _, err = h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest(""),
			)
			So(err.Error(), ShouldEqual, ErrGroupNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)
		})

		Convey("we should not be able to fetch bot channel for a user when they are not participant of the group", func() {

			creator, err := models.CreateAccountInBothDbsWithNick("canthefason")
			So(err, ShouldBeNil)

			nick := models.RandomName()
			s, _, _, err := h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest("hi"),
			)
			So(err.Error(), ShouldEqual, ErrAccountNotFound.Error())
			So(s, ShouldEqual, http.StatusBadRequest)
			//So(err, ShouldEqual, ErrAccountIsNotParticipant)

			account, err := models.CreateAccountInBothDbsWithNick("sinan")
			if err != nil {
				t.Fatalf("could not create test account: %s", err)
			}

			s, _, _, err = h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+account.Nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest("hi"),
			)
			So(err.Error(), ShouldEqual, ErrGroupNotFound.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			groupChannel := models.CreateTypedChannelWithTest(creator.Id, models.Channel_TYPE_GROUP)
			s, _, _, err = h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+account.Nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest(groupChannel.GroupName),
			)
			So(err.Error(), ShouldEqual, ErrAccountIsNotParticipant.Error())
			So(s, ShouldEqual, http.StatusBadRequest)
		})

		Convey("we should be able to fetch bot channel for the user with given nickname when the user is participant of the group", func() {
			creator, err := models.CreateAccountInBothDbsWithNick("canthefason")
			So(err, ShouldBeNil)

			account, err := models.CreateAccountInBothDbsWithNick("sinan")
			So(err, ShouldBeNil)

			groupChannel := models.CreateTypedChannelWithTest(creator.Id, models.Channel_TYPE_GROUP)
			_, err = groupChannel.AddParticipant(account.Id)
			So(err, ShouldBeNil)

			s, _, res, err := h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+account.Nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest(groupChannel.GroupName),
			)
			So(err, ShouldBeNil)
			So(s, ShouldEqual, http.StatusOK)
			So(res, ShouldNotBeNil)
			resultMap, ok := res.(map[string]string)
			So(ok, ShouldEqual, true)
			val, ok := resultMap["channelId"]
			So(ok, ShouldEqual, true)
			So(val, ShouldNotEqual, "")
		})

	})
}
<commit_msg>webhook: replace PrepareRequest with ServiceInput for tests<commit_after>package api

import (
	"koding/db/mongodb/modelhelper"
	"math/rand"
	"net/http"
	"socialapi/config"
	"socialapi/models"
	"socialapi/workers/common/mux"
	"socialapi/workers/integration/webhook"
	"socialapi/workers/integration/webhook/services"
	"testing"
	"time"

	"github.com/koding/logging"
	"github.com/koding/runner"
	"github.com/rcrowley/go-tigertonic/mocking"
	. "github.com/smartystreets/goconvey/convey"
)

var (
	r *runner.Runner
	h *Handler
	m *mux.Mux
)

func newRequest(body string, channelId int64, groupName string) *WebhookRequest {
	return &WebhookRequest{
		Message: &webhook.Message{
			Body:      body,
			ChannelId: channelId,
		},
		GroupName: groupName,
	}
}

func newBotChannelRequest(groupName string) *BotChannelRequest {
	return &BotChannelRequest{
		GroupName: groupName,
	}
}

func init() {
	var err error
	r = runner.New("test")
	if err := r.Init(); err != nil {
		panic(err)
	}
	r.Log.SetLevel(logging.CRITICAL)

	defer r.Close()
	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)

	mc := mux.NewConfig("testing", "", "")
	m = mux.New(mc, r.Log)

	h, err = NewHandler(r.Log)
	if err != nil {
		panic(err)
	}

	h.AddHandlers(m)

	rand.Seed(time.Now().UTC().UnixNano())
}

func TestWebhookListen(t *testing.T) {

	account, err := models.CreateAccountInBothDbsWithNick("sinan")
	if err != nil {
		t.Fatalf("could not create test account: %s", err)
	}

	channelIntegration := webhook.CreateTestChannelIntegration(t)

	Convey("while testing incoming webhook", t, func() {

		groupName := models.RandomName()
		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_TOPIC, groupName)

		Convey("users should not be able to send any message when they don't have valid token", func() {
			token := "123123"
			s, _, _, err := h.Push(
				mocking.URL(m, "POST", "/webhook/push/"+token),
				mocking.Header(nil),
				newRequest("hey", channel.Id, "koding"),
			)
			So(err, ShouldNotBeNil)
			So(s, ShouldEqual, http.StatusNotFound)

			token = ""
			s, _, _, err = h.Push(
				mocking.URL(m, "POST", "/webhook/push/"+token),
				mocking.Header(nil),
				newRequest("hey", channel.Id, "koding"),
			)
			So(err.Error(), ShouldEqual, ErrTokenNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)
		})

		Convey("users should not be able to send any message when their request does not include body or channel name", func() {

			token := "123123"
			s, _, _, err := h.Push(
				mocking.URL(m, "POST", "/webhook/push/"+token),
				mocking.Header(nil),
				newRequest("", channel.Id, "koding"),
			)
			So(err.Error(), ShouldEqual, ErrBodyNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			s, _, _, err = h.Push(
				mocking.URL(m, "POST", "/webhook/push/"+token),
				mocking.Header(nil),
				newRequest("hey", 0, "koding"),
			)
			So(err.Error(), ShouldEqual, ErrChannelNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			s, _, _, err = h.Push(
				mocking.URL(m, "POST", "/webhook/push/"+token),
				mocking.Header(nil),
				newRequest("hey", channel.Id, ""),
			)
			So(err.Error(), ShouldEqual, ErrGroupNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)
		})

		Convey("users should be able to send message when token is valid", func() {

			Convey("related integrations must be created", func() {

				token := channelIntegration.Token
				s, _, _, err := h.Push(
					mocking.URL(m, "POST", "/webhook/push/"+token),
					mocking.Header(nil),
					newRequest("hey", channel.Id, "koding"),
				)

				So(err, ShouldBeNil)
				So(s, ShouldEqual, http.StatusOK)
			})
		})

	})
}

func TestWebhookPrepare(t *testing.T) {

	webhook.CreateIterableIntegration(t)

	Convey("while testing incoming webhook", t, func() {

		Convey("users should not be able to send any message when they don't have valid token", func() {
			token := ""
			integrationName := "testing"
			s, _, _, err := h.Prepare(
				mocking.URL(m, "POST", "/webhook/"+integrationName+"/"+token),
				mocking.Header(nil),
				services.ServiceInput{},
			)
			So(err.Error(), ShouldEqual, ErrTokenNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			token = "123123123"
			integrationName = ""
			s, _, _, err = h.Prepare(
				mocking.URL(m, "POST", "/webhook/"+integrationName+"/"+token),
				mocking.Header(nil),
				services.ServiceInput{},
			)
			So(err.Error(), ShouldEqual, ErrNameNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			integrationName = "testing"
			s, _, _, err = h.Prepare(
				mocking.URL(m, "POST", "/webhook/"+integrationName+"/"+token),
				mocking.Header(nil),
				services.ServiceInput{},
			)
			So(err, ShouldNotBeNil)
			So(s, ShouldEqual, http.StatusNotFound)

			push = func(endPoint string, request map[string]string) error {
				return nil
			}
			s, _, _, err = h.Prepare(
				mocking.URL(m, "POST", "/webhook/iterable/"+token),
				mocking.Header(nil),
				services.ServiceInput{},
			)

			So(err, ShouldBeNil)
			So(s, ShouldEqual, http.StatusOK)
		})
	})
}

func TestWebhookFetchBotChannel(t *testing.T) {

	Convey("while testing bot channel fetcher", t, func() {

		Convey("users should not be able to fetch bot channel when they don't have valid nick or group name", func() {
			nick := ""
			s, _, _, err := h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest("hi"),
			)
			So(err.Error(), ShouldEqual, ErrUsernameNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			nick = "canthefason"
			s, _, _, err = h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest(""),
			)
			So(err.Error(), ShouldEqual, ErrGroupNotSet.Error())
			So(s, ShouldEqual, http.StatusBadRequest)
		})

		Convey("we should not be able to fetch bot channel for a user when they are not participant of the group", func() {

			creator, err := models.CreateAccountInBothDbsWithNick("canthefason")
			So(err, ShouldBeNil)

			nick := models.RandomName()
			s, _, _, err := h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest("hi"),
			)
			So(err.Error(), ShouldEqual, ErrAccountNotFound.Error())
			So(s, ShouldEqual, http.StatusBadRequest)
			//So(err, ShouldEqual, ErrAccountIsNotParticipant)

			account, err := models.CreateAccountInBothDbsWithNick("sinan")
			if err != nil {
				t.Fatalf("could not create test account: %s", err)
			}

			s, _, _, err = h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+account.Nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest("hi"),
			)
			So(err.Error(), ShouldEqual, ErrGroupNotFound.Error())
			So(s, ShouldEqual, http.StatusBadRequest)

			groupChannel := models.CreateTypedChannelWithTest(creator.Id, models.Channel_TYPE_GROUP)
			s, _, _, err = h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+account.Nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest(groupChannel.GroupName),
			)
			So(err.Error(), ShouldEqual, ErrAccountIsNotParticipant.Error())
			So(s, ShouldEqual, http.StatusBadRequest)
		})

		Convey("we should be able to fetch bot channel for the user with given nickname when the user is participant of the group", func() {
			creator, err := models.CreateAccountInBothDbsWithNick("canthefason")
			So(err, ShouldBeNil)

			account, err := models.CreateAccountInBothDbsWithNick("sinan")
			So(err, ShouldBeNil)

			groupChannel := models.CreateTypedChannelWithTest(creator.Id, models.Channel_TYPE_GROUP)
			_, err = groupChannel.AddParticipant(account.Id)
			So(err, ShouldBeNil)

			s, _, res, err := h.FetchBotChannel(
				mocking.URL(m, "POST", "/account/"+account.Nick+"/bot-channel"),
				mocking.Header(nil),
				newBotChannelRequest(groupChannel.GroupName),
			)
			So(err, ShouldBeNil)
			So(s, ShouldEqual, http.StatusOK)
			So(res, ShouldNotBeNil)
			resultMap, ok := res.(map[string]string)
			So(ok, ShouldEqual, true)
			val, ok := resultMap["channelId"]
			So(ok, ShouldEqual, true)
			So(val, ShouldNotEqual, "")
		})

	})
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package dsa implements the Digital Signature Algorithm, as defined in FIPS 186-3.
//
// The DSA operations in this package are not implemented using constant-time algorithms.
package dsa

import (
	"errors"
	"io"
	"math/big"

	"crypto/internal/randutil"
)

// Parameters represents the domain parameters for a key. These parameters can
// be shared across many keys. The bit length of Q must be a multiple of 8.
type Parameters struct {
	P, Q, G *big.Int
}

// PublicKey represents a DSA public key.
type PublicKey struct {
	Parameters
	Y *big.Int
}

// PrivateKey represents a DSA private key.
type PrivateKey struct {
	PublicKey
	X *big.Int
}

// ErrInvalidPublicKey results when a public key is not usable by this code.
// FIPS is quite strict about the format of DSA keys, but other code may be
// less so. Thus, when using keys which may have been generated by other code,
// this error must be handled.
var ErrInvalidPublicKey = errors.New("crypto/dsa: invalid public key")

// ParameterSizes is an enumeration of the acceptable bit lengths of the primes
// in a set of DSA parameters. See FIPS 186-3, section 4.2.
type ParameterSizes int

const (
	L1024N160 ParameterSizes = iota
	L2048N224
	L2048N256
	L3072N256
)

// numMRTests is the number of Miller-Rabin primality tests that we perform. We
// pick the largest recommended number from table C.1 of FIPS 186-3.
const numMRTests = 64

// GenerateParameters puts a random, valid set of DSA parameters into params.
// This function can take many seconds, even on fast machines.
func GenerateParameters(params *Parameters, rand io.Reader, sizes ParameterSizes) error {
	// This function doesn't follow FIPS 186-3 exactly in that it doesn't
	// use a verification seed to generate the primes. The verification
	// seed doesn't appear to be exported or used by other code and
	// omitting it makes the code cleaner.

	var L, N int
	switch sizes {
	case L1024N160:
		L = 1024
		N = 160
	case L2048N224:
		L = 2048
		N = 224
	case L2048N256:
		L = 2048
		N = 256
	case L3072N256:
		L = 3072
		N = 256
	default:
		return errors.New("crypto/dsa: invalid ParameterSizes")
	}

	qBytes := make([]byte, N/8)
	pBytes := make([]byte, L/8)

	q := new(big.Int)
	p := new(big.Int)
	rem := new(big.Int)
	one := new(big.Int)
	one.SetInt64(1)

GeneratePrimes:
	for {
		if _, err := io.ReadFull(rand, qBytes); err != nil {
			return err
		}

		qBytes[len(qBytes)-1] |= 1
		qBytes[0] |= 0x80
		q.SetBytes(qBytes)

		if !q.ProbablyPrime(numMRTests) {
			continue
		}

		for i := 0; i < 4*L; i++ {
			if _, err := io.ReadFull(rand, pBytes); err != nil {
				return err
			}

			pBytes[len(pBytes)-1] |= 1
			pBytes[0] |= 0x80

			p.SetBytes(pBytes)
			rem.Mod(p, q)
			rem.Sub(rem, one)
			p.Sub(p, rem)
			if p.BitLen() < L {
				continue
			}

			if !p.ProbablyPrime(numMRTests) {
				continue
			}

			params.P = p
			params.Q = q
			break GeneratePrimes
		}
	}

	h := new(big.Int)
	h.SetInt64(2)
	g := new(big.Int)

	pm1 := new(big.Int).Sub(p, one)
	e := new(big.Int).Div(pm1, q)

	for {
		g.Exp(h, e, p)
		if g.Cmp(one) == 0 {
			h.Add(h, one)
			continue
		}

		params.G = g
		return nil
	}
}

// GenerateKey generates a public&private key pair. The Parameters of the
// PrivateKey must already be valid (see GenerateParameters).
func GenerateKey(priv *PrivateKey, rand io.Reader) error {
	if priv.P == nil || priv.Q == nil || priv.G == nil {
		return errors.New("crypto/dsa: parameters not set up before generating key")
	}

	x := new(big.Int)
	xBytes := make([]byte, priv.Q.BitLen()/8)

	for {
		_, err := io.ReadFull(rand, xBytes)
		if err != nil {
			return err
		}
		x.SetBytes(xBytes)
		if x.Sign() != 0 && x.Cmp(priv.Q) < 0 {
			break
		}
	}

	priv.X = x
	priv.Y = new(big.Int)
	priv.Y.Exp(priv.G, x, priv.P)
	return nil
}

// fermatInverse calculates the inverse of k in GF(P) using Fermat's method.
// This has better constant-time properties than Euclid's method (implemented
// in math/big.Int.ModInverse) although math/big itself isn't strictly
// constant-time so it's not perfect.
func fermatInverse(k, P *big.Int) *big.Int {
	two := big.NewInt(2)
	pMinus2 := new(big.Int).Sub(P, two)
	return new(big.Int).Exp(k, pMinus2, P)
}

// Sign signs an arbitrary length hash (which should be the result of hashing a
// larger message) using the private key, priv. It returns the signature as a
// pair of integers. The security of the private key depends on the entropy of
// rand.
//
// Note that FIPS 186-3 section 4.6 specifies that the hash should be truncated
// to the byte-length of the subgroup. This function does not perform that
// truncation itself.
//
// Be aware that calling Sign with an attacker-controlled PrivateKey may
// require an arbitrary amount of CPU.
func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err error) {
	randutil.MaybeReadByte(rand)

	// FIPS 186-3, section 4.6

	n := priv.Q.BitLen()
	if priv.Q.Sign() <= 0 || priv.P.Sign() <= 0 || priv.G.Sign() <= 0 || priv.X.Sign() <= 0 || n&7 != 0 {
		err = ErrInvalidPublicKey
		return
	}
	n >>= 3

	var attempts int
	for attempts = 10; attempts > 0; attempts-- {
		k := new(big.Int)
		buf := make([]byte, n)
		for {
			_, err = io.ReadFull(rand, buf)
			if err != nil {
				return
			}
			k.SetBytes(buf)
			// priv.Q must be >= 128 because the test above
			// requires it to be > 0 and that
			//    ceil(log_2(Q)) mod 8 = 0
			// Thus this loop will quickly terminate.
			if k.Sign() > 0 && k.Cmp(priv.Q) < 0 {
				break
			}
		}

		kInv := fermatInverse(k, priv.Q)

		r = new(big.Int).Exp(priv.G, k, priv.P)
		r.Mod(r, priv.Q)

		if r.Sign() == 0 {
			continue
		}

		z := k.SetBytes(hash)

		s = new(big.Int).Mul(priv.X, r)
		s.Add(s, z)
		s.Mod(s, priv.Q)
		s.Mul(s, kInv)
		s.Mod(s, priv.Q)

		if s.Sign() != 0 {
			break
		}
	}

	// Only degenerate private keys will require more than a handful of
	// attempts.
	if attempts == 0 {
		return nil, nil, ErrInvalidPublicKey
	}

	return
}

// Verify verifies the signature in r, s of hash using the public key, pub. It
// reports whether the signature is valid.
//
// Note that FIPS 186-3 section 4.6 specifies that the hash should be truncated
// to the byte-length of the subgroup. This function does not perform that
// truncation itself.
func Verify(pub *PublicKey, hash []byte, r, s *big.Int) bool {
	// FIPS 186-3, section 4.7

	if pub.P.Sign() == 0 {
		return false
	}

	if r.Sign() < 1 || r.Cmp(pub.Q) >= 0 {
		return false
	}
	if s.Sign() < 1 || s.Cmp(pub.Q) >= 0 {
		return false
	}

	w := new(big.Int).ModInverse(s, pub.Q)

	n := pub.Q.BitLen()
	if n&7 != 0 {
		return false
	}
	z := new(big.Int).SetBytes(hash)

	u1 := new(big.Int).Mul(z, w)
	u1.Mod(u1, pub.Q)
	u2 := w.Mul(r, w)
	u2.Mod(u2, pub.Q)
	v := u1.Exp(pub.G, u1, pub.P)
	u2.Exp(pub.Y, u2, pub.P)
	v.Mul(v, u2)
	v.Mod(v, pub.P)
	v.Mod(v, pub.Q)

	return v.Cmp(r) == 0
}
<commit_msg>crypto/dsa: change bitwise checks to mod operations<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package dsa implements the Digital Signature Algorithm, as defined in FIPS 186-3.
//
// The DSA operations in this package are not implemented using constant-time algorithms.
package dsa

import (
	"errors"
	"io"
	"math/big"

	"crypto/internal/randutil"
)

// Parameters represents the domain parameters for a key. These parameters can
// be shared across many keys. The bit length of Q must be a multiple of 8.
type Parameters struct {
	P, Q, G *big.Int
}

// PublicKey represents a DSA public key.
type PublicKey struct {
	Parameters
	Y *big.Int
}

// PrivateKey represents a DSA private key.
type PrivateKey struct {
	PublicKey
	X *big.Int
}

// ErrInvalidPublicKey results when a public key is not usable by this code.
// FIPS is quite strict about the format of DSA keys, but other code may be
// less so. Thus, when using keys which may have been generated by other code,
// this error must be handled.
var ErrInvalidPublicKey = errors.New("crypto/dsa: invalid public key")

// ParameterSizes is an enumeration of the acceptable bit lengths of the primes
// in a set of DSA parameters. See FIPS 186-3, section 4.2.
type ParameterSizes int

const (
	L1024N160 ParameterSizes = iota
	L2048N224
	L2048N256
	L3072N256
)

// numMRTests is the number of Miller-Rabin primality tests that we perform. We
// pick the largest recommended number from table C.1 of FIPS 186-3.
const numMRTests = 64

// GenerateParameters puts a random, valid set of DSA parameters into params.
// This function can take many seconds, even on fast machines.
func GenerateParameters(params *Parameters, rand io.Reader, sizes ParameterSizes) error {
	// This function doesn't follow FIPS 186-3 exactly in that it doesn't
	// use a verification seed to generate the primes. The verification
	// seed doesn't appear to be exported or used by other code and
	// omitting it makes the code cleaner.

	var L, N int
	switch sizes {
	case L1024N160:
		L = 1024
		N = 160
	case L2048N224:
		L = 2048
		N = 224
	case L2048N256:
		L = 2048
		N = 256
	case L3072N256:
		L = 3072
		N = 256
	default:
		return errors.New("crypto/dsa: invalid ParameterSizes")
	}

	qBytes := make([]byte, N/8)
	pBytes := make([]byte, L/8)

	q := new(big.Int)
	p := new(big.Int)
	rem := new(big.Int)
	one := new(big.Int)
	one.SetInt64(1)

GeneratePrimes:
	for {
		if _, err := io.ReadFull(rand, qBytes); err != nil {
			return err
		}

		qBytes[len(qBytes)-1] |= 1
		qBytes[0] |= 0x80
		q.SetBytes(qBytes)

		if !q.ProbablyPrime(numMRTests) {
			continue
		}

		for i := 0; i < 4*L; i++ {
			if _, err := io.ReadFull(rand, pBytes); err != nil {
				return err
			}

			pBytes[len(pBytes)-1] |= 1
			pBytes[0] |= 0x80

			p.SetBytes(pBytes)
			rem.Mod(p, q)
			rem.Sub(rem, one)
			p.Sub(p, rem)
			if p.BitLen() < L {
				continue
			}

			if !p.ProbablyPrime(numMRTests) {
				continue
			}

			params.P = p
			params.Q = q
			break GeneratePrimes
		}
	}

	h := new(big.Int)
	h.SetInt64(2)
	g := new(big.Int)

	pm1 := new(big.Int).Sub(p, one)
	e := new(big.Int).Div(pm1, q)

	for {
		g.Exp(h, e, p)
		if g.Cmp(one) == 0 {
			h.Add(h, one)
			continue
		}

		params.G = g
		return nil
	}
}

// GenerateKey generates a public&private key pair. The Parameters of the
// PrivateKey must already be valid (see GenerateParameters).
func GenerateKey(priv *PrivateKey, rand io.Reader) error {
	if priv.P == nil || priv.Q == nil || priv.G == nil {
		return errors.New("crypto/dsa: parameters not set up before generating key")
	}

	x := new(big.Int)
	xBytes := make([]byte, priv.Q.BitLen()/8)

	for {
		_, err := io.ReadFull(rand, xBytes)
		if err != nil {
			return err
		}
		x.SetBytes(xBytes)
		if x.Sign() != 0 && x.Cmp(priv.Q) < 0 {
			break
		}
	}

	priv.X = x
	priv.Y = new(big.Int)
	priv.Y.Exp(priv.G, x, priv.P)
	return nil
}

// fermatInverse calculates the inverse of k in GF(P) using Fermat's method.
// This has better constant-time properties than Euclid's method (implemented
// in math/big.Int.ModInverse) although math/big itself isn't strictly
// constant-time so it's not perfect.
func fermatInverse(k, P *big.Int) *big.Int {
	two := big.NewInt(2)
	pMinus2 := new(big.Int).Sub(P, two)
	return new(big.Int).Exp(k, pMinus2, P)
}

// Sign signs an arbitrary length hash (which should be the result of hashing a
// larger message) using the private key, priv. It returns the signature as a
// pair of integers. The security of the private key depends on the entropy of
// rand.
//
// Note that FIPS 186-3 section 4.6 specifies that the hash should be truncated
// to the byte-length of the subgroup. This function does not perform that
// truncation itself.
//
// Be aware that calling Sign with an attacker-controlled PrivateKey may
// require an arbitrary amount of CPU.
func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err error) {
	randutil.MaybeReadByte(rand)

	// FIPS 186-3, section 4.6

	n := priv.Q.BitLen()
	if priv.Q.Sign() <= 0 || priv.P.Sign() <= 0 || priv.G.Sign() <= 0 || priv.X.Sign() <= 0 || n%8 != 0 {
		err = ErrInvalidPublicKey
		return
	}
	n >>= 3

	var attempts int
	for attempts = 10; attempts > 0; attempts-- {
		k := new(big.Int)
		buf := make([]byte, n)
		for {
			_, err = io.ReadFull(rand, buf)
			if err != nil {
				return
			}
			k.SetBytes(buf)
			// priv.Q must be >= 128 because the test above
			// requires it to be > 0 and that
			//    ceil(log_2(Q)) mod 8 = 0
			// Thus this loop will quickly terminate.
			if k.Sign() > 0 && k.Cmp(priv.Q) < 0 {
				break
			}
		}

		kInv := fermatInverse(k, priv.Q)

		r = new(big.Int).Exp(priv.G, k, priv.P)
		r.Mod(r, priv.Q)

		if r.Sign() == 0 {
			continue
		}

		z := k.SetBytes(hash)

		s = new(big.Int).Mul(priv.X, r)
		s.Add(s, z)
		s.Mod(s, priv.Q)
		s.Mul(s, kInv)
		s.Mod(s, priv.Q)

		if s.Sign() != 0 {
			break
		}
	}

	// Only degenerate private keys will require more than a handful of
	// attempts.
	if attempts == 0 {
		return nil, nil, ErrInvalidPublicKey
	}

	return
}

// Verify verifies the signature in r, s of hash using the public key, pub. It
// reports whether the signature is valid.
//
// Note that FIPS 186-3 section 4.6 specifies that the hash should be truncated
// to the byte-length of the subgroup. This function does not perform that
// truncation itself.
func Verify(pub *PublicKey, hash []byte, r, s *big.Int) bool {
	// FIPS 186-3, section 4.7

	if pub.P.Sign() == 0 {
		return false
	}

	if r.Sign() < 1 || r.Cmp(pub.Q) >= 0 {
		return false
	}
	if s.Sign() < 1 || s.Cmp(pub.Q) >= 0 {
		return false
	}

	w := new(big.Int).ModInverse(s, pub.Q)

	n := pub.Q.BitLen()
	if n%8 != 0 {
		return false
	}
	z := new(big.Int).SetBytes(hash)

	u1 := new(big.Int).Mul(z, w)
	u1.Mod(u1, pub.Q)
	u2 := w.Mul(r, w)
	u2.Mod(u2, pub.Q)
	v := u1.Exp(pub.G, u1, pub.P)
	u2.Exp(pub.Y, u2, pub.P)
	v.Mul(v, u2)
	v.Mod(v, pub.P)
	v.Mod(v, pub.Q)

	return v.Cmp(r) == 0
}
<|endoftext|>


<commit_before>package main

import (
	"crawler/linkparser"
	"flag"
	"fmt"
	"net/http"
	"os"
	"net/url"
)

func usage() {
	fmt.Println("usage: crawler https://example.com")
	flag.PrintDefaults()
	os.Exit(5)
}

func main() {
	flag.Parse()

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}
	crawlStartUri, err := url.Parse(args[0])
	if err != nil {
		fmt.Println("Bad start url: ", crawlStartUri)
		fmt.Println(err.Error())
		os.Exit(7)
	}

	linkQueue := make(chan string)

	go func() { linkQueue <- crawlStartUri.String() }()

	for link := range linkQueue {
		found := crawlUri(link)
		found = linkparser.ProcessLinks(found, crawlStartUri.String())
		processNewLinks(found, linkQueue)

	}
}

func crawlUri(uri string) []string {
	fmt.Println("Crawling ",uri)
	response, err := http.Get(uri)
	if err != nil {
		fmt.Println("Failed to parse ", uri, err.Error())
		os.Exit(6)
	}
	defer response.Body.Close()

	return linkparser.Parse(response.Body)
}

func processNewLinks(links []string, queue chan string) {
	for _, link := range links {
		go func() { queue <- link }()
	}
}<commit_msg>restrict domain, track visited<commit_after>package main

import (
	"crawler/linkparser"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
)

var (
	startUrl *url.URL
	visited  map[string]bool
)

func usage() {
	fmt.Println("usage: crawler https://example.com")
	flag.PrintDefaults()
	os.Exit(5)
}

func main() {
	flag.Parse()

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}
	var err error
	startUrl, err = url.Parse(args[0])
	if err != nil {
		fmt.Println("Bad start url: ", startUrl)
		fmt.Println(err.Error())
		os.Exit(7)
	}

	visited = map[string]bool{
		startUrl.Path: true,
	}

	linkQueue := make(chan string)

	go func() { linkQueue <- startUrl.String() }()

	for link := range linkQueue {
		found := crawlUri(link)
		found = linkparser.ProcessLinks(found, startUrl.String())
		processNewLinks(found, linkQueue)

	}
}

func crawlUri(uri string) []string {
	fmt.Println("Crawling ", uri)
	response, err := http.Get(uri)
	if err != nil {
		fmt.Println("Failed to parse ", uri, err.Error())
		os.Exit(6)
	}
	defer response.Body.Close()

	return linkparser.Parse(response.Body)
}

func processNewLinks(links []string, queue chan string) {
	for _, link := range links {
		linkobj, err := url.Parse(link)
		if !visited[linkobj.Path] {
			if err == nil && linkobj.Host == startUrl.Host {
				go func() { queue <- linkobj.String() }()
			}
			visited[linkobj.Path] = true
		}
	}
}
<|endoftext|>


<commit_before>package img

import (
	"bytes"
	"fmt"
	"github.com/pkg/errors"
	"log"
	"os/exec"
	"strconv"
)

type ImageMagickProcessor struct {
	convertCmd  string
	identifyCmd string
}

type imageInfo struct {
	format  string
	quality int
}

var convertOpts = []string{
	"-filter", "Triangle",
	"-define", "filter:support=2",
	"-unsharp", "0.25x0.08+8.3+0.045",
	"-dither", "None",
	"-posterize", "136",
	"-define", "jpeg:fancy-upsampling=off",
	"-define", "png:compression-filter=5",
	"-define", "png:compression-level=9",
	"-define", "png:compression-strategy=0",
	"-interlace", "None",
	"-colorspace", "sRGB",
	"-sampling-factor", "4:2:0",
	"-strip",
	"+profile", "*",
}

var cutToFitOpts = []string{
	"-gravity", "center",
}

//Creates new imagemagick processor. im is a path to
//IM convert executable that must be provided.
//idi is a path to IM identify command.
func NewProcessor(im string, idi string) (*ImageMagickProcessor, error) {
	if len(im) == 0 {
		log.Fatal("Command convert should be set by -imConvert flag")
		return nil, errors.New("Path to imagemagick convert executable must be provided")
	}
	if len(idi) == 0 {
		log.Fatal("Command identify should be set by -imIdentify flag")
		return nil, errors.New("Path to imagemagick identify executable must be provided")
	}

	_, err := exec.LookPath(im)
	if err != nil {
		return nil, err
	}
	_, err = exec.LookPath(idi)
	if err != nil {
		return nil, err
	}

	return &ImageMagickProcessor{
		convertCmd:  im,
		identifyCmd: idi,
	}, nil
}

// Resize image to the given size preserving aspect ratio. No cropping applying.
func (p *ImageMagickProcessor) Resize(data []byte, size string) ([]byte, error) {
	args := make([]string, 0)
	args = append(args, "-") //Input
	args = append(args, "-resize", size)
	args = append(args, convertOpts...)
	args = append(args, "-") //Output

	return p.execImagemagick(bytes.NewReader(data), args)
}

// Resize input image to exact size with cropping everything that out of the bounds.
// Size must specified in format WIDTHxHEIGHT. Both dimensions must be included.
func (p *ImageMagickProcessor) FitToSize(data []byte, size string) ([]byte, error) {
	args := make([]string, 0)
	args = append(args, "-") //Input
	args = append(args, "-resize", size+"^")
	args = append(args, convertOpts...)
	args = append(args, cutToFitOpts...)
	args = append(args, "-extent", size)
	args = append(args, "-") //Output

	return p.execImagemagick(bytes.NewReader(data), args)
}

func (p *ImageMagickProcessor) Optimise(data []byte) ([]byte, error) {
	imgInfo, err := p.loadImageInfo(bytes.NewReader(data))
	if err != nil {
		return nil, err
	}
	output := "-"
	if imgInfo.format == "PNG" {
		output = "PNG8:-"
	}
	quality := 82
	if imgInfo.quality > 0 && imgInfo.quality < quality {
		quality = imgInfo.quality
	}

	args := make([]string, 0)
	args = append(args, "-")                               //Input
	args = append(args, "-quality", strconv.Itoa(quality)) //Input
	args = append(args, convertOpts...)
	args = append(args, output) //Output

	return p.execImagemagick(bytes.NewReader(data), args)
}

func (p *ImageMagickProcessor) execImagemagick(in *bytes.Reader, args []string) ([]byte, error) {
	var out, cmderr bytes.Buffer
	cmd := exec.Command(p.convertCmd)

	cmd.Args = append(cmd.Args, args...)

	cmd.Stdin = in
	cmd.Stdout = &out
	cmd.Stderr = &cmderr

	log.Printf("Running resize command, args '%v'\n", cmd.Args)
	err := cmd.Run()
	if err != nil {
		log.Printf("Error executing convert command: %s\n", err.Error())
		log.Printf("ERROR: %s\n", cmderr.String())
		return nil, err
	}

	return out.Bytes(), nil
}

func (p *ImageMagickProcessor) loadImageInfo(in *bytes.Reader) (*imageInfo, error) {
	var out, cmderr bytes.Buffer
	cmd := exec.Command(p.identifyCmd)
	cmd.Args = append(cmd.Args, "-format", "%m %Q", "-")

	cmd.Stdin = in
	cmd.Stdout = &out
	cmd.Stderr = &cmderr

	log.Printf("Running identify command, args '%v'\n", cmd.Args)
	err := cmd.Run()
	if err != nil {
		log.Printf("Error executing identify command: %s\n", err.Error())
		log.Printf("ERROR: %s\n", cmderr.String())
		return nil, err
	}

	imageInfo := &imageInfo{}
	fmt.Sscanf(out.String(), "%s %d", &imageInfo.format, &imageInfo.quality)

	return imageInfo, nil
}
<commit_msg>* Returns original image from optimize if size of optimised image is bigger<commit_after>package img

import (
	"bytes"
	"fmt"
	"github.com/pkg/errors"
	"log"
	"os/exec"
	"strconv"
)

type ImageMagickProcessor struct {
	convertCmd  string
	identifyCmd string
}

type imageInfo struct {
	format  string
	quality int
}

var convertOpts = []string{
	"-filter", "Triangle",
	"-define", "filter:support=2",
	"-unsharp", "0.25x0.08+8.3+0.045",
	"-dither", "None",
	"-posterize", "136",
	"-define", "jpeg:fancy-upsampling=off",
	"-define", "png:compression-filter=5",
	"-define", "png:compression-level=9",
	"-define", "png:compression-strategy=0",
	"-interlace", "None",
	"-colorspace", "sRGB",
	"-sampling-factor", "4:2:0",
	"-strip",
	"+profile", "*",
}

var cutToFitOpts = []string{
	"-gravity", "center",
}

//Creates new imagemagick processor. im is a path to
//IM convert executable that must be provided.
//idi is a path to IM identify command.
func NewProcessor(im string, idi string) (*ImageMagickProcessor, error) {
	if len(im) == 0 {
		log.Fatal("Command convert should be set by -imConvert flag")
		return nil, errors.New("Path to imagemagick convert executable must be provided")
	}
	if len(idi) == 0 {
		log.Fatal("Command identify should be set by -imIdentify flag")
		return nil, errors.New("Path to imagemagick identify executable must be provided")
	}

	_, err := exec.LookPath(im)
	if err != nil {
		return nil, err
	}
	_, err = exec.LookPath(idi)
	if err != nil {
		return nil, err
	}

	return &ImageMagickProcessor{
		convertCmd:  im,
		identifyCmd: idi,
	}, nil
}

// Resize image to the given size preserving aspect ratio. No cropping applying.
func (p *ImageMagickProcessor) Resize(data []byte, size string) ([]byte, error) {
	args := make([]string, 0)
	args = append(args, "-") //Input
	args = append(args, "-resize", size)
	args = append(args, convertOpts...)
	args = append(args, "-") //Output

	return p.execImagemagick(bytes.NewReader(data), args)
}

// Resize input image to exact size with cropping everything that out of the bounds.
// Size must specified in format WIDTHxHEIGHT. Both dimensions must be included.
func (p *ImageMagickProcessor) FitToSize(data []byte, size string) ([]byte, error) {
	args := make([]string, 0)
	args = append(args, "-") //Input
	args = append(args, "-resize", size+"^")
	args = append(args, convertOpts...)
	args = append(args, cutToFitOpts...)
	args = append(args, "-extent", size)
	args = append(args, "-") //Output

	return p.execImagemagick(bytes.NewReader(data), args)
}

func (p *ImageMagickProcessor) Optimise(data []byte) ([]byte, error) {
	imgInfo, err := p.loadImageInfo(bytes.NewReader(data))
	if err != nil {
		return nil, err
	}
	output := "-"
	if imgInfo.format == "PNG" {
		output = "PNG8:-"
	}
	quality := 82
	if imgInfo.quality > 0 && imgInfo.quality < quality {
		quality = imgInfo.quality
	}

	args := make([]string, 0)
	args = append(args, "-")                               //Input
	args = append(args, "-quality", strconv.Itoa(quality)) //Input
	args = append(args, convertOpts...)
	args = append(args, output) //Output

	result, err := p.execImagemagick(bytes.NewReader(data), args)
	if err != nil {
		return nil, err
	}

	if len(result) > len(data) {
		log.Printf("WARNING: Optimised size [%d] is more than original [%d], fallback to original", len(result), len(data))
		result = data
	}

	return result, nil
}

func (p *ImageMagickProcessor) execImagemagick(in *bytes.Reader, args []string) ([]byte, error) {
	var out, cmderr bytes.Buffer
	cmd := exec.Command(p.convertCmd)

	cmd.Args = append(cmd.Args, args...)

	cmd.Stdin = in
	cmd.Stdout = &out
	cmd.Stderr = &cmderr

	log.Printf("Running resize command, args '%v'\n", cmd.Args)
	err := cmd.Run()
	if err != nil {
		log.Printf("Error executing convert command: %s\n", err.Error())
		log.Printf("ERROR: %s\n", cmderr.String())
		return nil, err
	}

	return out.Bytes(), nil
}

func (p *ImageMagickProcessor) loadImageInfo(in *bytes.Reader) (*imageInfo, error) {
	var out, cmderr bytes.Buffer
	cmd := exec.Command(p.identifyCmd)
	cmd.Args = append(cmd.Args, "-format", "%m %Q", "-")

	cmd.Stdin = in
	cmd.Stdout = &out
	cmd.Stderr = &cmderr

	log.Printf("Running identify command, args '%v'\n", cmd.Args)
	err := cmd.Run()
	if err != nil {
		log.Printf("Error executing identify command: %s\n", err.Error())
		log.Printf("ERROR: %s\n", cmderr.String())
		return nil, err
	}

	imageInfo := &imageInfo{}
	fmt.Sscanf(out.String(), "%s %d", &imageInfo.format, &imageInfo.quality)

	return imageInfo, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"encoding/json"
	"strconv"
)

type BaseResponse struct {
	Ok      bool        `json:"ok"`
	Index   string      `json:"_index,omitempty"`
	Type    string      `json:"_type,omitempty"`
	Id      string      `json:"_id,omitempty"`
	Source  interface{} `json:"_source,omitempty"` // depends on the schema you've defined
	Version int         `json:"_version,omitempty"`
	Found   bool        `json:"found,omitempty"`
	Exists  bool        `json:"exists,omitempty"`
	Matches []string    `json:"matches,omitempty"` // percolate matches
}

// StatusInt is required because /_optimize, at least, returns its status as
// strings instead of integers.
type StatusInt int

func (self *StatusInt) UnmarshalJSON(b []byte) error {
	s := ""
	if json.Unmarshal(b, &s) == nil {
		if i, err := strconv.Atoi(s); err == nil {
			*self = StatusInt(i)
		}
	}
	i := 0
	err := json.Unmarshal(b, &i)
	if err == nil {
		*self = StatusInt(i)
	}
	return err
}

func (self *StatusInt) MarshalJSON() ([]byte, error) {
	return json.Marshal(*self)
}

type Status struct {
	Total      StatusInt `json:"total"`
	Successful StatusInt `json:"successful"`
	Failed     StatusInt `json:"failed"`
}

type ExtendedStatus struct {
	Ok           bool   `json:"ok"`
	ShardsStatus Status `json:"_shards"`
}

type Match struct {
	OK          bool         `json:"ok"`
	Matches     []string     `json:"matches"`
	Explanation *Explanation `json:"explanation,omitempty"`
}

type Explanation struct {
	Value       float32        `json:"value"`
	Description string         `json:"description"`
	Details     []*Explanation `json:"details,omitempty"`
}

func ScrollDuration(duration string) string {
	scrollString := ""
	if duration != "" {
		scrollString = "&scroll=" + duration
	}
	return scrollString
}

// http://www.elasticsearch.org/guide/reference/api/search/search-type/
<commit_msg>fix UnmarshalJSON first case to return<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"encoding/json"
	"strconv"
)

type BaseResponse struct {
	Ok      bool        `json:"ok"`
	Index   string      `json:"_index,omitempty"`
	Type    string      `json:"_type,omitempty"`
	Id      string      `json:"_id,omitempty"`
	Source  interface{} `json:"_source,omitempty"` // depends on the schema you've defined
	Version int         `json:"_version,omitempty"`
	Found   bool        `json:"found,omitempty"`
	Exists  bool        `json:"exists,omitempty"`
	Matches []string    `json:"matches,omitempty"` // percolate matches
}

// StatusInt is required because /_optimize, at least, returns its status as
// strings instead of integers.
type StatusInt int

func (self *StatusInt) UnmarshalJSON(b []byte) error {
	s := ""
	if json.Unmarshal(b, &s) == nil {
		if i, err := strconv.Atoi(s); err == nil {
			*self = StatusInt(i)
			return nil
		}
	}
	i := 0
	err := json.Unmarshal(b, &i)
	if err == nil {
		*self = StatusInt(i)
	}
	return err
}

func (self *StatusInt) MarshalJSON() ([]byte, error) {
	return json.Marshal(*self)
}

type Status struct {
	Total      StatusInt `json:"total"`
	Successful StatusInt `json:"successful"`
	Failed     StatusInt `json:"failed"`
}

type ExtendedStatus struct {
	Ok           bool   `json:"ok"`
	ShardsStatus Status `json:"_shards"`
}

type Match struct {
	OK          bool         `json:"ok"`
	Matches     []string     `json:"matches"`
	Explanation *Explanation `json:"explanation,omitempty"`
}

type Explanation struct {
	Value       float32        `json:"value"`
	Description string         `json:"description"`
	Details     []*Explanation `json:"details,omitempty"`
}

func ScrollDuration(duration string) string {
	scrollString := ""
	if duration != "" {
		scrollString = "&scroll=" + duration
	}
	return scrollString
}

// http://www.elasticsearch.org/guide/reference/api/search/search-type/
<|endoftext|>


<commit_before>package node

import (
	"github.com/henrylee2cn/pholcus/node/crawlpool"
	"github.com/henrylee2cn/pholcus/node/spiderqueue"
	"github.com/henrylee2cn/pholcus/node/task"
	"github.com/henrylee2cn/pholcus/runtime/cache"
	"github.com/henrylee2cn/pholcus/runtime/status"
	"github.com/henrylee2cn/teleport"
	"log"
	"strconv"
	"time"
)

type Node struct {
	// 运行模式
	RunMode int
	// 服务器端口号
	Port string
	// 服务器地址（不含Port）
	Master string
	// socket长连接双工通信接口，json数据传输
	teleport.Teleport
	// 节点间传递的任务的存储库
	*task.TaskJar
	// 当前任务的蜘蛛队列
	Spiders spiderqueue.SpiderQueue
	// 爬行动作的回收池
	Crawls crawlpool.CrawlPool
	// 节点状态
	Status int
}

func newPholcus() *Node {
	return &Node{
		RunMode:  cache.Task.RunMode,
		Port:     ":" + strconv.Itoa(cache.Task.Port),
		Master:   cache.Task.Master,
		Teleport: teleport.New(),
		TaskJar:  task.NewTaskJar(),
		Spiders:  spiderqueue.New(),
		Crawls:   crawlpool.New(),
		Status:   status.RUN,
	}
}

// 声明实例
var Pholcus *Node = nil

// 运行节点
func PholcusRun() {
	if Pholcus != nil {
		return
	}
	Pholcus = newPholcus()
	switch Pholcus.RunMode {
	case status.SERVER:
		if Pholcus.checkPort() {
			log.Printf("                                                                                ！！当前运行模式为：[ 服务器 ] 模式！！")
			Pholcus.Teleport.SetAPI(ServerApi).SetUID("服务端").Server(Pholcus.Port)
		}

	case status.CLIENT:
		if Pholcus.checkAll() {
			log.Printf("                                                                                ！！当前运行模式为：[ 客户端 ] 模式！！")
			Pholcus.Teleport.SetAPI(ClientApi).Client(Pholcus.Master, Pholcus.Port)
		}
	// case status.OFFLINE:
	// 	fallthrough
	default:
		log.Printf("                                                                                ！！当前运行模式为：[ 单机 ] 模式！！")
		return
	}
	// 开启实时log发送
	go Pholcus.log()
}

// 返回节点数
func (self *Node) CountNodes() int {
	return self.Teleport.CountNodes()
}

// 生成task并添加至库，服务器模式专用
func (self *Node) AddNewTask() (tasksNum, spidersNum int) {
	length := self.Spiders.Len()
	t := task.Task{}

	// 从配置读取字段
	t.ThreadNum = cache.Task.ThreadNum
	t.BaseSleeptime = cache.Task.BaseSleeptime
	t.RandomSleepPeriod = cache.Task.RandomSleepPeriod
	t.OutType = cache.Task.OutType
	t.DockerCap = cache.Task.DockerCap
	t.DockerQueueCap = cache.Task.DockerQueueCap
	t.MaxPage = cache.Task.MaxPage

	for i, sp := range self.Spiders.GetAll() {

		t.Spiders = append(t.Spiders, map[string]string{"name": sp.GetName(), "keyword": sp.GetKeyword()})
		spidersNum++

		// 每十个蜘蛛存为一个任务
		if i > 0 && i%10 == 0 && length > 10 {
			// 存入
			one := t
			self.TaskJar.Push(&one)
			// log.Printf(" *     [新增任务]   详情： %#v", *t)

			tasksNum++

			// 清空spider
			t.Spiders = []map[string]string{}
		}
	}

	if len(t.Spiders) != 0 {
		// 存入
		one := t
		self.TaskJar.Push(&one)
		// log.Printf(" *     [新增任务]   详情： %#v", *t)
		tasksNum++
	}
	return
}

// 客户端请求获取任务
func (self *Node) GetTaskAlways() {
	self.Request(nil, "task")
}

// 客户端模式模式下获取任务
func (self *Node) DownTask() *task.Task {
ReStartLabel:
	for self.CountNodes() == 0 {
		if len(self.TaskJar.Tasks) != 0 {
			break
		}
		time.Sleep(5e7)
	}

	if len(self.TaskJar.Tasks) == 0 {
		self.GetTaskAlways()
		for len(self.TaskJar.Tasks) == 0 {
			if self.CountNodes() == 0 {
				goto ReStartLabel
			}
			time.Sleep(5e7)
		}
	}
	return self.TaskJar.Pull()
}

func (self *Node) log() {
	for {
		self.Teleport.Request(<-cache.SendChan, "log")
	}
}

func (self *Node) checkPort() bool {
	if cache.Task.Port == 0 {
		log.Println(" *     —— 亲，分布式端口不能为空哦~")
		return false
	}
	return true
}

func (self *Node) checkAll() bool {
	if cache.Task.Master == "" || !self.checkPort() {
		log.Println(" *     —— 亲，服务器地址不能为空哦~")
		return false
	}
	return true
}
<commit_msg>优化socket通信<commit_after>package node

import (
	"github.com/henrylee2cn/pholcus/node/crawlpool"
	"github.com/henrylee2cn/pholcus/node/spiderqueue"
	"github.com/henrylee2cn/pholcus/node/task"
	"github.com/henrylee2cn/pholcus/runtime/cache"
	"github.com/henrylee2cn/pholcus/runtime/status"
	"github.com/henrylee2cn/teleport"
	"log"
	"strconv"
	"time"
)

type Node struct {
	// 运行模式
	RunMode int
	// 服务器端口号
	Port string
	// 服务器地址（不含Port）
	Master string
	// socket长连接双工通信接口，json数据传输
	teleport.Teleport
	// 节点间传递的任务的存储库
	*task.TaskJar
	// 当前任务的蜘蛛队列
	Spiders spiderqueue.SpiderQueue
	// 爬行动作的回收池
	Crawls crawlpool.CrawlPool
	// 节点状态
	Status int
}

func newPholcus() *Node {
	return &Node{
		RunMode:  cache.Task.RunMode,
		Port:     ":" + strconv.Itoa(cache.Task.Port),
		Master:   cache.Task.Master,
		Teleport: teleport.New(),
		TaskJar:  task.NewTaskJar(),
		Spiders:  spiderqueue.New(),
		Crawls:   crawlpool.New(),
		Status:   status.RUN,
	}
}

// 声明实例
var Pholcus *Node = nil

// 运行节点
func PholcusRun() {
	if Pholcus != nil {
		return
	}
	Pholcus = newPholcus()
	switch Pholcus.RunMode {
	case status.SERVER:
		if Pholcus.checkPort() {
			log.Printf("                                                                                ！！当前运行模式为：[ 服务器 ] 模式！！")
			Pholcus.Teleport.SetAPI(ServerApi).SetUID("服务端").Server(Pholcus.Port)
		}

	case status.CLIENT:
		if Pholcus.checkAll() {
			log.Printf("                                                                                ！！当前运行模式为：[ 客户端 ] 模式！！")
			Pholcus.Teleport.SetAPI(ClientApi).Client(Pholcus.Master, Pholcus.Port)
		}
	// case status.OFFLINE:
	// 	fallthrough
	default:
		log.Printf("                                                                                ！！当前运行模式为：[ 单机 ] 模式！！")
		return
	}
	// 开启实时log发送
	go Pholcus.log()
}

// 返回节点数
func (self *Node) CountNodes() int {
	return self.Teleport.CountNodes()
}

// 生成task并添加至库，服务器模式专用
func (self *Node) AddNewTask() (tasksNum, spidersNum int) {
	length := self.Spiders.Len()
	t := task.Task{}

	// 从配置读取字段
	t.ThreadNum = cache.Task.ThreadNum
	t.BaseSleeptime = cache.Task.BaseSleeptime
	t.RandomSleepPeriod = cache.Task.RandomSleepPeriod
	t.OutType = cache.Task.OutType
	t.DockerCap = cache.Task.DockerCap
	t.DockerQueueCap = cache.Task.DockerQueueCap
	t.MaxPage = cache.Task.MaxPage

	for i, sp := range self.Spiders.GetAll() {

		t.Spiders = append(t.Spiders, map[string]string{"name": sp.GetName(), "keyword": sp.GetKeyword()})
		spidersNum++

		// 每十个蜘蛛存为一个任务
		if i > 0 && i%10 == 0 && length > 10 {
			// 存入
			one := t
			self.TaskJar.Push(&one)
			// log.Printf(" *     [新增任务]   详情： %#v", *t)

			tasksNum++

			// 清空spider
			t.Spiders = []map[string]string{}
		}
	}

	if len(t.Spiders) != 0 {
		// 存入
		one := t
		self.TaskJar.Push(&one)
		// log.Printf(" *     [新增任务]   详情： %#v", *t)
		tasksNum++
	}
	return
}

// 客户端请求获取任务
func (self *Node) GetTaskAlways() {
	self.Request(nil, "task")
}

// 客户端模式下获取任务
func (self *Node) DownTask() *task.Task {
ReStartLabel:
	for self.CountNodes() == 0 {
		if len(self.TaskJar.Tasks) != 0 {
			break
		}
		time.Sleep(5e7)
	}

	if len(self.TaskJar.Tasks) == 0 {
		self.GetTaskAlways()
		for len(self.TaskJar.Tasks) == 0 {
			if self.CountNodes() == 0 {
				goto ReStartLabel
			}
			time.Sleep(5e7)
		}
	}
	return self.TaskJar.Pull()
}

func (self *Node) log() {
	for {
		self.Teleport.Request(<-cache.SendChan, "log")
	}
}

func (self *Node) checkPort() bool {
	if cache.Task.Port == 0 {
		log.Println(" *     —— 亲，分布式端口不能为空哦~")
		return false
	}
	return true
}

func (self *Node) checkAll() bool {
	if cache.Task.Master == "" || !self.checkPort() {
		log.Println(" *     —— 亲，服务器地址不能为空哦~")
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package statshub

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"strings"
	"time"
)

// StatsUpdate posts stats with zero, one or more dimensions.  Stats
// include Counter (cumulative) and Gauges (point in time)
// (online/offline).
type StatsUpdate struct {
	Dims map[string]string `json:"dims"`
	Stats
	conn redis.Conn
}

type statWriter struct {
	// statType: the type of stat handled by this writer (i.e. "counter" or "gauge")
	statType string

	// writeDetail: writes the detail entry for the given key
	writeDetail func(redisKey string, val interface{}) error

	// writeDim: writes the dimension entry for the given key
	writeDim func(redisKey string, val interface{}) error
}

// update posts Counter and Gauge for the given id to redis
// using INCRBY and SET respectively.
func (stats *StatsUpdate) write(id string) (err error) {
	// Always treat dimensions as lower case
	lowercasedDims := make(map[string]string)
	for name, key := range stats.Dims {
		dimName := strings.ToLower(name)
		dimKey := strings.ToLower(key)
		if dimKey == "total" {
			return fmt.Errorf("Dimension key 'total' is not allowed because it is a reserved word")
		}
		lowercasedDims[dimName] = dimKey
	}
	stats.Dims = lowercasedDims

	if stats.conn, err = connectToRedis(); err != nil {
		return
	}
	defer stats.conn.Close()

	if err = stats.writeToRedis("counter", id, stats.Counters); err != nil {
		return
	}
	if err = stats.writeToRedis("gauge", id, stats.Gauges); err != nil {
		return
	}
	if err = stats.writeIncrements(id); err != nil {
		return
	}
	if err = stats.writeMembers(id); err != nil {
		return
	}

	// Save dims
	for name, value := range stats.Dims {
		err = stats.conn.Send("SADD", "dim", name)
		err = stats.conn.Send("SADD", "dim:"+name, value)
	}
	err = stats.conn.Flush()

	return
}

// writeIncrements increments counters in redis
func (stats *StatsUpdate) writeIncrements(id string) (err error) {
	return stats.doWrite(id, stats.Increments, &statWriter{
		statType: "counter",
		writeDetail: func(redisKey string, val interface{}) error {
			return stats.conn.Send("INCRBY", redisKey, val)
		},
		writeDim: func(redisKey string, val interface{}) error {
			return stats.conn.Send("INCRBY", redisKey, val)
		},
	})
}

// writeMembers adds members to redis
func (stats *StatsUpdate) writeMembers(id string) (err error) {
	return stats.doWriteStrings(id, stats.Members, &statWriter{
		statType: "member",
		writeDetail: func(redisKey string, val interface{}) error {
			return stats.conn.Send("SADD", redisKey, val)
		},
		writeDim: func(redisKey string, val interface{}) error {
			return stats.conn.Send("SADD", redisKey, val)
		},
	})
}

func (stats *StatsUpdate) doWrite(
	id string,
	values map[string]int64,
	writer *statWriter) (err error) {

	keyArgs := make([]interface{}, len(values)+1)
	keyArgs[0] = fmt.Sprintf("key:%s", writer.statType)
	i := 1

	for key, val := range values {
		keyArgs[i] = key
		i++
		detailKey := redisKey(writer.statType, fmt.Sprintf("detail:%s", id), key)
		err = writer.writeDetail(detailKey, val)
		for dimName, dimValue := range stats.Dims {
			dimKey := redisKey(writer.statType, fmt.Sprintf("dim:%s:%s", dimName, dimValue), key)
			err = writer.writeDim(dimKey, val)
		}
	}

	// Remember keys
	err = stats.conn.Send("SADD", keyArgs...)
	return
}

func (stats *StatsUpdate) doWriteStrings(
	id string,
	values map[string]string,
	writer *statWriter) (err error) {

	keyArgs := make([]interface{}, len(values)+1)
	keyArgs[0] = fmt.Sprintf("key:%s", writer.statType)
	i := 1

	for key, val := range values {
		keyArgs[i] = key
		i++
		detailKey := redisKey(writer.statType, fmt.Sprintf("detail:%s", id), key)
		err = writer.writeDetail(detailKey, val)
		for dimName, dimValue := range stats.Dims {
			dimKey := redisKey(writer.statType, fmt.Sprintf("dim:%s:%s", dimName, dimValue), key)
			err = writer.writeDim(dimKey, val)
		}
	}

	// Remember keys
	err = stats.conn.Send("SADD", keyArgs...)
	return
}

// writeToRedis writes values (counters or gauges) to redis
func (stats *StatsUpdate) writeToRedis(
	statType string,
	id string,
	values map[string]int64) (err error) {

	now := time.Now()
	now = now.Truncate(statsPeriod)
	expiration := now.Add(3 * statsPeriod)

	keyArgs := make([]interface{}, len(values)+1)
	keyArgs[0] = "key:" + statType
	i := 1

	if err = stats.drainReceiveBuffer(); err != nil {
		return
	}

	isGauge := statType == "gauge"

	qualifiedKey := func(key string) string {
		if isGauge {
			// For gauges, qualify the key with a date
			return fmt.Sprintf("%s:%d", key, now.Unix())
		} else {
			return key
		}
	}

	// Set details
	for key, value := range values {
		keyArgs[i] = key
		i++
		redisKey := redisKey(statType, fmt.Sprintf("detail:%s", id), qualifiedKey(key))
		err = stats.conn.Send("GETSET", redisKey, value)
	}

	if isGauge {
		// The reason we don't do EXPIREAT in the above loop is that the code for
		// dimensional rollups needs to read the return values from GETSET,
		// and we don't want to bother with interleaving those with the EXPIREAT
		// return values.
		for key, _ := range values {
			redisKey := redisKey(statType, fmt.Sprintf("detail:%s", id), qualifiedKey(key))
			err = stats.conn.Send("EXPIREAT", redisKey, expiration.Unix())
		}
	}

	err = stats.conn.Flush()

	// Roll up to dimensions
	for key, value := range values {
		var oldValue int64
		if oldValue, _, err = receive(stats.conn); err != nil {
			return
		}
		delta := value - oldValue
		for dimName, dimValue := range stats.Dims {
			dimKey := redisKey(statType, fmt.Sprintf("dim:%s:%s", dimName, dimValue), qualifiedKey(key))
			err = stats.conn.Send("INCRBY", dimKey, delta)
			if isGauge {
				err = stats.conn.Send("EXPIREAT", dimKey, expiration.Unix())
			}
		}
	}

	// Remember keys
	err = stats.conn.Send("SADD", keyArgs...)
	return
}

// drainReceiveBuffer drains any responses in the receive buffer that haven't been read
func (stats *StatsUpdate) drainReceiveBuffer() (err error) {
	_, err = stats.conn.Do("")
	return
}

// withLowerCaseKeys converts the keys in a map to lower case, returning a new
// map with the lower cased keys.
func withLowerCaseKeys(values map[string]uint64) (lowerCased map[string]uint64) {
	lowerCased = make(map[string]uint64)
	for key, value := range values {
		lowerCased[strings.ToLower(key)] = value
	}
	return
}
<commit_msg>More DRY improvements<commit_after>package statshub

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"strings"
	"time"
)

// StatsUpdate posts stats with zero, one or more dimensions.  Stats
// include Counter (cumulative) and Gauges (point in time)
// (online/offline).
type StatsUpdate struct {
	Dims map[string]string `json:"dims"`
	Stats
	conn redis.Conn
}

type statWriter struct {
	// statType: the type of stat handled by this writer (i.e. "counter" or "gauge")
	statType string

	// writeDetail: writes the detail entry for the given key
	writeDetail func(redisKey string, val interface{}, expiration time.Time) error

	// writeDim: writes the dimension entry for the given key
	writeDim func(redisKey string, val interface{}, expiration time.Time) error
}

// update posts Counter and Gauge for the given id to redis
// using INCRBY and SET respectively.
func (stats *StatsUpdate) write(id string) (err error) {
	// Always treat dimensions as lower case
	lowercasedDims := make(map[string]string)
	for name, key := range stats.Dims {
		dimName := strings.ToLower(name)
		dimKey := strings.ToLower(key)
		if dimKey == "total" {
			return fmt.Errorf("Dimension key 'total' is not allowed because it is a reserved word")
		}
		lowercasedDims[dimName] = dimKey
	}
	stats.Dims = lowercasedDims

	if stats.conn, err = connectToRedis(); err != nil {
		return
	}
	defer stats.conn.Close()

	if err = stats.writeToRedis("counter", id, stats.Counters); err != nil {
		return
	}
	if err = stats.writeToRedis("gauge", id, stats.Gauges); err != nil {
		return
	}
	if err = stats.writeIncrements(id); err != nil {
		return
	}
	if err = stats.writeMembers(id); err != nil {
		return
	}

	// Save dims
	for name, value := range stats.Dims {
		err = stats.conn.Send("SADD", "dim", name)
		err = stats.conn.Send("SADD", "dim:"+name, value)
	}
	err = stats.conn.Flush()

	return
}

// writeIncrements increments counters in redis
func (stats *StatsUpdate) writeIncrements(id string) (err error) {
	return stats.doWriteInt(id, stats.Increments, &statWriter{
		statType: "counter",
		writeDetail: func(redisKey string, val interface{}, expiration time.Time) error {
			return stats.conn.Send("INCRBY", redisKey, val)
		},
		writeDim: func(redisKey string, val interface{}, expiration time.Time) error {
			return stats.conn.Send("INCRBY", redisKey, val)
		},
	})
}

// writeMembers adds members to redis
func (stats *StatsUpdate) writeMembers(id string) (err error) {
	return stats.doWriteString(id, stats.Members, &statWriter{
		statType: "member",
		writeDetail: func(redisKey string, val interface{}, expiration time.Time) error {
			return stats.conn.Send("SADD", redisKey, val)
		},
		writeDim: func(redisKey string, val interface{}, expiration time.Time) error {
			return stats.conn.Send("SADD", redisKey, val)
		},
	})
}

func (stats *StatsUpdate) doWriteInt(
	id string,
	values map[string]int64,
	writer *statWriter) (err error) {

	i := 1
	return stats.doWrite(
		len(values),
		writer,
		func(keyArgs []interface{}, expiration time.Time) error {
			for key, val := range values {
				keyArgs[i] = key
				i++
				detailKey := redisKey(writer.statType, fmt.Sprintf("detail:%s", id), key)
				err = writer.writeDetail(detailKey, val, expiration)
				for dimName, dimValue := range stats.Dims {
					dimKey := redisKey(writer.statType, fmt.Sprintf("dim:%s:%s", dimName, dimValue), key)
					err = writer.writeDim(dimKey, val, expiration)
				}
			}
			return nil
		})
}

func (stats *StatsUpdate) doWriteString(
	id string,
	values map[string]string,
	writer *statWriter) (err error) {

	i := 1
	return stats.doWrite(
		len(values),
		writer,
		func(keyArgs []interface{}, expiration time.Time) error {
			for key, val := range values {
				keyArgs[i] = key
				i++
				detailKey := redisKey(writer.statType, fmt.Sprintf("detail:%s", id), key)
				err = writer.writeDetail(detailKey, val, expiration)
				for dimName, dimValue := range stats.Dims {
					dimKey := redisKey(writer.statType, fmt.Sprintf("dim:%s:%s", dimName, dimValue), key)
					err = writer.writeDim(dimKey, val, expiration)
				}
			}
			return nil
		})
}

func (stats *StatsUpdate) doWrite(
	numValues int,
	writer *statWriter,
	writeIt func(keyArgs []interface{}, expiration time.Time) error) error {
	now := time.Now()
	now = now.Truncate(statsPeriod)
	expiration := now.Add(3 * statsPeriod)

	keyArgs := make([]interface{}, numValues+1)
	keyArgs[0] = fmt.Sprintf("key:%s", writer.statType)

	writeIt(keyArgs, expiration)

	// Remember keys
	return stats.conn.Send("SADD", keyArgs...)
}

// writeToRedis writes values (counters or gauges) to redis
func (stats *StatsUpdate) writeToRedis(
	statType string,
	id string,
	values map[string]int64) (err error) {

	now := time.Now()
	now = now.Truncate(statsPeriod)
	expiration := now.Add(3 * statsPeriod)

	keyArgs := make([]interface{}, len(values)+1)
	keyArgs[0] = "key:" + statType
	i := 1

	if err = stats.drainReceiveBuffer(); err != nil {
		return
	}

	isGauge := statType == "gauge"

	qualifiedKey := func(key string) string {
		if isGauge {
			// For gauges, qualify the key with a date
			return fmt.Sprintf("%s:%d", key, now.Unix())
		} else {
			return key
		}
	}

	// Set details
	for key, value := range values {
		keyArgs[i] = key
		i++
		redisKey := redisKey(statType, fmt.Sprintf("detail:%s", id), qualifiedKey(key))
		err = stats.conn.Send("GETSET", redisKey, value)
	}

	if isGauge {
		// The reason we don't do EXPIREAT in the above loop is that the code for
		// dimensional rollups needs to read the return values from GETSET,
		// and we don't want to bother with interleaving those with the EXPIREAT
		// return values.
		for key, _ := range values {
			redisKey := redisKey(statType, fmt.Sprintf("detail:%s", id), qualifiedKey(key))
			err = stats.conn.Send("EXPIREAT", redisKey, expiration.Unix())
		}
	}

	err = stats.conn.Flush()

	// Roll up to dimensions
	for key, value := range values {
		var oldValue int64
		if oldValue, _, err = receive(stats.conn); err != nil {
			return
		}
		delta := value - oldValue
		for dimName, dimValue := range stats.Dims {
			dimKey := redisKey(statType, fmt.Sprintf("dim:%s:%s", dimName, dimValue), qualifiedKey(key))
			err = stats.conn.Send("INCRBY", dimKey, delta)
			if isGauge {
				err = stats.conn.Send("EXPIREAT", dimKey, expiration.Unix())
			}
		}
	}

	// Remember keys
	err = stats.conn.Send("SADD", keyArgs...)
	return
}

// drainReceiveBuffer drains any responses in the receive buffer that haven't been read
func (stats *StatsUpdate) drainReceiveBuffer() (err error) {
	_, err = stats.conn.Do("")
	return
}

// withLowerCaseKeys converts the keys in a map to lower case, returning a new
// map with the lower cased keys.
func withLowerCaseKeys(values map[string]uint64) (lowerCased map[string]uint64) {
	lowerCased = make(map[string]uint64)
	for key, value := range values {
		lowerCased[strings.ToLower(key)] = value
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"time"

	"github.com/play-with-docker/play-with-docker/config"
	"github.com/play-with-docker/play-with-docker/docker"
	"github.com/play-with-docker/play-with-docker/event"
	"github.com/play-with-docker/play-with-docker/handlers"
	"github.com/play-with-docker/play-with-docker/id"
	"github.com/play-with-docker/play-with-docker/k8s"
	"github.com/play-with-docker/play-with-docker/provisioner"
	"github.com/play-with-docker/play-with-docker/pwd"
	"github.com/play-with-docker/play-with-docker/pwd/types"
	"github.com/play-with-docker/play-with-docker/scheduler"
	"github.com/play-with-docker/play-with-docker/scheduler/task"
	"github.com/play-with-docker/play-with-docker/storage"
)

func main() {
	config.ParseFlags()

	e := initEvent()
	s := initStorage()
	df := initDockerFactory(s)
	kf := initK8sFactory(s)

	ipf := provisioner.NewInstanceProvisionerFactory(provisioner.NewWindowsASG(df, s), provisioner.NewDinD(id.XIDGenerator{}, df, s))
	sp := provisioner.NewOverlaySessionProvisioner(df)

	core := pwd.NewPWD(df, e, s, sp, ipf)

	tasks := []scheduler.Task{
		task.NewCheckPorts(e, df),
		task.NewCheckSwarmPorts(e, df),
		task.NewCheckSwarmStatus(e, df),
		task.NewCollectStats(e, df, s),
		task.NewCheckK8sClusterStatus(e, kf),
		task.NewCheckK8sClusterExposedPorts(e, kf),
	}
	sch, err := scheduler.NewScheduler(tasks, s, e, core)
	if err != nil {
		log.Fatal("Error initializing the scheduler: ", err)
	}

	sch.Start()

	d, err := time.ParseDuration("4h")
	if err != nil {
		log.Fatalf("Cannot parse duration Got: %v", err)
	}

	playground := types.Playground{Domain: config.PlaygroundDomain, DefaultDinDInstanceImage: "franela/dind", AllowWindowsInstances: config.NoWindows, DefaultSessionDuration: d, Tasks: []string{".*"}, Extras: map[string]interface{}{"LoginRedirect": "http://localhost:3000"}}
	if _, err := core.PlaygroundNew(playground); err != nil {
		log.Fatalf("Cannot create default playground. Got: %v", err)
	}

	handlers.Bootstrap(core, e)
	handlers.Register(nil)
}

func initStorage() storage.StorageApi {
	s, err := storage.NewFileStorage(config.SessionsFile)
	if err != nil && !os.IsNotExist(err) {
		log.Fatal("Error initializing StorageAPI: ", err)
	}
	return s
}

func initEvent() event.EventApi {
	return event.NewLocalBroker()
}

func initDockerFactory(s storage.StorageApi) docker.FactoryApi {
	return docker.NewLocalCachedFactory(s)
}

func initK8sFactory(s storage.StorageApi) k8s.FactoryApi {
	return k8s.NewLocalCachedFactory(s)
}
<commit_msg>Remove incorrect parameters<commit_after>package main

import (
	"log"
	"os"
	"time"

	"github.com/play-with-docker/play-with-docker/config"
	"github.com/play-with-docker/play-with-docker/docker"
	"github.com/play-with-docker/play-with-docker/event"
	"github.com/play-with-docker/play-with-docker/handlers"
	"github.com/play-with-docker/play-with-docker/id"
	"github.com/play-with-docker/play-with-docker/k8s"
	"github.com/play-with-docker/play-with-docker/provisioner"
	"github.com/play-with-docker/play-with-docker/pwd"
	"github.com/play-with-docker/play-with-docker/pwd/types"
	"github.com/play-with-docker/play-with-docker/scheduler"
	"github.com/play-with-docker/play-with-docker/scheduler/task"
	"github.com/play-with-docker/play-with-docker/storage"
)

func main() {
	config.ParseFlags()

	e := initEvent()
	s := initStorage()
	df := initDockerFactory(s)
	kf := initK8sFactory(s)

	ipf := provisioner.NewInstanceProvisionerFactory(provisioner.NewWindowsASG(df, s), provisioner.NewDinD(id.XIDGenerator{}, df, s))
	sp := provisioner.NewOverlaySessionProvisioner(df)

	core := pwd.NewPWD(df, e, s, sp, ipf)

	tasks := []scheduler.Task{
		task.NewCheckPorts(e, df),
		task.NewCheckSwarmPorts(e, df),
		task.NewCheckSwarmStatus(e, df),
		task.NewCollectStats(e, df, s),
		task.NewCheckK8sClusterStatus(e, kf),
		task.NewCheckK8sClusterExposedPorts(e, kf),
	}
	sch, err := scheduler.NewScheduler(tasks, s, e, core)
	if err != nil {
		log.Fatal("Error initializing the scheduler: ", err)
	}

	sch.Start()

	d, err := time.ParseDuration("4h")
	if err != nil {
		log.Fatalf("Cannot parse duration Got: %v", err)
	}

	playground := types.Playground{Domain: config.PlaygroundDomain, DefaultDinDInstanceImage: "franela/dind", AvailableDinDInstanceImages: []string{"franela/dind"}, AllowWindowsInstances: config.NoWindows, DefaultSessionDuration: d, Tasks: []string{".*"}, Extras: map[string]interface{}{"LoginRedirect": "http://localhost:3000"}}
	if _, err := core.PlaygroundNew(playground); err != nil {
		log.Fatalf("Cannot create default playground. Got: %v", err)
	}

	handlers.Bootstrap(core, e)
	handlers.Register(nil)
}

func initStorage() storage.StorageApi {
	s, err := storage.NewFileStorage(config.SessionsFile)
	if err != nil && !os.IsNotExist(err) {
		log.Fatal("Error initializing StorageAPI: ", err)
	}
	return s
}

func initEvent() event.EventApi {
	return event.NewLocalBroker()
}

func initDockerFactory(s storage.StorageApi) docker.FactoryApi {
	return docker.NewLocalCachedFactory(s)
}

func initK8sFactory(s storage.StorageApi) k8s.FactoryApi {
	return k8s.NewLocalCachedFactory(s)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"exec"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"unicode"
	"utf8"
)

// Environment for commands.
var (
	XGC       []string // 6g -I _test -o _xtest_.6
	GC        []string // 6g -I _test _testmain.go
	GL        []string // 6l -L _test _testmain.6
	GOARCH    string
	GOROOT    string
	GORUN     string
	O         string
	args      []string // arguments passed to gotest; also passed to the binary
	fileNames []string
	env       = os.Environ()
)

// These strings are created by getTestNames.
var (
	insideFileNames  []string // list of *.go files inside the package.
	outsideFileNames []string // list of *.go files outside the package (in package foo_test).
)

var (
	files      []*File
	importPath string
)

// Flags for our own purposes. We do our own flag processing.
var (
	cFlag bool
	xFlag bool
)

// File represents a file that contains tests.
type File struct {
	name       string
	pkg        string
	file       *os.File
	astFile    *ast.File
	tests      []string // The names of the TestXXXs.
	benchmarks []string // The names of the BenchmarkXXXs.
}

func main() {
	flags()
	needMakefile()
	setEnvironment()
	getTestFileNames()
	parseFiles()
	getTestNames()
	run("gomake", "testpackage-clean")
	run("gomake", "testpackage", fmt.Sprintf("GOTESTFILES=%s", strings.Join(insideFileNames, " ")))
	if len(outsideFileNames) > 0 {
		run(append(XGC, outsideFileNames...)...)
	}
	importPath = runWithStdout("gomake", "-s", "importpath")
	writeTestmainGo()
	run(GC...)
	run(GL...)
	if !cFlag {
		runTestWithArgs("./" + O + ".out")
	}
}

// needMakefile tests that we have a Makefile in this directory.
func needMakefile() {
	if _, err := os.Stat("Makefile"); err != nil {
		Fatalf("please create a Makefile for gotest; see http://golang.org/doc/code.html for details")
	}
}

// Fatalf formats its arguments, prints the message with a final newline, and exits.
func Fatalf(s string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "gotest: "+s+"\n", args...)
	os.Exit(2)
}

// theChar is the map from architecture to object character.
var theChar = map[string]string{
	"arm":   "5",
	"amd64": "6",
	"386":   "8",
}

// addEnv adds a name=value pair to the environment passed to subcommands.
// If the item is already in the environment, addEnv replaces the value.
func addEnv(name, value string) {
	for i := 0; i < len(env); i++ {
		if strings.HasPrefix(env[i], name+"=") {
			env[i] = name + "=" + value
			return
		}
	}
	env = append(env, name+"="+value)
}

// setEnvironment assembles the configuration for gotest and its subcommands.
func setEnvironment() {
	// Basic environment.
	GOROOT = runtime.GOROOT()
	addEnv("GOROOT", GOROOT)
	GOARCH = runtime.GOARCH
	addEnv("GOARCH", GOARCH)
	O = theChar[GOARCH]
	if O == "" {
		Fatalf("unknown architecture %s", GOARCH)
	}

	// Commands and their flags.
	gc := os.Getenv("GC")
	if gc == "" {
		gc = O + "g"
	}
	XGC = []string{gc, "-I", "_test", "-o", "_xtest_." + O}
	GC = []string{gc, "-I", "_test", "_testmain.go"}
	gl := os.Getenv("GL")
	if gl == "" {
		gl = O + "l"
	}
	GL = []string{gl, "-L", "_test", "_testmain." + O}

	// Silence make on Linux
	addEnv("MAKEFLAGS", "")
	addEnv("MAKELEVEL", "")
}

// getTestFileNames gets the set of files we're looking at.
// If gotest has no arguments, it scans the current directory for _test.go files.
func getTestFileNames() {
	names := fileNames
	if len(names) == 0 {
		names = filepath.Glob("[^.]*_test.go")
		if len(names) == 0 {
			Fatalf(`no test files found: no match for "*_test.go"`)
		}
	}
	for _, n := range names {
		fd, err := os.Open(n, os.O_RDONLY, 0)
		if err != nil {
			Fatalf("%s: %s", n, err)
		}
		f := &File{name: n, file: fd}
		files = append(files, f)
	}
}

// parseFiles parses the files and remembers the packages we find.
func parseFiles() {
	fileSet := token.NewFileSet()
	for _, f := range files {
		// Report declaration errors so we can abort if the files are incorrect Go.
		file, err := parser.ParseFile(fileSet, f.name, nil, parser.DeclarationErrors)
		if err != nil {
			Fatalf("parse error: %s", err)
		}
		f.astFile = file
		f.pkg = file.Name.String()
		if f.pkg == "" {
			Fatalf("cannot happen: no package name in %s", f.name)
		}
	}
}

// getTestNames extracts the names of tests and benchmarks.  They are all
// top-level functions that are not methods.
func getTestNames() {
	for _, f := range files {
		for _, d := range f.astFile.Decls {
			n, ok := d.(*ast.FuncDecl)
			if !ok {
				continue
			}
			if n.Recv != nil { // a method, not a function.
				continue
			}
			name := n.Name.String()
			if isTest(name, "Test") {
				f.tests = append(f.tests, name)
			} else if isTest(name, "Benchmark") {
				f.benchmarks = append(f.benchmarks, name)
			}
			// TODO: worth checking the signature? Probably not.
		}
		if strings.HasSuffix(f.pkg, "_test") {
			outsideFileNames = append(outsideFileNames, f.name)
		} else {
			insideFileNames = append(insideFileNames, f.name)
		}
	}
}

// isTest tells whether name looks like a test (or benchmark, according to prefix).
// It is a Test (say) if there is a character after Test that is not a lower-case letter.
// We don't want TesticularCancer.
func isTest(name, prefix string) bool {
	if !strings.HasPrefix(name, prefix) {
		return false
	}
	if len(name) == len(prefix) { // "Test" is ok
		return true
	}
	rune, _ := utf8.DecodeRuneInString(name[len(prefix):])
	return !unicode.IsLower(rune)
}

func run(args ...string) {
	doRun(args, false)
}

// runWithStdout is like run, but returns the text of standard output with the last newline dropped.
func runWithStdout(argv ...string) string {
	s := doRun(argv, true)
	if len(s) == 0 {
		Fatalf("no output from command %s", strings.Join(argv, " "))
	}
	if s[len(s)-1] == '\n' {
		s = s[:len(s)-1]
	}
	return s
}

// runTestWithArgs appends gotest's runs the provided binary with the args passed on the command line.
func runTestWithArgs(binary string) {
	doRun(append([]string{binary}, args...), false)
}

// doRun is the general command runner.  The flag says whether we want to
// retrieve standard output.
func doRun(argv []string, returnStdout bool) string {
	if xFlag {
		fmt.Printf("gotest: %s\n", strings.Join(argv, " "))
	}
	var err os.Error
	argv[0], err = exec.LookPath(argv[0])
	if err != nil {
		Fatalf("can't find %s: %s", argv[0], err)
	}
	procAttr := &os.ProcAttr{
		Env: env,
		Files: []*os.File{
			os.Stdin,
			os.Stdout,
			os.Stderr,
		},
	}
	var r, w *os.File
	if returnStdout {
		r, w, err = os.Pipe()
		if err != nil {
			Fatalf("can't create pipe: %s", err)
		}
		procAttr.Files[1] = w
	}
	proc, err := os.StartProcess(argv[0], argv, procAttr)
	if err != nil {
		Fatalf("make failed to start: %s", err)
	}
	if returnStdout {
		defer r.Close()
		w.Close()
	}
	waitMsg, err := proc.Wait(0)
	if err != nil || waitMsg == nil {
		Fatalf("%s failed: %s", argv[0], err)
	}
	if !waitMsg.Exited() || waitMsg.ExitStatus() != 0 {
		Fatalf("%q failed: %s", strings.Join(argv, " "), waitMsg)
	}
	if returnStdout {
		b, err := ioutil.ReadAll(r)
		if err != nil {
			Fatalf("can't read output from command: %s", err)
		}
		return string(b)
	}
	return ""
}

// writeTestmainGo generates the test program to be compiled, "./_testmain.go".
func writeTestmainGo() {
	f, err := os.Open("_testmain.go", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		Fatalf("can't create _testmain.go: %s", err)
	}
	defer f.Close()
	b := bufio.NewWriter(f)
	defer b.Flush()

	// Package and imports.
	fmt.Fprint(b, "package main\n\n")
	// Are there tests from a package other than the one we're testing?
	// We can't just use file names because some of the things we compiled
	// contain no tests.
	outsideTests := false
	insideTests := false
	for _, f := range files {
		//println(f.name, f.pkg)
		if len(f.tests) == 0 && len(f.benchmarks) == 0 {
			continue
		}
		if strings.HasSuffix(f.pkg, "_test") {
			outsideTests = true
		} else {
			insideTests = true
		}
	}
	if insideTests {
		switch importPath {
		case "testing":
		case "main":
			// Import path main is reserved, so import with
			// explicit reference to ./_test/main instead.
			// Also, the file we are writing defines a function named main,
			// so rename this import to __main__ to avoid name conflict.
			fmt.Fprintf(b, "import __main__ %q\n", "./_test/main")
		default:
			fmt.Fprintf(b, "import %q\n", importPath)
		}
	}
	if outsideTests {
		fmt.Fprintf(b, "import %q\n", "./_xtest_")
	}
	fmt.Fprintf(b, "import %q\n", "testing")
	fmt.Fprintf(b, "import __os__     %q\n", "os")     // rename in case tested package is called os
	fmt.Fprintf(b, "import __regexp__ %q\n", "regexp") // rename in case tested package is called regexp
	fmt.Fprintln(b)                                    // for gofmt

	// Tests.
	fmt.Fprintln(b, "var tests = []testing.InternalTest{")
	for _, f := range files {
		for _, t := range f.tests {
			fmt.Fprintf(b, "\t{\"%s.%s\", %s.%s},\n", f.pkg, t, notMain(f.pkg), t)
		}
	}
	fmt.Fprintln(b, "}")
	fmt.Fprintln(b)

	// Benchmarks.
	fmt.Fprintln(b, "var benchmarks = []testing.InternalBenchmark{")
	for _, f := range files {
		for _, bm := range f.benchmarks {
			fmt.Fprintf(b, "\t{\"%s.%s\", %s.%s},\n", f.pkg, bm, notMain(f.pkg), bm)
		}
	}
	fmt.Fprintln(b, "}")

	// Body.
	fmt.Fprintln(b, testBody)
}

// notMain returns the package, renaming as appropriate if it's "main".
func notMain(pkg string) string {
	if pkg == "main" {
		return "__main__"
	}
	return pkg
}

// testBody is just copied to the output. It's the code that runs the tests.
var testBody = `
var matchPat string
var matchRe *__regexp__.Regexp

func matchString(pat, str string) (result bool, err __os__.Error) {
	if matchRe == nil || matchPat != pat {
		matchPat = pat
		matchRe, err = __regexp__.Compile(matchPat)
		if err != nil {
			return
		}
	}
	return matchRe.MatchString(str), nil
}

func main() {
	testing.Main(matchString, tests, benchmarks)
}`
<commit_msg>gotest: execute gomake properly on Windows<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"exec"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"unicode"
	"utf8"
)

// Environment for commands.
var (
	XGC       []string // 6g -I _test -o _xtest_.6
	GC        []string // 6g -I _test _testmain.go
	GL        []string // 6l -L _test _testmain.6
	GOARCH    string
	GOROOT    string
	GORUN     string
	O         string
	args      []string // arguments passed to gotest; also passed to the binary
	fileNames []string
	env       = os.Environ()
)

// These strings are created by getTestNames.
var (
	insideFileNames  []string // list of *.go files inside the package.
	outsideFileNames []string // list of *.go files outside the package (in package foo_test).
)

var (
	files      []*File
	importPath string
)

// Flags for our own purposes. We do our own flag processing.
var (
	cFlag bool
	xFlag bool
)

// File represents a file that contains tests.
type File struct {
	name       string
	pkg        string
	file       *os.File
	astFile    *ast.File
	tests      []string // The names of the TestXXXs.
	benchmarks []string // The names of the BenchmarkXXXs.
}

func main() {
	flags()
	needMakefile()
	setEnvironment()
	getTestFileNames()
	parseFiles()
	getTestNames()
	run("gomake", "testpackage-clean")
	run("gomake", "testpackage", fmt.Sprintf("GOTESTFILES=%s", strings.Join(insideFileNames, " ")))
	if len(outsideFileNames) > 0 {
		run(append(XGC, outsideFileNames...)...)
	}
	importPath = runWithStdout("gomake", "-s", "importpath")
	writeTestmainGo()
	run(GC...)
	run(GL...)
	if !cFlag {
		runTestWithArgs("./" + O + ".out")
	}
}

// needMakefile tests that we have a Makefile in this directory.
func needMakefile() {
	if _, err := os.Stat("Makefile"); err != nil {
		Fatalf("please create a Makefile for gotest; see http://golang.org/doc/code.html for details")
	}
}

// Fatalf formats its arguments, prints the message with a final newline, and exits.
func Fatalf(s string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "gotest: "+s+"\n", args...)
	os.Exit(2)
}

// theChar is the map from architecture to object character.
var theChar = map[string]string{
	"arm":   "5",
	"amd64": "6",
	"386":   "8",
}

// addEnv adds a name=value pair to the environment passed to subcommands.
// If the item is already in the environment, addEnv replaces the value.
func addEnv(name, value string) {
	for i := 0; i < len(env); i++ {
		if strings.HasPrefix(env[i], name+"=") {
			env[i] = name + "=" + value
			return
		}
	}
	env = append(env, name+"="+value)
}

// setEnvironment assembles the configuration for gotest and its subcommands.
func setEnvironment() {
	// Basic environment.
	GOROOT = runtime.GOROOT()
	addEnv("GOROOT", GOROOT)
	GOARCH = runtime.GOARCH
	addEnv("GOARCH", GOARCH)
	O = theChar[GOARCH]
	if O == "" {
		Fatalf("unknown architecture %s", GOARCH)
	}

	// Commands and their flags.
	gc := os.Getenv("GC")
	if gc == "" {
		gc = O + "g"
	}
	XGC = []string{gc, "-I", "_test", "-o", "_xtest_." + O}
	GC = []string{gc, "-I", "_test", "_testmain.go"}
	gl := os.Getenv("GL")
	if gl == "" {
		gl = O + "l"
	}
	GL = []string{gl, "-L", "_test", "_testmain." + O}

	// Silence make on Linux
	addEnv("MAKEFLAGS", "")
	addEnv("MAKELEVEL", "")
}

// getTestFileNames gets the set of files we're looking at.
// If gotest has no arguments, it scans the current directory for _test.go files.
func getTestFileNames() {
	names := fileNames
	if len(names) == 0 {
		names = filepath.Glob("[^.]*_test.go")
		if len(names) == 0 {
			Fatalf(`no test files found: no match for "*_test.go"`)
		}
	}
	for _, n := range names {
		fd, err := os.Open(n, os.O_RDONLY, 0)
		if err != nil {
			Fatalf("%s: %s", n, err)
		}
		f := &File{name: n, file: fd}
		files = append(files, f)
	}
}

// parseFiles parses the files and remembers the packages we find.
func parseFiles() {
	fileSet := token.NewFileSet()
	for _, f := range files {
		// Report declaration errors so we can abort if the files are incorrect Go.
		file, err := parser.ParseFile(fileSet, f.name, nil, parser.DeclarationErrors)
		if err != nil {
			Fatalf("parse error: %s", err)
		}
		f.astFile = file
		f.pkg = file.Name.String()
		if f.pkg == "" {
			Fatalf("cannot happen: no package name in %s", f.name)
		}
	}
}

// getTestNames extracts the names of tests and benchmarks.  They are all
// top-level functions that are not methods.
func getTestNames() {
	for _, f := range files {
		for _, d := range f.astFile.Decls {
			n, ok := d.(*ast.FuncDecl)
			if !ok {
				continue
			}
			if n.Recv != nil { // a method, not a function.
				continue
			}
			name := n.Name.String()
			if isTest(name, "Test") {
				f.tests = append(f.tests, name)
			} else if isTest(name, "Benchmark") {
				f.benchmarks = append(f.benchmarks, name)
			}
			// TODO: worth checking the signature? Probably not.
		}
		if strings.HasSuffix(f.pkg, "_test") {
			outsideFileNames = append(outsideFileNames, f.name)
		} else {
			insideFileNames = append(insideFileNames, f.name)
		}
	}
}

// isTest tells whether name looks like a test (or benchmark, according to prefix).
// It is a Test (say) if there is a character after Test that is not a lower-case letter.
// We don't want TesticularCancer.
func isTest(name, prefix string) bool {
	if !strings.HasPrefix(name, prefix) {
		return false
	}
	if len(name) == len(prefix) { // "Test" is ok
		return true
	}
	rune, _ := utf8.DecodeRuneInString(name[len(prefix):])
	return !unicode.IsLower(rune)
}

func run(args ...string) {
	doRun(args, false)
}

// runWithStdout is like run, but returns the text of standard output with the last newline dropped.
func runWithStdout(argv ...string) string {
	s := doRun(argv, true)
	if len(s) == 0 {
		Fatalf("no output from command %s", strings.Join(argv, " "))
	}
	if s[len(s)-1] == '\n' {
		s = s[:len(s)-1]
	}
	return s
}

// runTestWithArgs appends gotest's runs the provided binary with the args passed on the command line.
func runTestWithArgs(binary string) {
	doRun(append([]string{binary}, args...), false)
}

// doRun is the general command runner.  The flag says whether we want to
// retrieve standard output.
func doRun(argv []string, returnStdout bool) string {
	if xFlag {
		fmt.Printf("gotest: %s\n", strings.Join(argv, " "))
	}
	if runtime.GOOS == "windows" && argv[0] == "gomake" {
		// gomake is a shell script and it cannot be executed directly on Windows.
		argv = append([]string{"cmd", "/c", "sh", "-c"}, strings.Join(argv, " "))
	}
	var err os.Error
	argv[0], err = exec.LookPath(argv[0])
	if err != nil {
		Fatalf("can't find %s: %s", argv[0], err)
	}
	procAttr := &os.ProcAttr{
		Env: env,
		Files: []*os.File{
			os.Stdin,
			os.Stdout,
			os.Stderr,
		},
	}
	var r, w *os.File
	if returnStdout {
		r, w, err = os.Pipe()
		if err != nil {
			Fatalf("can't create pipe: %s", err)
		}
		procAttr.Files[1] = w
	}
	proc, err := os.StartProcess(argv[0], argv, procAttr)
	if err != nil {
		Fatalf("make failed to start: %s", err)
	}
	if returnStdout {
		defer r.Close()
		w.Close()
	}
	waitMsg, err := proc.Wait(0)
	if err != nil || waitMsg == nil {
		Fatalf("%s failed: %s", argv[0], err)
	}
	if !waitMsg.Exited() || waitMsg.ExitStatus() != 0 {
		Fatalf("%q failed: %s", strings.Join(argv, " "), waitMsg)
	}
	if returnStdout {
		b, err := ioutil.ReadAll(r)
		if err != nil {
			Fatalf("can't read output from command: %s", err)
		}
		return string(b)
	}
	return ""
}

// writeTestmainGo generates the test program to be compiled, "./_testmain.go".
func writeTestmainGo() {
	f, err := os.Open("_testmain.go", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		Fatalf("can't create _testmain.go: %s", err)
	}
	defer f.Close()
	b := bufio.NewWriter(f)
	defer b.Flush()

	// Package and imports.
	fmt.Fprint(b, "package main\n\n")
	// Are there tests from a package other than the one we're testing?
	// We can't just use file names because some of the things we compiled
	// contain no tests.
	outsideTests := false
	insideTests := false
	for _, f := range files {
		//println(f.name, f.pkg)
		if len(f.tests) == 0 && len(f.benchmarks) == 0 {
			continue
		}
		if strings.HasSuffix(f.pkg, "_test") {
			outsideTests = true
		} else {
			insideTests = true
		}
	}
	if insideTests {
		switch importPath {
		case "testing":
		case "main":
			// Import path main is reserved, so import with
			// explicit reference to ./_test/main instead.
			// Also, the file we are writing defines a function named main,
			// so rename this import to __main__ to avoid name conflict.
			fmt.Fprintf(b, "import __main__ %q\n", "./_test/main")
		default:
			fmt.Fprintf(b, "import %q\n", importPath)
		}
	}
	if outsideTests {
		fmt.Fprintf(b, "import %q\n", "./_xtest_")
	}
	fmt.Fprintf(b, "import %q\n", "testing")
	fmt.Fprintf(b, "import __os__     %q\n", "os")     // rename in case tested package is called os
	fmt.Fprintf(b, "import __regexp__ %q\n", "regexp") // rename in case tested package is called regexp
	fmt.Fprintln(b)                                    // for gofmt

	// Tests.
	fmt.Fprintln(b, "var tests = []testing.InternalTest{")
	for _, f := range files {
		for _, t := range f.tests {
			fmt.Fprintf(b, "\t{\"%s.%s\", %s.%s},\n", f.pkg, t, notMain(f.pkg), t)
		}
	}
	fmt.Fprintln(b, "}")
	fmt.Fprintln(b)

	// Benchmarks.
	fmt.Fprintln(b, "var benchmarks = []testing.InternalBenchmark{")
	for _, f := range files {
		for _, bm := range f.benchmarks {
			fmt.Fprintf(b, "\t{\"%s.%s\", %s.%s},\n", f.pkg, bm, notMain(f.pkg), bm)
		}
	}
	fmt.Fprintln(b, "}")

	// Body.
	fmt.Fprintln(b, testBody)
}

// notMain returns the package, renaming as appropriate if it's "main".
func notMain(pkg string) string {
	if pkg == "main" {
		return "__main__"
	}
	return pkg
}

// testBody is just copied to the output. It's the code that runs the tests.
var testBody = `
var matchPat string
var matchRe *__regexp__.Regexp

func matchString(pat, str string) (result bool, err __os__.Error) {
	if matchRe == nil || matchPat != pat {
		matchPat = pat
		matchRe, err = __regexp__.Compile(matchPat)
		if err != nil {
			return
		}
	}
	return matchRe.MatchString(str), nil
}

func main() {
	testing.Main(matchString, tests, benchmarks)
}`
<|endoftext|>