<commit_before>// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tester

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/coreos/etcd/functional/rpcpb"

	"go.uber.org/zap"
)

// Case defines failure/test injection interface.
// To add a test case:
//  1. implement "Case" interface
//  2. define fail case name in "rpcpb.Case"
type Case interface {
	// Inject injeccts the failure into the testing cluster at the given
	// round. When calling the function, the cluster should be in health.
	Inject(clus *Cluster) error
	// Recover recovers the injected failure caused by the injection of the
	// given round and wait for the recovery of the testing cluster.
	Recover(clus *Cluster) error
	// Desc returns a description of the failure
	Desc() string
	// TestCase returns "rpcpb.Case" enum type.
	TestCase() rpcpb.Case
}

type injectMemberFunc func(*Cluster, int) error
type recoverMemberFunc func(*Cluster, int) error

type caseByFunc struct {
	desc          string
	rpcpbCase     rpcpb.Case
	injectMember  injectMemberFunc
	recoverMember recoverMemberFunc
}

func (c *caseByFunc) Desc() string {
	if c.desc != "" {
		return c.desc
	}
	return c.rpcpbCase.String()
}

func (c *caseByFunc) TestCase() rpcpb.Case {
	return c.rpcpbCase
}

type caseFollower struct {
	caseByFunc
	last int
	lead int
}

func (c *caseFollower) updateIndex(clus *Cluster) error {
	lead, err := clus.GetLeader()
	if err != nil {
		return err
	}
	c.lead = lead

	n := len(clus.Members)
	if c.last == -1 { // first run
		c.last = clus.rd % n
		if c.last == c.lead {
			c.last = (c.last + 1) % n
		}
	} else {
		c.last = (c.last + 1) % n
		if c.last == c.lead {
			c.last = (c.last + 1) % n
		}
	}
	return nil
}

func (c *caseFollower) Inject(clus *Cluster) error {
	if err := c.updateIndex(clus); err != nil {
		return err
	}
	return c.injectMember(clus, c.last)
}

func (c *caseFollower) Recover(clus *Cluster) error {
	return c.recoverMember(clus, c.last)
}

func (c *caseFollower) Desc() string {
	if c.desc != "" {
		return c.desc
	}
	return c.rpcpbCase.String()
}

func (c *caseFollower) TestCase() rpcpb.Case {
	return c.rpcpbCase
}

type caseLeader struct {
	caseByFunc
	last int
	lead int
}

func (c *caseLeader) updateIndex(clus *Cluster) error {
	lead, err := clus.GetLeader()
	if err != nil {
		return err
	}
	c.lead = lead
	c.last = lead
	return nil
}

func (c *caseLeader) Inject(clus *Cluster) error {
	if err := c.updateIndex(clus); err != nil {
		return err
	}
	return c.injectMember(clus, c.last)
}

func (c *caseLeader) Recover(clus *Cluster) error {
	return c.recoverMember(clus, c.last)
}

func (c *caseLeader) TestCase() rpcpb.Case {
	return c.rpcpbCase
}

type caseQuorum struct {
	caseByFunc
	injected map[int]struct{}
}

func (c *caseQuorum) Inject(clus *Cluster) error {
	c.injected = pickQuorum(len(clus.Members))
	for idx := range c.injected {
		if err := c.injectMember(clus, idx); err != nil {
			return err
		}
	}
	return nil
}

func (c *caseQuorum) Recover(clus *Cluster) error {
	for idx := range c.injected {
		if err := c.recoverMember(clus, idx); err != nil {
			return err
		}
	}
	return nil
}

func (c *caseQuorum) Desc() string {
	if c.desc != "" {
		return c.desc
	}
	return c.rpcpbCase.String()
}

func (c *caseQuorum) TestCase() rpcpb.Case {
	return c.rpcpbCase
}

func pickQuorum(size int) (picked map[int]struct{}) {
	picked = make(map[int]struct{})
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	quorum := size/2 + 1
	for len(picked) < quorum {
		idx := r.Intn(size)
		picked[idx] = struct{}{}
	}
	return picked
}

type caseAll caseByFunc

func (c *caseAll) Inject(clus *Cluster) error {
	for i := range clus.Members {
		if err := c.injectMember(clus, i); err != nil {
			return err
		}
	}
	return nil
}

func (c *caseAll) Recover(clus *Cluster) error {
	for i := range clus.Members {
		if err := c.recoverMember(clus, i); err != nil {
			return err
		}
	}
	return nil
}

func (c *caseAll) Desc() string {
	if c.desc != "" {
		return c.desc
	}
	return c.rpcpbCase.String()
}

func (c *caseAll) TestCase() rpcpb.Case {
	return c.rpcpbCase
}

// caseUntilSnapshot injects a failure/test and waits for a snapshot event
type caseUntilSnapshot struct {
	desc      string
	rpcpbCase rpcpb.Case
	Case
}

// all delay failure cases except the ones failing with latency
// greater than election timeout (trigger leader election and
// cluster keeps operating anyways)
var slowCases = map[rpcpb.Case]bool{
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER:                        true,
	rpcpb.Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT:        true,
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT: true,
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER:                              true,
	rpcpb.Case_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT:              true,
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT:       true,
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM:                              true,
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ALL:                                 true,
}

func (c *caseUntilSnapshot) Inject(clus *Cluster) error {
	if err := c.Case.Inject(clus); err != nil {
		return err
	}

	snapshotCount := clus.Members[0].Etcd.SnapshotCount

	now := time.Now()
	clus.lg.Info(
		"trigger snapshot START",
		zap.String("desc", c.Desc()),
		zap.Int64("etcd-snapshot-count", snapshotCount),
	)

	// maxRev may fail since failure just injected, retry if failed.
	startRev, err := clus.maxRev()
	for i := 0; i < 10 && startRev == 0; i++ {
		startRev, err = clus.maxRev()
	}
	if startRev == 0 {
		return err
	}
	lastRev := startRev

	// healthy cluster could accept 1000 req/sec at least.
	// 3x time to trigger snapshot.
	retries := int(snapshotCount) / 1000 * 3
	if v, ok := slowCases[c.TestCase()]; v && ok {
		// slow network takes more retries
		retries *= 5
	}

	for i := 0; i < retries; i++ {
		lastRev, err = clus.maxRev()
		// If the number of proposals committed is bigger than snapshot count,
		// a new snapshot should have been created.
		diff := lastRev - startRev
		if diff > snapshotCount {
			clus.lg.Info(
				"trigger snapshot PASS",
				zap.Int("retries", i),
				zap.String("desc", c.Desc()),
				zap.Int64("committed-entries", diff),
				zap.Int64("etcd-snapshot-count", snapshotCount),
				zap.Int64("start-revision", startRev),
				zap.Int64("last-revision", lastRev),
				zap.Duration("took", time.Since(now)),
			)
			return nil
		}

		dur := time.Second
		if diff < 0 || err != nil {
			dur = 3 * time.Second
		}
		clus.lg.Info(
			"trigger snapshot PROGRESS",
			zap.Int("retries", i),
			zap.Int64("committed-entries", diff),
			zap.Int64("etcd-snapshot-count", snapshotCount),
			zap.Int64("start-revision", startRev),
			zap.Int64("last-revision", lastRev),
			zap.Duration("took", time.Since(now)),
			zap.Error(err),
		)
		time.Sleep(dur)
	}

	return fmt.Errorf("cluster too slow: only %d commits in %d retries", lastRev-startRev, retries)
}

func (c *caseUntilSnapshot) Desc() string {
	if c.desc != "" {
		return c.desc
	}
	if c.rpcpbCase.String() != "" {
		return c.rpcpbCase.String()
	}
	return c.Case.Desc()
}

func (c *caseUntilSnapshot) TestCase() rpcpb.Case {
	return c.rpcpbCase
}
<commit_msg>functional/tester: fix trigger snapshot retry<commit_after>// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tester

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/coreos/etcd/functional/rpcpb"

	"go.uber.org/zap"
)

// Case defines failure/test injection interface.
// To add a test case:
//  1. implement "Case" interface
//  2. define fail case name in "rpcpb.Case"
type Case interface {
	// Inject injeccts the failure into the testing cluster at the given
	// round. When calling the function, the cluster should be in health.
	Inject(clus *Cluster) error
	// Recover recovers the injected failure caused by the injection of the
	// given round and wait for the recovery of the testing cluster.
	Recover(clus *Cluster) error
	// Desc returns a description of the failure
	Desc() string
	// TestCase returns "rpcpb.Case" enum type.
	TestCase() rpcpb.Case
}

type injectMemberFunc func(*Cluster, int) error
type recoverMemberFunc func(*Cluster, int) error

type caseByFunc struct {
	desc          string
	rpcpbCase     rpcpb.Case
	injectMember  injectMemberFunc
	recoverMember recoverMemberFunc
}

func (c *caseByFunc) Desc() string {
	if c.desc != "" {
		return c.desc
	}
	return c.rpcpbCase.String()
}

func (c *caseByFunc) TestCase() rpcpb.Case {
	return c.rpcpbCase
}

type caseFollower struct {
	caseByFunc
	last int
	lead int
}

func (c *caseFollower) updateIndex(clus *Cluster) error {
	lead, err := clus.GetLeader()
	if err != nil {
		return err
	}
	c.lead = lead

	n := len(clus.Members)
	if c.last == -1 { // first run
		c.last = clus.rd % n
		if c.last == c.lead {
			c.last = (c.last + 1) % n
		}
	} else {
		c.last = (c.last + 1) % n
		if c.last == c.lead {
			c.last = (c.last + 1) % n
		}
	}
	return nil
}

func (c *caseFollower) Inject(clus *Cluster) error {
	if err := c.updateIndex(clus); err != nil {
		return err
	}
	return c.injectMember(clus, c.last)
}

func (c *caseFollower) Recover(clus *Cluster) error {
	return c.recoverMember(clus, c.last)
}

func (c *caseFollower) Desc() string {
	if c.desc != "" {
		return c.desc
	}
	return c.rpcpbCase.String()
}

func (c *caseFollower) TestCase() rpcpb.Case {
	return c.rpcpbCase
}

type caseLeader struct {
	caseByFunc
	last int
	lead int
}

func (c *caseLeader) updateIndex(clus *Cluster) error {
	lead, err := clus.GetLeader()
	if err != nil {
		return err
	}
	c.lead = lead
	c.last = lead
	return nil
}

func (c *caseLeader) Inject(clus *Cluster) error {
	if err := c.updateIndex(clus); err != nil {
		return err
	}
	return c.injectMember(clus, c.last)
}

func (c *caseLeader) Recover(clus *Cluster) error {
	return c.recoverMember(clus, c.last)
}

func (c *caseLeader) TestCase() rpcpb.Case {
	return c.rpcpbCase
}

type caseQuorum struct {
	caseByFunc
	injected map[int]struct{}
}

func (c *caseQuorum) Inject(clus *Cluster) error {
	c.injected = pickQuorum(len(clus.Members))
	for idx := range c.injected {
		if err := c.injectMember(clus, idx); err != nil {
			return err
		}
	}
	return nil
}

func (c *caseQuorum) Recover(clus *Cluster) error {
	for idx := range c.injected {
		if err := c.recoverMember(clus, idx); err != nil {
			return err
		}
	}
	return nil
}

func (c *caseQuorum) Desc() string {
	if c.desc != "" {
		return c.desc
	}
	return c.rpcpbCase.String()
}

func (c *caseQuorum) TestCase() rpcpb.Case {
	return c.rpcpbCase
}

func pickQuorum(size int) (picked map[int]struct{}) {
	picked = make(map[int]struct{})
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	quorum := size/2 + 1
	for len(picked) < quorum {
		idx := r.Intn(size)
		picked[idx] = struct{}{}
	}
	return picked
}

type caseAll caseByFunc

func (c *caseAll) Inject(clus *Cluster) error {
	for i := range clus.Members {
		if err := c.injectMember(clus, i); err != nil {
			return err
		}
	}
	return nil
}

func (c *caseAll) Recover(clus *Cluster) error {
	for i := range clus.Members {
		if err := c.recoverMember(clus, i); err != nil {
			return err
		}
	}
	return nil
}

func (c *caseAll) Desc() string {
	if c.desc != "" {
		return c.desc
	}
	return c.rpcpbCase.String()
}

func (c *caseAll) TestCase() rpcpb.Case {
	return c.rpcpbCase
}

// caseUntilSnapshot injects a failure/test and waits for a snapshot event
type caseUntilSnapshot struct {
	desc      string
	rpcpbCase rpcpb.Case
	Case
}

// all delay failure cases except the ones failing with latency
// greater than election timeout (trigger leader election and
// cluster keeps operating anyways)
var slowCases = map[rpcpb.Case]bool{
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER:                        true,
	rpcpb.Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT:        true,
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT: true,
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER:                              true,
	rpcpb.Case_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT:              true,
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT:       true,
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM:                              true,
	rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ALL:                                 true,
}

func (c *caseUntilSnapshot) Inject(clus *Cluster) error {
	if err := c.Case.Inject(clus); err != nil {
		return err
	}

	snapshotCount := clus.Members[0].Etcd.SnapshotCount

	now := time.Now()
	clus.lg.Info(
		"trigger snapshot START",
		zap.String("desc", c.Desc()),
		zap.Int64("etcd-snapshot-count", snapshotCount),
	)

	// maxRev may fail since failure just injected, retry if failed.
	startRev, err := clus.maxRev()
	for i := 0; i < 10 && startRev == 0; i++ {
		startRev, err = clus.maxRev()
	}
	if startRev == 0 {
		return err
	}
	lastRev := startRev

	// healthy cluster could accept 1000 req/sec at least.
	// 3x time to trigger snapshot.
	retries := int(snapshotCount) / 1000 * 3
	if v, ok := slowCases[c.TestCase()]; v && ok {
		// slow network takes more retries
		retries *= 5
	}

	for i := 0; i < retries; i++ {
		lastRev, err = clus.maxRev()
		if lastRev == 0 {
			clus.lg.Info(
				"trigger snapshot RETRY",
				zap.Int("retries", i),
				zap.Int64("etcd-snapshot-count", snapshotCount),
				zap.Int64("start-revision", startRev),
				zap.Error(err),
			)
			time.Sleep(3 * time.Second)
			continue
		}

		// If the number of proposals committed is bigger than snapshot count,
		// a new snapshot should have been created.
		diff := lastRev - startRev
		if diff > snapshotCount {
			clus.lg.Info(
				"trigger snapshot PASS",
				zap.Int("retries", i),
				zap.String("desc", c.Desc()),
				zap.Int64("committed-entries", diff),
				zap.Int64("etcd-snapshot-count", snapshotCount),
				zap.Int64("start-revision", startRev),
				zap.Int64("last-revision", lastRev),
				zap.Duration("took", time.Since(now)),
			)
			return nil
		}

		clus.lg.Info(
			"trigger snapshot RETRY",
			zap.Int("retries", i),
			zap.Int64("committed-entries", diff),
			zap.Int64("etcd-snapshot-count", snapshotCount),
			zap.Int64("start-revision", startRev),
			zap.Int64("last-revision", lastRev),
			zap.Duration("took", time.Since(now)),
			zap.Error(err),
		)
		time.Sleep(time.Second)
		if err != nil {
			time.Sleep(2 * time.Second)
		}
	}

	return fmt.Errorf("cluster too slow: only %d commits in %d retries", lastRev-startRev, retries)
}

func (c *caseUntilSnapshot) Desc() string {
	if c.desc != "" {
		return c.desc
	}
	if c.rpcpbCase.String() != "" {
		return c.rpcpbCase.String()
	}
	return c.Case.Desc()
}

func (c *caseUntilSnapshot) TestCase() rpcpb.Case {
	return c.rpcpbCase
}
<|endoftext|>


<commit_before>package test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/adamdecaf/cert-manage/tools/file"
)

type dockerfile struct {
	// Local fs path to the Dockerfile
	base string

	// Commands represents a series of commands to be ran in the image
	commands []*Cmd

	// -t flag with build/run
	tag string

	// any errors specific to the dockerfile from `docker` commands
	err error

	// only run build, tag and run steps once
	wg sync.WaitGroup

	// used for cert-manage init
	sync.Once
}

func Dockerfile(where string) *dockerfile {
	if !strings.HasSuffix(where, "Dockerfile") {
		where = filepath.Join(where, "/Dockerfile")
	}

	// Grab the env name (e.g. envs/$env_name/Dockerfile)
	dir := filepath.Dir(where)
	now := time.Now().Unix()
	tag := fmt.Sprintf("cert-manage:%s-%d", filepath.Base(dir), now)

	return &dockerfile{
		base: where,
		tag:  tag,
	}
}

func (d *dockerfile) Run(cmd string, args ...string) {
	d.commands = append(d.commands, Command(cmd, args...))
}

func (d *dockerfile) CertManage(args ...string) {
	d.Do(func() {
		d.Run("chmod", "+x", "/bin/cert-manage")
	})
	d.Run("/bin/cert-manage", args...)
}

func (d *dockerfile) SuccessT(t *testing.T) {
	if runtime.GOOS == "darwin" {
		err := exec.Command("docker", "verison").Run()
		if err == nil {
			t.Fatal("travis-ci supports docker on OSX?? - https://docs.travis-ci.com/user/docker/")
		}
		if inCI() {
			t.Skip("travis-ci doesn't support docker on OSX - https://docs.travis-ci.com/user/docker/")
		}
	}

	if !d.enabled() {
		t.Skip("docker isn't enabled")
	}

	d.prep()
	t.Helper()

	if d.err != nil {
		t.Fatal(d.err)
	}
}

func (d *dockerfile) build() {
	d.wg.Add(1)
	defer d.wg.Done()

	// Copy our original image's contents into the dst file
	dir, err := ioutil.TempDir("", d.tag)
	if err != nil {
		d.err = fmt.Errorf("tempfile create err=%v", err)
		return
	}

	// Copy cert-manage to the temp directory and assume it's linux
	err = file.CopyFile("../bin/cert-manage-linux-amd64", filepath.Join(dir, "cert-manage"))
	if err != nil {
		d.err = fmt.Errorf("error copying cert-manage to tmp dir, err=%v", err)
		return
	}

	dst, err := os.Create(filepath.Join(dir, "Dockerfile"))
	if err != nil {
		d.err = fmt.Errorf("tmp Dockerfile create err=%v", err)
		return
	}
	defer os.Remove(dst.Name())

	src, err := os.Open(d.base)
	if err != nil {
		d.err = fmt.Errorf("tmpfile open err=%v", err)
		return
	}
	if _, err := io.Copy(dst, src); err != nil {
		d.err = fmt.Errorf("src->dst copy err=%v", err)
		return
	}
	if err := src.Close(); err != nil {
		d.err = fmt.Errorf("src close err=%v", err)
		return
	}

	// Add all commands to the Dockerfile
	initial := "CMD "
	command := initial
	for i := range d.commands {
		if command != initial {
			command += " && "
		}
		command += fmt.Sprintf("%s %s", d.commands[i].command, strings.Join(d.commands[i].args, " "))
	}
	if _, err := dst.WriteString(command); err != nil {
		d.err = fmt.Errorf("command=%s err=%v", command, err)
		return
	}

	// Force all writes into our Dockerfile
	if err := dst.Sync(); err != nil {
		d.err = fmt.Errorf("dst fsync err=%v", err)
		return
	}

	cmd := exec.Command("docker", "build", "-t", d.tag, dir)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, stderr.String())
	}
}

func (d *dockerfile) run() {
	d.wg.Add(1)
	defer d.wg.Done()

	// don't attempt anything if we've already failed
	if d.err != nil {
		return
	}

	cmd := exec.Command("docker", "run", "-t", d.tag)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, stderr.String())
	}
}

func (d *dockerfile) prep() {
	d.build()
	d.run()
	d.wg.Wait()
}

func (d *dockerfile) enabled() bool {
	out, _ := exec.Command("docker", "ps").CombinedOutput()
	if bytes.Contains(out, []byte("Cannot connect to the Docker daemon")) {
		return false
	}
	return true
}
<commit_msg>test/docker: copy a whitelist over for docker images<commit_after>package test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/adamdecaf/cert-manage/tools/file"
)

type dockerfile struct {
	// Local fs path to the Dockerfile
	base string

	// Commands represents a series of commands to be ran in the image
	commands []*Cmd

	// -t flag with build/run
	tag string

	// any errors specific to the dockerfile from `docker` commands
	err error

	// only run build, tag and run steps once
	wg sync.WaitGroup

	// used for cert-manage init
	sync.Once
}

func Dockerfile(where string) *dockerfile {
	if !strings.HasSuffix(where, "Dockerfile") {
		where = filepath.Join(where, "/Dockerfile")
	}

	// Grab the env name (e.g. envs/$env_name/Dockerfile)
	dir := filepath.Dir(where)
	now := time.Now().Unix()
	tag := fmt.Sprintf("cert-manage:%s-%d", filepath.Base(dir), now)

	return &dockerfile{
		base: where,
		tag:  tag,
	}
}

func (d *dockerfile) Run(cmd string, args ...string) {
	d.commands = append(d.commands, Command(cmd, args...))
}

func (d *dockerfile) CertManage(args ...string) {
	d.Do(func() {
		d.Run("chmod", "+x", "/bin/cert-manage")
	})
	d.Run("/bin/cert-manage", args...)
}

func (d *dockerfile) SuccessT(t *testing.T) {
	if runtime.GOOS == "darwin" {
		err := exec.Command("docker", "verison").Run()
		if err == nil {
			t.Fatal("travis-ci supports docker on OSX?? - https://docs.travis-ci.com/user/docker/")
		}
		if inCI() {
			t.Skip("travis-ci doesn't support docker on OSX - https://docs.travis-ci.com/user/docker/")
		}
	}

	if !d.enabled() {
		t.Skip("docker isn't enabled")
	}

	d.prep()
	t.Helper()

	if d.err != nil {
		t.Fatal(d.err)
	}
}

func (d *dockerfile) build() {
	d.wg.Add(1)
	defer d.wg.Done()

	// Copy our original image's contents into the dst file
	dir, err := ioutil.TempDir("", d.tag)
	if err != nil {
		d.err = fmt.Errorf("tempfile create err=%v", err)
		return
	}

	// Copy cert-manage and whitelist to the temp directory and assume it's linux
	copyable := []string{"../bin/cert-manage-linux-amd64", "../testdata/globalsign-whitelist.json"}
	for i := range copyable {
		name := filepath.Base(copyable[i])
		err = file.CopyFile(copyable[i], filepath.Join(dir, name))
		if err != nil {
			d.err = fmt.Errorf("error copying %s to tmp dir, err=%v", name, err)
			return
		}
	}

	dst, err := os.Create(filepath.Join(dir, "Dockerfile"))
	if err != nil {
		d.err = fmt.Errorf("tmp Dockerfile create err=%v", err)
		return
	}
	defer os.Remove(dst.Name())

	src, err := os.Open(d.base)
	if err != nil {
		d.err = fmt.Errorf("tmpfile open err=%v", err)
		return
	}
	if _, err := io.Copy(dst, src); err != nil {
		d.err = fmt.Errorf("src->dst copy err=%v", err)
		return
	}
	if err := src.Close(); err != nil {
		d.err = fmt.Errorf("src close err=%v", err)
		return
	}

	// Add all commands to the Dockerfile
	initial := "CMD "
	command := initial
	for i := range d.commands {
		if command != initial {
			command += " && "
		}
		command += fmt.Sprintf("%s %s", d.commands[i].command, strings.Join(d.commands[i].args, " "))
	}
	if _, err := dst.WriteString(command); err != nil {
		d.err = fmt.Errorf("command=%s err=%v", command, err)
		return
	}

	// Force all writes into our Dockerfile
	if err := dst.Sync(); err != nil {
		d.err = fmt.Errorf("dst fsync err=%v", err)
		return
	}

	cmd := exec.Command("docker", "build", "-t", d.tag, dir)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, stderr.String())
	}
}

func (d *dockerfile) run() {
	d.wg.Add(1)
	defer d.wg.Done()

	// don't attempt anything if we've already failed
	if d.err != nil {
		return
	}

	cmd := exec.Command("docker", "run", "-t", d.tag)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, stderr.String())
	}
}

func (d *dockerfile) prep() {
	d.build()
	d.run()
	d.wg.Wait()
}

func (d *dockerfile) enabled() bool {
	out, _ := exec.Command("docker", "ps").CombinedOutput()
	if bytes.Contains(out, []byte("Cannot connect to the Docker daemon")) {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package kitepinger

import (
	"errors"
	"fmt"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/koding/kite"
)

type fakePinger struct {
	ReturnStatus Status
}

func (p *fakePinger) Ping() Status {
	return p.ReturnStatus
}

func TestSubscribe(t *testing.T) {
	p := &PingTracker{
		nodata:      struct{}{},
		subscribers: map[chan<- ChangeSummary]struct{}{},
	}

	// broadcast with no subscribers, to make sure we don't panic/etc
	p.broadcastSummary(ChangeSummary{})

	a := make(chan ChangeSummary, 1)
	p.Subscribe(a)

	_, ok := p.subscribers[a]
	if !ok {
		t.Error("Expected Register to store channels")
	}
}

func TestUnsubscribe(t *testing.T) {
	p := &PingTracker{
		nodata:      struct{}{},
		subscribers: map[chan<- ChangeSummary]struct{}{},
		ticker:      time.NewTicker(time.Minute),
		pinging:     true,
	}

	// broadcast with no subscribers, to make sure we don't panic/etc
	p.broadcastSummary(ChangeSummary{})

	a := make(chan ChangeSummary, 1)
	b := make(chan ChangeSummary, 1)
	c := make(chan ChangeSummary, 1)
	p.Subscribe(a)
	p.Subscribe(b)
	p.Subscribe(c)

	p.Unsubscribe(b)

	_, ok := p.subscribers[b]
	if ok {
		t.Error("Expected Unregister to remove channels")
	}

	if len(p.subscribers) != 2 {
		t.Errorf(
			"Expected to leave existing channels alone. Wanted %d, have %d",
			2, len(p.subscribers),
		)
	}

	if p.pinging != true {
		t.Error("Expected Unsubscribe to not call Stop() if there are still subs")
	}

	p.Unsubscribe(a)
	p.Unsubscribe(c)

	if len(p.subscribers) != 0 {
		t.Errorf(
			"Expected to all subs to be gone. Wanted %d, have %d",
			0, len(p.subscribers),
		)
	}

	if p.pinging != false {
		t.Error("Expected Unsubscribe to call Stop() if there are no subs")
	}
}

func TestBroadcastSummary(t *testing.T) {
	p := &PingTracker{
		nodata:      struct{}{},
		subscribers: map[chan<- ChangeSummary]struct{}{},
	}

	// broadcast with no subscribers, to make sure we don't panic/etc
	p.broadcastSummary(ChangeSummary{})

	a := make(chan ChangeSummary, 1)
	p.Subscribe(a)
	p.broadcastSummary(ChangeSummary{
		NewStatus: Success,
	})
	select {
	case s := <-a:
		// Just check if we get the proper value
		if s.NewStatus != Success {
			t.Errorf(
				"Expected broadcastSummary() to send the given Summary to a single channel. Wanted %s, Got %s",
				Success, s.NewStatus,
			)
		}
	case <-time.After(200 * time.Millisecond):
		t.Error("Expected broadcastSummary() to call a single channel.")
	}

	b := make(chan ChangeSummary, 1)
	p.Subscribe(b)

	// Use different data than above
	p.broadcastSummary(ChangeSummary{
		NewStatus: Failure,
	})

	aCount, bCount := 0, 0
	timeout := time.After(200 * time.Millisecond)
	for i := 0; i < 2; i++ {
		select {
		case s := <-a:
			aCount++
			if s.NewStatus != Failure {
				t.Errorf(
					"Expected broadcastSummary() to send the given Summary to a multiple channels. Wanted %s, Got %s",
					Failure, s.NewStatus,
				)
			}
		case s := <-b:
			bCount++
			if s.NewStatus != Failure {
				t.Errorf(
					"Expected broadcastSummary() to send the given Summary to a multiple channels. Wanted %s, Got %s",
					Failure, s.NewStatus,
				)
			}
		case <-timeout:
			t.Error("Timed out waiting for Notify() to call all channels")
		}
	}

	if aCount != 1 {
		t.Errorf(
			"Expected broadcastSummary to call each channel once. Wanted aCount of %d, got %d",
			1, aCount,
		)
	}

	if bCount != 1 {
		t.Errorf(
			"Expected broadcastSummary to call each channel once. Wanted bCount of %d, got %d",
			1, bCount,
		)
	}

	// Using a non-buffered channel here. Please see the explanation below
	// on the line that checks for:
	//
	//		if notBufferedCount != 0
	notBuffered := make(chan ChangeSummary)

	p.Subscribe(notBuffered)
	// Use different data than above
	p.broadcastSummary(ChangeSummary{
		NewStatus: Success,
	})

	aCount, bCount, notBufferedCount := 0, 0, 0
	timeout = time.After(200 * time.Millisecond)
	for i := 0; i < 2; i++ {
		select {
		case s := <-a:
			aCount++
			if s.NewStatus != Success {
				t.Errorf(
					"Expected broadcastSummary() to send the given Summary despite non-receiving channels. Wanted %s, Got %s",
					Success, s.NewStatus,
				)
			}
		case s := <-b:
			bCount++
			if s.NewStatus != Success {
				t.Errorf(
					"Expected broadcastSummary() to send the given Summary despite non-receiving channels. Wanted %s, Got %s",
					Success, s.NewStatus,
				)
			}
		case <-notBuffered:
			notBufferedCount++
		case <-timeout:
			t.Error("Timed out waiting for Notify() to call all channels")
		}
	}

	if aCount != 1 {
		t.Errorf(
			"Expected broadcastSummary to call each channel once. Wanted aCount of %d, got %d",
			1, aCount,
		)
	}

	if bCount != 1 {
		t.Errorf(
			"Expected broadcastSummary to call each channel once. Wanted bCount of %d, got %d",
			1, bCount,
		)
	}

	// Not buffered, means the broadcastSummary will be unable to send to the channel
	// because nothing is *currently* receiving from the channel when
	// broadcastSummary() occurs. broadcastSummary() should not block and wait for it.
	//
	// If broadcastSummary() does block, the receiver (in this test) will never get
	// a chance to listen either.. so really this shouldn't be possible..
	if notBufferedCount != 0 {
		t.Errorf("Expected broadcastSummary() to ignored the notBuffered channel because, at the time of sending, it was not receiving. Wanted %d, Got %d",
			0, notBufferedCount)
	}
}

func TestPing(t *testing.T) {
	r := kite.New("remote", "0.0.0")
	r.Config.DisableAuthentication = true
	ts := httptest.NewServer(r)

	var (
		shouldError bool
		pingCount   int
	)

	r.HandleFunc("kite.ping", func(req *kite.Request) (interface{}, error) {
		pingCount++

		if shouldError {
			return nil, errors.New("Told to error")
		}

		return "pong", nil
	})

	l := kite.New("local", "0.0.0").NewClient(fmt.Sprintf("%s/kite", ts.URL))

	// Connect the local client to the remote
	err := l.Dial()
	if err != nil {
		t.Fatal("Failed to connect to testing Kite", err)
	}

	p := &PingTracker{
		pinger:      NewKitePinger(l),
		nodata:      struct{}{},
		subscribers: map[chan<- ChangeSummary]struct{}{},
	}

	p.Ping()

	if pingCount != 1 {
		t.Errorf(
			"Expected Ping() to ping the remote client. Wanted a PingCount of %d, Got %d",
			1, pingCount,
		)
	}

	// Tell the handler to return an error
	shouldError = true
	p.Ping()
	sFailure := p.GetSummary()

	// Tell the handler to not return an error
	shouldError = false
	p.Ping()
	sSuccess := p.GetSummary()

	if pingCount != 3 {
		t.Errorf(
			"Expected Ping() to ping the remote client. Wanted a PingCount of %d, Got %d",
			3, pingCount,
		)
	}

	if sFailure.Status != Failure {
		t.Errorf(
			"Expected Ping() to return the correct status. Wanted %s, Got %s",
			Failure, sFailure.Status,
		)
	}

	if sSuccess.Status != Success {
		t.Errorf(
			"Expected Ping() to return the correct status. Wanted %s, Got %s",
			Success, sSuccess.Status,
		)
	}

	// Sleeping will increase the time that Ping has held the same status, ie
	// the CurrentSummary.StatusDur duration.
	time.Sleep(time.Millisecond * 100)
	p.Ping()

	// Allow for a 10ms variation in reported runtime.
	msDur := p.GetSummary().StatusDur.Nanoseconds() / 1000000
	if msDur < 90 || msDur > 120 {
		t.Errorf(
			"Expected CurrentSummary.StatusDur to be similar to the elapsed time. Wanted larger than %d and less than %d, Got %d",
			90, 120, msDur,
		)
	}

	// Record the value that KitePinger thinks it pinged at, for a future test
	lastPingedAt := p.pingedAt

	// Make a new subscriber and subscribe, then cause a failed ping
	a := make(chan ChangeSummary, 1)
	p.Subscribe(a)

	shouldError = true
	p.Ping()

	var summaryFromChan ChangeSummary
	select {
	case summaryFromChan = <-a:
	case <-time.After(100 * time.Millisecond):
		t.Error("Expected broadcastSummary() to call a single channel.")
	}

	if summaryFromChan.NewStatus != Failure {
		t.Errorf(
			"Expected Ping() to return the new status. Wanted %s, Got %s",
			Success, summaryFromChan.NewStatus,
		)
	}

	if summaryFromChan.OldStatus != Success {
		t.Errorf(
			"Expected Ping() to return the new status. Wanted %s, Got %s",
			Success, summaryFromChan.NewStatus,
		)
	}

	msSince := time.Since(summaryFromChan.NewStatusTime).Nanoseconds() / 1000000
	if msSince > 5 {
		t.Errorf(
			"Expected the new status time to me from a X ms ago. Wanted %dms, Got %dms",
			5, msSince,
		)
	}

	if p.pingedAt == lastPingedAt {
		t.Errorf(
			"Expected pingedAt to be set with each new ping. Wanted %s, Got %s",
			p.pingedAt, lastPingedAt,
		)
	}
}

func TestStartPinging(t *testing.T) {
	// Used as a reference point to test with time values and keep consistent
	// relative to this single point in time.
	now := time.Now()

	p := &PingTracker{
		nodata:      struct{}{},
		subscribers: map[chan<- ChangeSummary]struct{}{},
	}

	// Manually add our subscriber
	sCh := make(chan ChangeSummary, 1)
	p.subscribers[sCh] = p.nodata

	// Set the date values, such as pingedAt and expiration, so that our startPinging
	// call has to invalidate the success
	p.statusExpiration = time.Second
	p.pingedAt = now.Add(-(time.Second * 2))
	p.lastStatus = Success

	tCh := make(chan time.Time, 1)
	go p.startPinging(tCh)
	tCh <- now

	select {
	case s := <-sCh:
		if s.NewStatus != Failure {
			t.Errorf(
				"Expected startPinging() to broadcast a the fake Status. Wanted %s, Got %s",
				Failure, s.NewStatus,
			)
		}

		if s.NewStatusTime != now.Add(-(time.Second * 2)) {
			t.Errorf(
				"Expected startPinging() to broadcast an offset time. Wanted %s, Got %s",
				now.Add(-(time.Second * 2)), s.NewStatusTime,
			)
		}

	case <-time.After(200 * time.Millisecond):
		t.Error("Expected startPinging() to broadcast in the test. Channel timed out.")
	}

	if p.pingedAt != now {
		t.Errorf(
			"Expected startPinging() to set the pingedAt to the tick time. Wanted %s, Got %s",
			now, p.pingedAt,
		)
	}
}
<commit_msg>klient/remote: increase timeout for pingtracker test<commit_after>package kitepinger

import (
	"errors"
	"fmt"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/koding/kite"
)

type fakePinger struct {
	ReturnStatus Status
}

func (p *fakePinger) Ping() Status {
	return p.ReturnStatus
}

func TestSubscribe(t *testing.T) {
	p := &PingTracker{
		nodata:      struct{}{},
		subscribers: map[chan<- ChangeSummary]struct{}{},
	}

	// broadcast with no subscribers, to make sure we don't panic/etc
	p.broadcastSummary(ChangeSummary{})

	a := make(chan ChangeSummary, 1)
	p.Subscribe(a)

	_, ok := p.subscribers[a]
	if !ok {
		t.Error("Expected Register to store channels")
	}
}

func TestUnsubscribe(t *testing.T) {
	p := &PingTracker{
		nodata:      struct{}{},
		subscribers: map[chan<- ChangeSummary]struct{}{},
		ticker:      time.NewTicker(time.Minute),
		pinging:     true,
	}

	// broadcast with no subscribers, to make sure we don't panic/etc
	p.broadcastSummary(ChangeSummary{})

	a := make(chan ChangeSummary, 1)
	b := make(chan ChangeSummary, 1)
	c := make(chan ChangeSummary, 1)
	p.Subscribe(a)
	p.Subscribe(b)
	p.Subscribe(c)

	p.Unsubscribe(b)

	_, ok := p.subscribers[b]
	if ok {
		t.Error("Expected Unregister to remove channels")
	}

	if len(p.subscribers) != 2 {
		t.Errorf(
			"Expected to leave existing channels alone. Wanted %d, have %d",
			2, len(p.subscribers),
		)
	}

	if p.pinging != true {
		t.Error("Expected Unsubscribe to not call Stop() if there are still subs")
	}

	p.Unsubscribe(a)
	p.Unsubscribe(c)

	if len(p.subscribers) != 0 {
		t.Errorf(
			"Expected to all subs to be gone. Wanted %d, have %d",
			0, len(p.subscribers),
		)
	}

	if p.pinging != false {
		t.Error("Expected Unsubscribe to call Stop() if there are no subs")
	}
}

func TestBroadcastSummary(t *testing.T) {
	p := &PingTracker{
		nodata:      struct{}{},
		subscribers: map[chan<- ChangeSummary]struct{}{},
	}

	// broadcast with no subscribers, to make sure we don't panic/etc
	p.broadcastSummary(ChangeSummary{})

	a := make(chan ChangeSummary, 1)
	p.Subscribe(a)
	p.broadcastSummary(ChangeSummary{
		NewStatus: Success,
	})
	select {
	case s := <-a:
		// Just check if we get the proper value
		if s.NewStatus != Success {
			t.Errorf(
				"Expected broadcastSummary() to send the given Summary to a single channel. Wanted %s, Got %s",
				Success, s.NewStatus,
			)
		}
	case <-time.After(200 * time.Millisecond):
		t.Error("Expected broadcastSummary() to call a single channel.")
	}

	b := make(chan ChangeSummary, 1)
	p.Subscribe(b)

	// Use different data than above
	p.broadcastSummary(ChangeSummary{
		NewStatus: Failure,
	})

	aCount, bCount := 0, 0
	timeout := time.After(200 * time.Millisecond)
	for i := 0; i < 2; i++ {
		select {
		case s := <-a:
			aCount++
			if s.NewStatus != Failure {
				t.Errorf(
					"Expected broadcastSummary() to send the given Summary to a multiple channels. Wanted %s, Got %s",
					Failure, s.NewStatus,
				)
			}
		case s := <-b:
			bCount++
			if s.NewStatus != Failure {
				t.Errorf(
					"Expected broadcastSummary() to send the given Summary to a multiple channels. Wanted %s, Got %s",
					Failure, s.NewStatus,
				)
			}
		case <-timeout:
			t.Error("Timed out waiting for Notify() to call all channels")
		}
	}

	if aCount != 1 {
		t.Errorf(
			"Expected broadcastSummary to call each channel once. Wanted aCount of %d, got %d",
			1, aCount,
		)
	}

	if bCount != 1 {
		t.Errorf(
			"Expected broadcastSummary to call each channel once. Wanted bCount of %d, got %d",
			1, bCount,
		)
	}

	// Using a non-buffered channel here. Please see the explanation below
	// on the line that checks for:
	//
	//		if notBufferedCount != 0
	notBuffered := make(chan ChangeSummary)

	p.Subscribe(notBuffered)
	// Use different data than above
	p.broadcastSummary(ChangeSummary{
		NewStatus: Success,
	})

	aCount, bCount, notBufferedCount := 0, 0, 0
	timeout = time.After(200 * time.Millisecond)
	for i := 0; i < 2; i++ {
		select {
		case s := <-a:
			aCount++
			if s.NewStatus != Success {
				t.Errorf(
					"Expected broadcastSummary() to send the given Summary despite non-receiving channels. Wanted %s, Got %s",
					Success, s.NewStatus,
				)
			}
		case s := <-b:
			bCount++
			if s.NewStatus != Success {
				t.Errorf(
					"Expected broadcastSummary() to send the given Summary despite non-receiving channels. Wanted %s, Got %s",
					Success, s.NewStatus,
				)
			}
		case <-notBuffered:
			notBufferedCount++
		case <-timeout:
			t.Error("Timed out waiting for Notify() to call all channels")
		}
	}

	if aCount != 1 {
		t.Errorf(
			"Expected broadcastSummary to call each channel once. Wanted aCount of %d, got %d",
			1, aCount,
		)
	}

	if bCount != 1 {
		t.Errorf(
			"Expected broadcastSummary to call each channel once. Wanted bCount of %d, got %d",
			1, bCount,
		)
	}

	// Not buffered, means the broadcastSummary will be unable to send to the channel
	// because nothing is *currently* receiving from the channel when
	// broadcastSummary() occurs. broadcastSummary() should not block and wait for it.
	//
	// If broadcastSummary() does block, the receiver (in this test) will never get
	// a chance to listen either.. so really this shouldn't be possible..
	if notBufferedCount != 0 {
		t.Errorf("Expected broadcastSummary() to ignored the notBuffered channel because, at the time of sending, it was not receiving. Wanted %d, Got %d",
			0, notBufferedCount)
	}
}

func TestPing(t *testing.T) {
	r := kite.New("remote", "0.0.0")
	r.Config.DisableAuthentication = true
	ts := httptest.NewServer(r)

	var (
		shouldError bool
		pingCount   int
	)

	r.HandleFunc("kite.ping", func(req *kite.Request) (interface{}, error) {
		pingCount++

		if shouldError {
			return nil, errors.New("Told to error")
		}

		return "pong", nil
	})

	l := kite.New("local", "0.0.0").NewClient(fmt.Sprintf("%s/kite", ts.URL))

	// Connect the local client to the remote
	err := l.Dial()
	if err != nil {
		t.Fatal("Failed to connect to testing Kite", err)
	}

	p := &PingTracker{
		pinger:      NewKitePinger(l),
		nodata:      struct{}{},
		subscribers: map[chan<- ChangeSummary]struct{}{},
	}

	p.Ping()

	if pingCount != 1 {
		t.Errorf(
			"Expected Ping() to ping the remote client. Wanted a PingCount of %d, Got %d",
			1, pingCount,
		)
	}

	// Tell the handler to return an error
	shouldError = true
	p.Ping()
	sFailure := p.GetSummary()

	// Tell the handler to not return an error
	shouldError = false
	p.Ping()
	sSuccess := p.GetSummary()

	if pingCount != 3 {
		t.Errorf(
			"Expected Ping() to ping the remote client. Wanted a PingCount of %d, Got %d",
			3, pingCount,
		)
	}

	if sFailure.Status != Failure {
		t.Errorf(
			"Expected Ping() to return the correct status. Wanted %s, Got %s",
			Failure, sFailure.Status,
		)
	}

	if sSuccess.Status != Success {
		t.Errorf(
			"Expected Ping() to return the correct status. Wanted %s, Got %s",
			Success, sSuccess.Status,
		)
	}

	// Sleeping will increase the time that Ping has held the same status, ie
	// the CurrentSummary.StatusDur duration.
	time.Sleep(time.Millisecond * 100)
	p.Ping()

	// Allow for a 10ms variation in reported runtime.
	msDur := p.GetSummary().StatusDur.Nanoseconds() / 1000000
	if msDur < 90 || msDur > 120 {
		t.Errorf(
			"Expected CurrentSummary.StatusDur to be similar to the elapsed time. Wanted larger than %d and less than %d, Got %d",
			90, 120, msDur,
		)
	}

	// Record the value that KitePinger thinks it pinged at, for a future test
	lastPingedAt := p.pingedAt

	// Make a new subscriber and subscribe, then cause a failed ping
	a := make(chan ChangeSummary, 1)
	p.Subscribe(a)

	shouldError = true
	p.Ping()

	var summaryFromChan ChangeSummary
	select {
	case summaryFromChan = <-a:
	case <-time.After(100 * time.Millisecond):
		t.Error("Expected broadcastSummary() to call a single channel.")
	}

	if summaryFromChan.NewStatus != Failure {
		t.Errorf(
			"Expected Ping() to return the new status. Wanted %s, Got %s",
			Success, summaryFromChan.NewStatus,
		)
	}

	if summaryFromChan.OldStatus != Success {
		t.Errorf(
			"Expected Ping() to return the new status. Wanted %s, Got %s",
			Success, summaryFromChan.NewStatus,
		)
	}

	msSince := time.Since(summaryFromChan.NewStatusTime).Nanoseconds() / 1000000
	if msSince > 10 {
		t.Errorf(
			"Expected the new status time to me from a X ms ago. Wanted %dms, Got %dms",
			10, msSince,
		)
	}

	if p.pingedAt == lastPingedAt {
		t.Errorf(
			"Expected pingedAt to be set with each new ping. Wanted %s, Got %s",
			p.pingedAt, lastPingedAt,
		)
	}
}

func TestStartPinging(t *testing.T) {
	// Used as a reference point to test with time values and keep consistent
	// relative to this single point in time.
	now := time.Now()

	p := &PingTracker{
		nodata:      struct{}{},
		subscribers: map[chan<- ChangeSummary]struct{}{},
	}

	// Manually add our subscriber
	sCh := make(chan ChangeSummary, 1)
	p.subscribers[sCh] = p.nodata

	// Set the date values, such as pingedAt and expiration, so that our startPinging
	// call has to invalidate the success
	p.statusExpiration = time.Second
	p.pingedAt = now.Add(-(time.Second * 2))
	p.lastStatus = Success

	tCh := make(chan time.Time, 1)
	go p.startPinging(tCh)
	tCh <- now

	select {
	case s := <-sCh:
		if s.NewStatus != Failure {
			t.Errorf(
				"Expected startPinging() to broadcast a the fake Status. Wanted %s, Got %s",
				Failure, s.NewStatus,
			)
		}

		if s.NewStatusTime != now.Add(-(time.Second * 2)) {
			t.Errorf(
				"Expected startPinging() to broadcast an offset time. Wanted %s, Got %s",
				now.Add(-(time.Second * 2)), s.NewStatusTime,
			)
		}

	case <-time.After(200 * time.Millisecond):
		t.Error("Expected startPinging() to broadcast in the test. Channel timed out.")
	}

	if p.pingedAt != now {
		t.Errorf(
			"Expected startPinging() to set the pingedAt to the tick time. Wanted %s, Got %s",
			now, p.pingedAt,
		)
	}
}
<|endoftext|>


<commit_before>package message

import (
	"net/http"
	"net/url"
	"socialapi/models"
	"socialapi/workers/api/modules/helpers"

	"github.com/jinzhu/gorm"
)

func Create(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	channelId, err := helpers.GetId(u)
	if err != nil {
		return helpers.NewBadRequestResponse()
	}

	// override message type
	// all of the messages coming from client-side
	// should be marked as POST
	req.Type = models.POST

	if err := req.Save(); err != nil {
		// todo this should be internal server error
		return helpers.NewBadRequestResponse()
	}

	cml := models.NewChannelMessageList()

	// override channel id
	cml.ChannelId = channelId
	cml.MessageId = req.Id
	if err := cml.Save(); err != nil {
		// todo this should be internal server error
		return helpers.NewBadRequestResponse()
	}

	return helpers.NewOKResponse(req)
}

func Delete(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetId(u)
	if err != nil {
		return helpers.NewBadRequestResponse()
	}

	req.Id = id

	if err := req.Delete(); err != nil {
		return helpers.NewBadRequestResponse()
	}
	// yes it is deleted but not removed completely from our system
	return helpers.NewDeletedResponse()
}

func Update(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetId(u)
	if err != nil {
		return helpers.NewBadRequestResponse()
	}
	req.Id = id

	if req.Id == 0 {
		return helpers.NewBadRequestResponse()
	}

	if err := req.Update(); err != nil {
		return helpers.NewBadRequestResponse()
	}

	return helpers.NewOKResponse(req)
}

func Get(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetId(u)
	if err != nil {
		return helpers.NewBadRequestResponse()
	}

	req.Id = id
	if err := req.Fetch(); err != nil {
		if err == gorm.RecordNotFound {
			return helpers.NewNotFoundResponse()
		}
		return helpers.NewBadRequestResponse()
	}

	return helpers.NewOKResponse(req)
}
<commit_msg>Social: change Save method to Create<commit_after>package message

import (
	"net/http"
	"net/url"
	"socialapi/models"
	"socialapi/workers/api/modules/helpers"

	"github.com/jinzhu/gorm"
)

func Create(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	channelId, err := helpers.GetId(u)
	if err != nil {
		return helpers.NewBadRequestResponse()
	}

	// override message type
	// all of the messages coming from client-side
	// should be marked as POST
	req.Type = models.POST

	if err := req.Create(); err != nil {
		// todo this should be internal server error
		return helpers.NewBadRequestResponse()
	}

	cml := models.NewChannelMessageList()

	// override channel id
	cml.ChannelId = channelId
	cml.MessageId = req.Id
	if err := cml.Save(); err != nil {
		// todo this should be internal server error
		return helpers.NewBadRequestResponse()
	}

	return helpers.NewOKResponse(req)
}

func Delete(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetId(u)
	if err != nil {
		return helpers.NewBadRequestResponse()
	}

	req.Id = id

	if err := req.Delete(); err != nil {
		return helpers.NewBadRequestResponse()
	}
	// yes it is deleted but not removed completely from our system
	return helpers.NewDeletedResponse()
}

func Update(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetId(u)
	if err != nil {
		return helpers.NewBadRequestResponse()
	}
	req.Id = id

	if req.Id == 0 {
		return helpers.NewBadRequestResponse()
	}

	if err := req.Update(); err != nil {
		return helpers.NewBadRequestResponse()
	}

	return helpers.NewOKResponse(req)
}

func Get(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetId(u)
	if err != nil {
		return helpers.NewBadRequestResponse()
	}

	req.Id = id
	if err := req.Fetch(); err != nil {
		if err == gorm.RecordNotFound {
			return helpers.NewNotFoundResponse()
		}
		return helpers.NewBadRequestResponse()
	}

	return helpers.NewOKResponse(req)
}
<|endoftext|>


<commit_before>package awsprovider

import (
	"errors"
	"fmt"
	"koding/db/models"
	"koding/db/mongodb"
	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/contexthelper/request"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/dnsstorage"
	"koding/kites/kloud/eventer"
	"koding/kites/kloud/kloud"
	"koding/kites/kloud/kloudctl/command"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/pkg/dnsclient"
	"koding/kites/kloud/userdata"
	"time"

	"github.com/fatih/structs"
	"github.com/koding/kite"
	"github.com/koding/logging"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/ec2"
	"golang.org/x/net/context"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Provider struct {
	DB         *mongodb.MongoDB
	Log        logging.Logger
	Kite       *kite.Kite
	DNSClient  *dnsclient.Route53
	DNSStorage *dnsstorage.MongodbStorage
	Userdata   *userdata.Userdata
}

type Credential struct {
	Id        bson.ObjectId `bson:"_id" json:"-"`
	PublicKey string        `bson:"publicKey"`
	Meta      struct {
		AccessKey string `bson:"access_key"`
		SecretKey string `bson:"secret_key"`
	} `bson:"meta"`
}

func (p *Provider) Machine(ctx context.Context, id string) (interface{}, error) {
	if !bson.IsObjectIdHex(id) {
		return nil, fmt.Errorf("Invalid machine id: %q", id)
	}

	// let's first check if the id exists, because we are going to use
	// findAndModify() and it would be difficult to distinguish if the id
	// really doesn't exist or if there is an assignee which is a different
	// thing. (Because findAndModify() also returns "not found" for the case
	// where the id exist but someone else is the assignee).
	machine := &Machine{}
	if err := p.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.FindId(bson.ObjectIdHex(id)).One(&machine)
	}); err == mgo.ErrNotFound {
		return nil, kloud.NewError(kloud.ErrMachineNotFound)
	}

	req, ok := request.FromContext(ctx)
	if !ok {
		return nil, errors.New("request context is not available")
	}

	if machine.Meta.Region == "" {
		return nil, errors.New("region is not set")
	}

	p.Log.Debug("Using region: %s", machine.Meta.Region)

	if err := p.attachSession(ctx, machine); err != nil {
		return nil, err
	}

	// check for validation and permission
	if err := p.validate(machine, req); err != nil {
		return nil, err
	}

	return machine, nil
}

func (p *Provider) attachSession(ctx context.Context, machine *Machine) error {
	// get user model which contains user ssh keys or the list of users that
	// are allowed to use this machine
	if len(machine.Users) == 0 {
		return errors.New("permitted users list is empty")
	}

	user, err := p.getOwner(machine.Users)
	if err != nil {
		return err
	}

	creds, err := p.credential(machine.Credential)
	if err != nil {
		return err
	}

	client := ec2.New(
		aws.Auth{
			AccessKey: creds.Meta.AccessKey,
			SecretKey: creds.Meta.SecretKey,
		},
		aws.Regions[machine.Meta.Region],
	)

	amazonClient, err := amazon.New(structs.Map(machine.Meta), client)
	if err != nil {
		return fmt.Errorf("koding-amazon err: %s", err)
	}

	// attach user specific log
	machine.Log = p.Log.New(machine.Id.Hex())

	sess := &session.Session{
		DB:         p.DB,
		Kite:       p.Kite,
		DNSClient:  p.DNSClient,
		DNSStorage: p.DNSStorage,
		Userdata:   p.Userdata,
		AWSClient:  amazonClient,
		Log:        machine.Log,
	}

	// we use session a lot of in Machine owned methods, so that's why we
	// assign it to a field for easy access
	machine.Session = sess

	// we pass it also to the context, so other packages, such as plans checker
	// can make use of it.
	ctx = session.NewContext(ctx, sess)

	machine.Username = user.Name
	machine.User = user
	machine.cleanFuncs = make([]func(), 0)

	ev, ok := eventer.FromContext(ctx)
	if ok {
		machine.Session.Eventer = ev
	}

	return nil
}

// getOwner returns the owner of the machine, if it's not found it returns an
// error.
func (p *Provider) getOwner(users []models.Permissions) (*models.User, error) {
	var ownerId bson.ObjectId

	for _, user := range users {
		if user.Sudo && user.Owner {
			ownerId = user.Id
		}
	}

	if !ownerId.Valid() {
		return nil, errors.New("owner not found")
	}

	var user *models.User
	err := p.DB.Run("jUsers", func(c *mgo.Collection) error {
		return c.FindId(users[0].Id).One(&user)
	})

	if err == mgo.ErrNotFound {
		return nil, fmt.Errorf("User with Id not found: %s", ownerId.Hex())
	}
	if err != nil {
		return nil, fmt.Errorf("username lookup error: %v", err)
	}

	return user, nil
}

func (p *Provider) validate(m *Machine, r *kite.Request) error {
	m.Log.Debug("validating for method '%s'", r.Method)

	// give access to kloudctl immediately
	if r.Auth != nil {
		if r.Auth.Key == command.KloudSecretKey {
			return nil
		}
	}

	if r.Username != m.User.Name {
		return errors.New("username is not permitted to make any action")
	}

	// check for user permissions
	if err := p.checkUser(m.User.ObjectId, m.Users); err != nil {
		return err
	}

	if m.User.Status != "confirmed" {
		return kloud.NewError(kloud.ErrUserNotConfirmed)
	}

	return nil
}

// checkUser checks whether the given username is available in the users list
// and has permission
func (p *Provider) checkUser(userId bson.ObjectId, users []models.Permissions) error {
	// check if the incoming user is in the list of permitted user list
	for _, u := range users {
		if userId == u.Id && u.Owner {
			return nil // ok he/she is good to go!
		}
	}

	return fmt.Errorf("permission denied. user not in the list of permitted users")
}

func (p *Provider) credential(publicKey string) (*Credential, error) {
	credential := &Credential{}
	// we neglect errors because credential is optional
	err := p.DB.Run("jCredentialDatas", func(c *mgo.Collection) error {
		return c.Find(bson.M{"publicKey": publicKey}).One(credential)
	})
	if err != nil {
		return nil, err
	}

	return credential, nil
}

func (m *Machine) UpdateState(reason string, state machinestate.State) error {
	m.Log.Debug("Updating state to '%v'", state)
	err := m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.Update(
			bson.M{
				"_id": m.Id,
			},
			bson.M{
				"$set": bson.M{
					"status.state":      state.String(),
					"status.modifiedAt": time.Now().UTC(),
					"status.reason":     reason,
				},
			},
		)
	})

	if err != nil {
		return fmt.Errorf("Couldn't update state to '%s' for document: '%s' err: %s",
			state, m.Id.Hex(), err)
	}

	return nil
}
<commit_msg>kloud: add custom timeout for AWS, as we do for Koding<commit_after>package awsprovider

import (
	"errors"
	"fmt"
	"koding/db/models"
	"koding/db/mongodb"
	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/contexthelper/request"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/dnsstorage"
	"koding/kites/kloud/eventer"
	"koding/kites/kloud/kloud"
	"koding/kites/kloud/kloudctl/command"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/pkg/dnsclient"
	"koding/kites/kloud/userdata"
	"time"

	"github.com/fatih/structs"
	"github.com/koding/kite"
	"github.com/koding/logging"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/ec2"
	"golang.org/x/net/context"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

// include it here to because the library is not exporting it.
var retryingTransport = &aws.ResilientTransport{
	Deadline: func() time.Time {
		return time.Now().Add(60 * time.Second)
	},
	DialTimeout: 45 * time.Second, // this is 10 seconds in original
	MaxTries:    3,
	Wait:        aws.ExpBackoff,
}

type Provider struct {
	DB         *mongodb.MongoDB
	Log        logging.Logger
	Kite       *kite.Kite
	DNSClient  *dnsclient.Route53
	DNSStorage *dnsstorage.MongodbStorage
	Userdata   *userdata.Userdata
}

type Credential struct {
	Id        bson.ObjectId `bson:"_id" json:"-"`
	PublicKey string        `bson:"publicKey"`
	Meta      struct {
		AccessKey string `bson:"access_key"`
		SecretKey string `bson:"secret_key"`
	} `bson:"meta"`
}

func (p *Provider) Machine(ctx context.Context, id string) (interface{}, error) {
	if !bson.IsObjectIdHex(id) {
		return nil, fmt.Errorf("Invalid machine id: %q", id)
	}

	// let's first check if the id exists, because we are going to use
	// findAndModify() and it would be difficult to distinguish if the id
	// really doesn't exist or if there is an assignee which is a different
	// thing. (Because findAndModify() also returns "not found" for the case
	// where the id exist but someone else is the assignee).
	machine := &Machine{}
	if err := p.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.FindId(bson.ObjectIdHex(id)).One(&machine)
	}); err == mgo.ErrNotFound {
		return nil, kloud.NewError(kloud.ErrMachineNotFound)
	}

	req, ok := request.FromContext(ctx)
	if !ok {
		return nil, errors.New("request context is not available")
	}

	if machine.Meta.Region == "" {
		return nil, errors.New("region is not set")
	}

	p.Log.Debug("Using region: %s", machine.Meta.Region)

	if err := p.attachSession(ctx, machine); err != nil {
		return nil, err
	}

	// check for validation and permission
	if err := p.validate(machine, req); err != nil {
		return nil, err
	}

	return machine, nil
}

func (p *Provider) attachSession(ctx context.Context, machine *Machine) error {
	// get user model which contains user ssh keys or the list of users that
	// are allowed to use this machine
	if len(machine.Users) == 0 {
		return errors.New("permitted users list is empty")
	}

	user, err := p.getOwner(machine.Users)
	if err != nil {
		return err
	}

	creds, err := p.credential(machine.Credential)
	if err != nil {
		return err
	}

	client := ec2.NewWithClient(
		aws.Auth{
			AccessKey: creds.Meta.AccessKey,
			SecretKey: creds.Meta.SecretKey,
		},
		aws.Regions[machine.Meta.Region],
		aws.NewClient(retryingTransport),
	)

	amazonClient, err := amazon.New(structs.Map(machine.Meta), client)
	if err != nil {
		return fmt.Errorf("koding-amazon err: %s", err)
	}

	// attach user specific log
	machine.Log = p.Log.New(machine.Id.Hex())

	sess := &session.Session{
		DB:         p.DB,
		Kite:       p.Kite,
		DNSClient:  p.DNSClient,
		DNSStorage: p.DNSStorage,
		Userdata:   p.Userdata,
		AWSClient:  amazonClient,
		Log:        machine.Log,
	}

	// we use session a lot of in Machine owned methods, so that's why we
	// assign it to a field for easy access
	machine.Session = sess

	// we pass it also to the context, so other packages, such as plans checker
	// can make use of it.
	ctx = session.NewContext(ctx, sess)

	machine.Username = user.Name
	machine.User = user
	machine.cleanFuncs = make([]func(), 0)

	ev, ok := eventer.FromContext(ctx)
	if ok {
		machine.Session.Eventer = ev
	}

	return nil
}

// getOwner returns the owner of the machine, if it's not found it returns an
// error.
func (p *Provider) getOwner(users []models.Permissions) (*models.User, error) {
	var ownerId bson.ObjectId

	for _, user := range users {
		if user.Sudo && user.Owner {
			ownerId = user.Id
		}
	}

	if !ownerId.Valid() {
		return nil, errors.New("owner not found")
	}

	var user *models.User
	err := p.DB.Run("jUsers", func(c *mgo.Collection) error {
		return c.FindId(users[0].Id).One(&user)
	})

	if err == mgo.ErrNotFound {
		return nil, fmt.Errorf("User with Id not found: %s", ownerId.Hex())
	}
	if err != nil {
		return nil, fmt.Errorf("username lookup error: %v", err)
	}

	return user, nil
}

func (p *Provider) validate(m *Machine, r *kite.Request) error {
	m.Log.Debug("validating for method '%s'", r.Method)

	// give access to kloudctl immediately
	if r.Auth != nil {
		if r.Auth.Key == command.KloudSecretKey {
			return nil
		}
	}

	if r.Username != m.User.Name {
		return errors.New("username is not permitted to make any action")
	}

	// check for user permissions
	if err := p.checkUser(m.User.ObjectId, m.Users); err != nil {
		return err
	}

	if m.User.Status != "confirmed" {
		return kloud.NewError(kloud.ErrUserNotConfirmed)
	}

	return nil
}

// checkUser checks whether the given username is available in the users list
// and has permission
func (p *Provider) checkUser(userId bson.ObjectId, users []models.Permissions) error {
	// check if the incoming user is in the list of permitted user list
	for _, u := range users {
		if userId == u.Id && u.Owner {
			return nil // ok he/she is good to go!
		}
	}

	return fmt.Errorf("permission denied. user not in the list of permitted users")
}

func (p *Provider) credential(publicKey string) (*Credential, error) {
	credential := &Credential{}
	// we neglect errors because credential is optional
	err := p.DB.Run("jCredentialDatas", func(c *mgo.Collection) error {
		return c.Find(bson.M{"publicKey": publicKey}).One(credential)
	})
	if err != nil {
		return nil, err
	}

	return credential, nil
}

func (m *Machine) UpdateState(reason string, state machinestate.State) error {
	m.Log.Debug("Updating state to '%v'", state)
	err := m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.Update(
			bson.M{
				"_id": m.Id,
			},
			bson.M{
				"$set": bson.M{
					"status.state":      state.String(),
					"status.modifiedAt": time.Now().UTC(),
					"status.reason":     reason,
				},
			},
		)
	})

	if err != nil {
		return fmt.Errorf("Couldn't update state to '%s' for document: '%s' err: %s",
			state, m.Id.Hex(), err)
	}

	return nil
}
<|endoftext|>


<commit_before>package handler

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"time"

	"labix.org/v2/mgo"

	"koding/tools/utils"
	"socialapi/models"
	"socialapi/workers/common/response"
	"strconv"
	"strings"

	"koding/db/mongodb/modelhelper"

	"github.com/PuerkitoBio/throttled"
	"github.com/koding/bongo"
	"github.com/koding/logging"
	kmetrics "github.com/koding/metrics"
	"github.com/koding/redis"
	"github.com/koding/runner"
	tigertonic "github.com/rcrowley/go-tigertonic"

	gometrics "github.com/rcrowley/go-metrics"
)

const (
	PostRequest   = "POST"
	GetRequest    = "GET"
	DeleteRequest = "DELETE"
)

var (
	// TODO allowed origins must be configurable
	cors         = tigertonic.NewCORSBuilder().AddAllowedOrigins("*")
	genericError = errors.New("an error occurred")
)

type Request struct {
	Handler        interface{}
	Endpoint       string
	Type           string
	Name           string
	CollectMetrics bool
	Metrics        *kmetrics.Metrics

	// Securer holds the secure functions for handlers
	Securer interface{}

	// used for external requests
	Params    map[string]string
	Cookie    string
	Cookies   []*http.Cookie
	Body      interface{}
	Headers   map[string]string
	Ratelimit *throttled.Throttler
}

var throttleErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
	runner.MustGetLogger().Error("Throttling error: %s", err)

	writeJSONError(w, genericError)
}

var throttleDenyHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	writeJSONError(w, response.LimitRateExceededError{})
})

// getAccount tries to retrieve account information from incoming request,
// should always return a valid account, not nil
func getAccount(r *http.Request, groupName string) *models.Account {
	cookie, err := r.Cookie("clientId")
	if err != nil {
		return models.NewAccount()
	}

	// if cookie doenst exists return empty account
	if cookie.Value == "" {
		return models.NewAccount()
	}

	session, err := models.Cache.Session.ById(cookie.Value)
	if err != nil {
		return models.NewAccount()
	}

	// if session doesnt have username, return empty account
	if session.Username == "" {
		return models.NewAccount()
	}

	acc, err := makeSureAccount(groupName, session.Username)
	if err != nil {
		if err != bongo.RecordNotFound && err != mgo.ErrNotFound {
			runner.MustGetLogger().Error("Err while getting account: %s, err :%s", session.Username, err.Error())
		}

		return models.NewAccount()
	}

	groupChannel, err := models.Cache.Channel.ByGroupName(groupName)
	if err != nil {
		if err != bongo.RecordNotFound && err != models.ErrGroupNotFound {
			runner.MustGetLogger().Error("Err while getting group channel: %s, err :%s", groupName, err.Error())
		}

		return models.NewAccount()
	}

	if err := makeSureMembership(groupChannel, acc.Id); err != nil {
		runner.MustGetLogger().Error("Err while making sure account: %s, err :%s", groupName, err.Error())
		return models.NewAccount()
	}

	return acc
}

// makeSureAccount checks if incoming account is in postgres, if not creates it
// lazily and sets socialapi id of it in mongo
func makeSureAccount(groupName string, username string) (*models.Account, error) {
	// try to fetch account from postgres
	acc := models.NewAccount()
	err := acc.ByNick(username)
	if err == nil {
		return acc, nil
	}

	if err != bongo.RecordNotFound {
		return acc, err
	}

	// if account is not in postgres, try to create it

	// we need account first
	macc, err := modelhelper.GetAccount(username)
	if err != nil {
		return nil, err
	}

	acc = models.NewAccount() // account is nil, set it
	acc.OldId = macc.Id.Hex()
	acc.Nick = username
	if err := acc.Create(); err != nil {
		return nil, err
	}

	// set it to cache, in case we may need it
	if err := models.Cache.Account.SetToCache(acc); err != nil {
		return nil, err
	}

	// we just created the account in postgres, so update mongo with
	// socialApiId
	s := modelhelper.Selector{
		"profile.nickname": username,
	}
	o := modelhelper.Selector{"$set": modelhelper.Selector{
		"socialApiId": strconv.FormatInt(acc.Id, 10),
	}}
	if err := modelhelper.UpdateAccount(s, o); err != nil {
		return nil, err
	}

	return acc, nil
}

// makeSureMembership checks if the incoming account is a member of the group
// channel that is persisted within incoming session data
func makeSureMembership(groupChannel *models.Channel, accountId int64) error {
	isMember, err := models.Cache.Participant.ByChannelIdAndAccountId(groupChannel.Id, accountId)
	if err != nil {
		return err
	}
	// we dont need to do anything if the user is a member of group channel
	if isMember {
		return nil
	}

	_, err = groupChannel.AddParticipant(accountId)
	if err != nil && err != models.ErrAccountIsAlreadyInTheChannel {
		return err
	}

	return models.Cache.Participant.SetToCache(groupChannel.Id, accountId)
}

func getGroupName(r *http.Request) string {
	const groupName = "koding"

	cookie, err := r.Cookie("clientId")
	if err != nil {
		return groupName
	}

	// if cookie doenst exists return empty account
	if cookie.Value == "" {
		return groupName
	}

	session, err := models.Cache.Session.ById(cookie.Value)
	if err != nil {
		return groupName
	}

	if session.GroupName == "" {
		return groupName
	}

	return session.GroupName
}

const timedOutMsg = `{"description":"request timed out","error":"koding.RequestTimedoutError"}`
const timeoutDuration = time.Second * 30

func Wrapper(r Request) http.Handler {
	handler := r.Handler

	var hHandler http.Handler

	if r.Securer != nil {
		hHandler = Secure(handler, r.Securer, r.Name)
	} else {
		hHandler = tigertonic.Marshaled(handler)
	}

	hHandler = buildHandlerWithTimeTracking(hHandler, r)

	// every request should return under 30 secs, if not return 503 service
	// unavailable
	hHandler = http.TimeoutHandler(hHandler, timeoutDuration, timedOutMsg)

	// set rate limiting if the handler has one
	if r.Ratelimit != nil {
		hHandler = BuildHandlerWithRateLimit(hHandler, r.Ratelimit)
	}

	// create the final handler
	return cors.Build(hHandler)
}

// add request time tracking
func buildHandlerWithTimeTracking(handler http.Handler, r Request) http.Handler {
	var registry gometrics.Registry

	if r.Metrics != nil {
		registry = r.Metrics.Registry
	}

	return tigertonic.Timed(
		handler,
		r.Name,
		registry,
	)
}

func BuildHandlerWithContext(handler http.Handler, redis *redis.RedisSession, log logging.Logger) http.Handler {
	// add context
	return tigertonic.If(
		func(r *http.Request) (http.Header, error) {
			context := models.NewContext(redis, log)
			context.GroupName = getGroupName(r)
			context.Client = &models.Client{
				Account: getAccount(r, context.GroupName),
				IP:      net.ParseIP(utils.GetIpAddress(r)),
			}

			*(tigertonic.Context(r).(*models.Context)) = *context
			return nil, nil
		},
		handler,
	)
}

func BuildHandlerWithRateLimit(handler http.Handler, t *throttled.Throttler) http.Handler {
	throttled.Error = throttleErrorHandler
	throttled.DefaultDeniedHandler = throttleDenyHandler

	return t.Throttle(handler)
}

func DoRequest(request *Request) (*http.Response, error) {
	if request.Cookie != "" {
		request.Cookies = parseCookiesToArray(request.Cookie)
	}

	request.Endpoint = prepareQueryString(request.Endpoint, request.Params)

	client := new(http.Client)

	var byteData io.Reader
	if request.Body != nil {
		body, err := json.Marshal(request.Body)
		if err != nil {
			return nil, err
		}
		byteData = bytes.NewReader(body)
	}

	req, err := http.NewRequest(request.Type, request.Endpoint, byteData)
	req = prepareHeaders(req, request.Headers)

	if err != nil {
		return nil, err
	}

	// Add cookies
	for _, cookie := range request.Cookies {
		req.AddCookie(cookie)
	}

	return client.Do(req)
}

func prepareQueryString(endpoint string, params map[string]string) string {
	if len(params) == 0 {
		return endpoint
	}

	fullPath := fmt.Sprintf("%s?", endpoint)

	for key, value := range params {
		fullPath = fmt.Sprintf("%s%s=%s&", fullPath, key, value)
	}

	return fullPath[0 : len(fullPath)-1]
}

func prepareHeaders(req *http.Request, headers map[string]string) *http.Request {
	newReq := new(http.Request)
	newReq = newReq
	*newReq = *req
	for k, v := range headers {
		newReq.Header.Set(k, v)
	}

	return newReq
}

func parseCookiesToArray(cookie string) []*http.Cookie {
	pairs := strings.Split(cookie, "; ")
	cookies := make([]*http.Cookie, 0)

	if len(pairs) == 0 {
		return cookies
	}

	for _, val := range pairs {
		cp := strings.Split(val, "=")
		if len(cp) != 2 {
			continue
		}

		c := new(http.Cookie)
		c.Name = cp[0]
		c.Value = cp[1]

		cookies = append(cookies, c)
	}

	return cookies
}
<commit_msg>fix another bug on team creation<commit_after>package handler

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"time"

	"labix.org/v2/mgo"

	"koding/tools/utils"
	"socialapi/models"
	"socialapi/workers/common/response"
	"strconv"
	"strings"

	"koding/db/mongodb/modelhelper"

	"github.com/PuerkitoBio/throttled"
	"github.com/koding/bongo"
	"github.com/koding/logging"
	kmetrics "github.com/koding/metrics"
	"github.com/koding/redis"
	"github.com/koding/runner"
	tigertonic "github.com/rcrowley/go-tigertonic"

	gometrics "github.com/rcrowley/go-metrics"
)

const (
	PostRequest   = "POST"
	GetRequest    = "GET"
	DeleteRequest = "DELETE"
)

var (
	// TODO allowed origins must be configurable
	cors         = tigertonic.NewCORSBuilder().AddAllowedOrigins("*")
	genericError = errors.New("an error occurred")
)

type Request struct {
	Handler        interface{}
	Endpoint       string
	Type           string
	Name           string
	CollectMetrics bool
	Metrics        *kmetrics.Metrics

	// Securer holds the secure functions for handlers
	Securer interface{}

	// used for external requests
	Params    map[string]string
	Cookie    string
	Cookies   []*http.Cookie
	Body      interface{}
	Headers   map[string]string
	Ratelimit *throttled.Throttler
}

var throttleErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
	runner.MustGetLogger().Error("Throttling error: %s", err)

	writeJSONError(w, genericError)
}

var throttleDenyHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	writeJSONError(w, response.LimitRateExceededError{})
})

// getAccount tries to retrieve account information from incoming request,
// should always return a valid account, not nil
func getAccount(r *http.Request, groupName string) *models.Account {
	cookie, err := r.Cookie("clientId")
	if err != nil {
		return models.NewAccount()
	}

	// if cookie doenst exists return empty account
	if cookie.Value == "" {
		return models.NewAccount()
	}

	session, err := models.Cache.Session.ById(cookie.Value)
	if err != nil {
		return models.NewAccount()
	}

	// if session doesnt have username, return empty account
	if session.Username == "" {
		return models.NewAccount()
	}

	acc, err := makeSureAccount(groupName, session.Username)
	if err != nil {
		if err != bongo.RecordNotFound && err != mgo.ErrNotFound {
			runner.MustGetLogger().Error("Err while getting account: %s, err :%s", session.Username, err.Error())
		}

		return models.NewAccount()
	}

	groupChannel, err := models.Cache.Channel.ByGroupName(groupName)
	if err != nil {
		if err != bongo.RecordNotFound && err != models.ErrGroupNotFound {
			runner.MustGetLogger().Error("Err while getting group channel: %s, err :%s", groupName, err.Error())

			return models.NewAccount()
		}

		// for creating the group channel for the first time, we should not return
		// here with empty account
		return acc
	}

	if err := makeSureMembership(groupChannel, acc.Id); err != nil {
		runner.MustGetLogger().Error("Err while making sure account: %s, err :%s", groupName, err.Error())
		return models.NewAccount()
	}

	return acc
}

// makeSureAccount checks if incoming account is in postgres, if not creates it
// lazily and sets socialapi id of it in mongo
func makeSureAccount(groupName string, username string) (*models.Account, error) {
	// try to fetch account from postgres
	acc := models.NewAccount()
	err := acc.ByNick(username)
	if err == nil {
		return acc, nil
	}

	if err != bongo.RecordNotFound {
		return acc, err
	}

	// if account is not in postgres, try to create it

	// we need account first
	macc, err := modelhelper.GetAccount(username)
	if err != nil {
		return nil, err
	}

	acc = models.NewAccount() // account is nil, set it
	acc.OldId = macc.Id.Hex()
	acc.Nick = username
	if err := acc.Create(); err != nil {
		return nil, err
	}

	// set it to cache, in case we may need it
	if err := models.Cache.Account.SetToCache(acc); err != nil {
		return nil, err
	}

	// we just created the account in postgres, so update mongo with
	// socialApiId
	s := modelhelper.Selector{
		"profile.nickname": username,
	}
	o := modelhelper.Selector{"$set": modelhelper.Selector{
		"socialApiId": strconv.FormatInt(acc.Id, 10),
	}}
	if err := modelhelper.UpdateAccount(s, o); err != nil {
		return nil, err
	}

	return acc, nil
}

// makeSureMembership checks if the incoming account is a member of the group
// channel that is persisted within incoming session data
func makeSureMembership(groupChannel *models.Channel, accountId int64) error {
	isMember, err := models.Cache.Participant.ByChannelIdAndAccountId(groupChannel.Id, accountId)
	if err != nil {
		return err
	}
	// we dont need to do anything if the user is a member of group channel
	if isMember {
		return nil
	}

	_, err = groupChannel.AddParticipant(accountId)
	if err != nil && err != models.ErrAccountIsAlreadyInTheChannel {
		return err
	}

	return models.Cache.Participant.SetToCache(groupChannel.Id, accountId)
}

func getGroupName(r *http.Request) string {
	const groupName = "koding"

	cookie, err := r.Cookie("clientId")
	if err != nil {
		return groupName
	}

	// if cookie doenst exists return empty account
	if cookie.Value == "" {
		return groupName
	}

	session, err := models.Cache.Session.ById(cookie.Value)
	if err != nil {
		return groupName
	}

	if session.GroupName == "" {
		return groupName
	}

	return session.GroupName
}

const timedOutMsg = `{"description":"request timed out","error":"koding.RequestTimedoutError"}`
const timeoutDuration = time.Second * 30

func Wrapper(r Request) http.Handler {
	handler := r.Handler

	var hHandler http.Handler

	if r.Securer != nil {
		hHandler = Secure(handler, r.Securer, r.Name)
	} else {
		hHandler = tigertonic.Marshaled(handler)
	}

	hHandler = buildHandlerWithTimeTracking(hHandler, r)

	// every request should return under 30 secs, if not return 503 service
	// unavailable
	hHandler = http.TimeoutHandler(hHandler, timeoutDuration, timedOutMsg)

	// set rate limiting if the handler has one
	if r.Ratelimit != nil {
		hHandler = BuildHandlerWithRateLimit(hHandler, r.Ratelimit)
	}

	// create the final handler
	return cors.Build(hHandler)
}

// add request time tracking
func buildHandlerWithTimeTracking(handler http.Handler, r Request) http.Handler {
	var registry gometrics.Registry

	if r.Metrics != nil {
		registry = r.Metrics.Registry
	}

	return tigertonic.Timed(
		handler,
		r.Name,
		registry,
	)
}

func BuildHandlerWithContext(handler http.Handler, redis *redis.RedisSession, log logging.Logger) http.Handler {
	// add context
	return tigertonic.If(
		func(r *http.Request) (http.Header, error) {
			context := models.NewContext(redis, log)
			context.GroupName = getGroupName(r)
			context.Client = &models.Client{
				Account: getAccount(r, context.GroupName),
				IP:      net.ParseIP(utils.GetIpAddress(r)),
			}

			*(tigertonic.Context(r).(*models.Context)) = *context
			return nil, nil
		},
		handler,
	)
}

func BuildHandlerWithRateLimit(handler http.Handler, t *throttled.Throttler) http.Handler {
	throttled.Error = throttleErrorHandler
	throttled.DefaultDeniedHandler = throttleDenyHandler

	return t.Throttle(handler)
}

func DoRequest(request *Request) (*http.Response, error) {
	if request.Cookie != "" {
		request.Cookies = parseCookiesToArray(request.Cookie)
	}

	request.Endpoint = prepareQueryString(request.Endpoint, request.Params)

	client := new(http.Client)

	var byteData io.Reader
	if request.Body != nil {
		body, err := json.Marshal(request.Body)
		if err != nil {
			return nil, err
		}
		byteData = bytes.NewReader(body)
	}

	req, err := http.NewRequest(request.Type, request.Endpoint, byteData)
	req = prepareHeaders(req, request.Headers)

	if err != nil {
		return nil, err
	}

	// Add cookies
	for _, cookie := range request.Cookies {
		req.AddCookie(cookie)
	}

	return client.Do(req)
}

func prepareQueryString(endpoint string, params map[string]string) string {
	if len(params) == 0 {
		return endpoint
	}

	fullPath := fmt.Sprintf("%s?", endpoint)

	for key, value := range params {
		fullPath = fmt.Sprintf("%s%s=%s&", fullPath, key, value)
	}

	return fullPath[0 : len(fullPath)-1]
}

func prepareHeaders(req *http.Request, headers map[string]string) *http.Request {
	newReq := new(http.Request)
	newReq = newReq
	*newReq = *req
	for k, v := range headers {
		newReq.Header.Set(k, v)
	}

	return newReq
}

func parseCookiesToArray(cookie string) []*http.Cookie {
	pairs := strings.Split(cookie, "; ")
	cookies := make([]*http.Cookie, 0)

	if len(pairs) == 0 {
		return cookies
	}

	for _, val := range pairs {
		cp := strings.Split(val, "=")
		if len(cp) != 2 {
			continue
		}

		c := new(http.Cookie)
		c.Name = cp[0]
		c.Value = cp[1]

		cookies = append(cookies, c)
	}

	return cookies
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"google.golang.org/api/cloudresourcemanager/v1"
	"os"
	"reflect"
	"testing"
)

var DENIED_ORG_POLICIES = []string{
	"maps-ios-backend.googleapis.com",
	"placesios.googleapis.com",
}

func TestAccGoogleOrganizationPolicy_boolean_enforced(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_boolean(org, true),
				Check:  testAccCheckGoogleOrganizationBooleanPolicy("bool", true),
			},
		},
	})

}

func TestAccGoogleOrganizationPolicy_boolean_notEnforced(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_boolean(org, false),
				Check:  testAccCheckGoogleOrganizationBooleanPolicy("bool", false),
			},
		},
	})
}

func TestAccGoogleOrganizationPolicy_boolean_update(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_boolean(org, true),
				Check:  testAccCheckGoogleOrganizationBooleanPolicy("bool", true),
			},
			{
				Config: testAccGoogleOrganizationPolicy_boolean(org, false),
				Check:  testAccCheckGoogleOrganizationBooleanPolicy("bool", false),
			},
			{
				Config: testAccGoogleOrganizationPolicy_boolean(org, true),
				Check:  testAccCheckGoogleOrganizationBooleanPolicy("bool", true),
			},
		},
	})
}

func TestAccGoogleOrganizationPolicy_list_allowAll(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_list_allowAll(org),
				Check:  testAccCheckGoogleOrganizationListPolicyAll("list", "ALLOW"),
			},
		},
	})
}

func TestAccGoogleOrganizationPolicy_list_allowSome(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")
	project := getTestProjectFromEnv()

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_list_allowSome(org, project),
				Check:  testAccCheckGoogleOrganizationListPolicyAllowedValues("list", []string{project}),
			},
		},
	})
}

func TestAccGoogleOrganizationPolicy_list_denySome(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_list_denySome(org),
				Check:  testAccCheckGoogleOrganizationListPolicyDeniedValues("list", DENIED_ORG_POLICIES),
			},
		},
	})
}

func TestAccGoogleOrganizationPolicy_list_update(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_list_allowAll(org),
				Check:  testAccCheckGoogleOrganizationListPolicyAll("list", "ALLOW"),
			},
			{
				Config: testAccGoogleOrganizationPolicy_list_denySome(org),
				Check:  testAccCheckGoogleOrganizationListPolicyDeniedValues("list", DENIED_ORG_POLICIES),
			},
		},
	})
}

func testAccCheckGoogleOrganizationPolicyDestroy(s *terraform.State) error {
	config := testAccProvider.Meta().(*Config)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "google_organization_policy" {
			continue
		}

		org := "organizations/" + rs.Primary.Attributes["org_id"]
		constraint := canonicalOrgPolicyConstraint(rs.Primary.Attributes["constraint"])
		policy, err := config.clientResourceManager.Organizations.GetOrgPolicy(org, &cloudresourcemanager.GetOrgPolicyRequest{
			Constraint: constraint,
		}).Do()

		if err != nil {
			return err
		}

		if policy.ListPolicy != nil || policy.BooleanPolicy != nil {
			return fmt.Errorf("Org policy with constraint '%s' hasn't been cleared", constraint)
		}
	}
	return nil
}

func testAccCheckGoogleOrganizationBooleanPolicy(n string, enforced bool) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		policy, err := getGoogleOrganizationPolicyTestResource(s, n)
		if err != nil {
			return err
		}

		if policy.BooleanPolicy.Enforced != enforced {
			return fmt.Errorf("Expected boolean policy enforcement to be '%t', got '%t'", enforced, policy.BooleanPolicy.Enforced)
		}

		return nil
	}
}

func testAccCheckGoogleOrganizationListPolicyAll(n, policyType string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		policy, err := getGoogleOrganizationPolicyTestResource(s, n)
		if err != nil {
			return err
		}

		if len(policy.ListPolicy.AllowedValues) > 0 || len(policy.ListPolicy.DeniedValues) > 0 {
			return fmt.Errorf("The `values` field shouldn't be set")
		}

		if policy.ListPolicy.AllValues != policyType {
			return fmt.Errorf("Expected the list policy to '%s' all values, got '%s'", policyType, policy.ListPolicy.AllValues)
		}

		return nil
	}
}

func testAccCheckGoogleOrganizationListPolicyAllowedValues(n string, values []string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		policy, err := getGoogleOrganizationPolicyTestResource(s, n)
		if err != nil {
			return err
		}

		if !reflect.DeepEqual(policy.ListPolicy.AllowedValues, values) {
			return fmt.Errorf("Expected the list policy to allow '%s', instead allowed '%s'", values, policy.ListPolicy.AllowedValues)
		}

		return nil
	}
}

func testAccCheckGoogleOrganizationListPolicyDeniedValues(n string, values []string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		policy, err := getGoogleOrganizationPolicyTestResource(s, n)
		if err != nil {
			return err
		}

		if !reflect.DeepEqual(policy.ListPolicy.DeniedValues, values) {
			return fmt.Errorf("Expected the list policy to deny '%s', instead denied '%s'", values, policy.ListPolicy.DeniedValues)
		}

		return nil
	}
}

func getGoogleOrganizationPolicyTestResource(s *terraform.State, n string) (*cloudresourcemanager.OrgPolicy, error) {
	rn := "google_organization_policy." + n
	rs, ok := s.RootModule().Resources[rn]
	if !ok {
		return nil, fmt.Errorf("Not found: %s", rn)
	}

	if rs.Primary.ID == "" {
		return nil, fmt.Errorf("No ID is set")
	}

	config := testAccProvider.Meta().(*Config)

	return config.clientResourceManager.Organizations.GetOrgPolicy("organizations/"+rs.Primary.Attributes["org_id"], &cloudresourcemanager.GetOrgPolicyRequest{
		Constraint: rs.Primary.Attributes["constraint"],
	}).Do()
}

func testAccGoogleOrganizationPolicy_boolean(org string, enforced bool) string {
	return fmt.Sprintf(`
resource "google_organization_policy" "bool" {
	org_id = "%s"
	constraint = "constraints/compute.disableSerialPortAccess"

	boolean_policy {
		enforced = %t
	}
}
`, org, enforced)
}

func testAccGoogleOrganizationPolicy_list_allowAll(org string) string {
	return fmt.Sprintf(`
resource "google_organization_policy" "list" {
	org_id = "%s"
	constraint = "constraints/serviceuser.services"

	list_policy {
		allow {
			all = true
		}
	}
}
`, org)
}

func testAccGoogleOrganizationPolicy_list_allowSome(org, project string) string {
	return fmt.Sprintf(`
resource "google_organization_policy" "list" {
	org_id = "%s"
	constraint = "constraints/compute.trustedImageProjects"

	list_policy {
		allow {
			values = [
				"%s",
			]
		}
  }
}
`, org, project)
}

func testAccGoogleOrganizationPolicy_list_denySome(org string) string {
	return fmt.Sprintf(`
resource "google_organization_policy" "list" {
	org_id = "%s"
 	constraint = "serviceuser.services"

  	list_policy {
		deny {
			values = [
				"maps-ios-backend.googleapis.com",
				"placesios.googleapis.com",
			]
		}
	}
}
`, org)
}
<commit_msg>Serialize org policy boolean tests to avoid flakiness when tests are run in parallel (#708)<commit_after>package google

import (
	"fmt"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"google.golang.org/api/cloudresourcemanager/v1"
	"os"
	"reflect"
	"testing"
)

var DENIED_ORG_POLICIES = []string{
	"maps-ios-backend.googleapis.com",
	"placesios.googleapis.com",
}

func TestAccGoogleOrganizationPolicy_boolean(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				// Test creation of an enforced boolean policy
				Config: testAccGoogleOrganizationPolicy_boolean(org, true),
				Check:  testAccCheckGoogleOrganizationBooleanPolicy("bool", true),
			},
			{
				// Test update from enforced to not
				Config: testAccGoogleOrganizationPolicy_boolean(org, false),
				Check:  testAccCheckGoogleOrganizationBooleanPolicy("bool", false),
			},
			{
				Config:  " ",
				Destroy: true,
			},
			{
				// Test creation of a not enforced boolean policy
				Config: testAccGoogleOrganizationPolicy_boolean(org, false),
				Check:  testAccCheckGoogleOrganizationBooleanPolicy("bool", false),
			},
			{
				// Test update from not enforced to enforced
				Config: testAccGoogleOrganizationPolicy_boolean(org, true),
				Check:  testAccCheckGoogleOrganizationBooleanPolicy("bool", true),
			},
		},
	})

}

func TestAccGoogleOrganizationPolicy_list_allowAll(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_list_allowAll(org),
				Check:  testAccCheckGoogleOrganizationListPolicyAll("list", "ALLOW"),
			},
		},
	})
}

func TestAccGoogleOrganizationPolicy_list_allowSome(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")
	project := getTestProjectFromEnv()

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_list_allowSome(org, project),
				Check:  testAccCheckGoogleOrganizationListPolicyAllowedValues("list", []string{project}),
			},
		},
	})
}

func TestAccGoogleOrganizationPolicy_list_denySome(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_list_denySome(org),
				Check:  testAccCheckGoogleOrganizationListPolicyDeniedValues("list", DENIED_ORG_POLICIES),
			},
		},
	})
}

func TestAccGoogleOrganizationPolicy_list_update(t *testing.T) {
	t.Parallel()

	skipIfEnvNotSet(t, "GOOGLE_ORG")
	org := os.Getenv("GOOGLE_ORG")

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGoogleOrganizationPolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGoogleOrganizationPolicy_list_allowAll(org),
				Check:  testAccCheckGoogleOrganizationListPolicyAll("list", "ALLOW"),
			},
			{
				Config: testAccGoogleOrganizationPolicy_list_denySome(org),
				Check:  testAccCheckGoogleOrganizationListPolicyDeniedValues("list", DENIED_ORG_POLICIES),
			},
		},
	})
}

func testAccCheckGoogleOrganizationPolicyDestroy(s *terraform.State) error {
	config := testAccProvider.Meta().(*Config)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "google_organization_policy" {
			continue
		}

		org := "organizations/" + rs.Primary.Attributes["org_id"]
		constraint := canonicalOrgPolicyConstraint(rs.Primary.Attributes["constraint"])
		policy, err := config.clientResourceManager.Organizations.GetOrgPolicy(org, &cloudresourcemanager.GetOrgPolicyRequest{
			Constraint: constraint,
		}).Do()

		if err != nil {
			return err
		}

		if policy.ListPolicy != nil || policy.BooleanPolicy != nil {
			return fmt.Errorf("Org policy with constraint '%s' hasn't been cleared", constraint)
		}
	}
	return nil
}

func testAccCheckGoogleOrganizationBooleanPolicy(n string, enforced bool) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		policy, err := getGoogleOrganizationPolicyTestResource(s, n)
		if err != nil {
			return err
		}

		if policy.BooleanPolicy.Enforced != enforced {
			return fmt.Errorf("Expected boolean policy enforcement to be '%t', got '%t'", enforced, policy.BooleanPolicy.Enforced)
		}

		return nil
	}
}

func testAccCheckGoogleOrganizationListPolicyAll(n, policyType string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		policy, err := getGoogleOrganizationPolicyTestResource(s, n)
		if err != nil {
			return err
		}

		if len(policy.ListPolicy.AllowedValues) > 0 || len(policy.ListPolicy.DeniedValues) > 0 {
			return fmt.Errorf("The `values` field shouldn't be set")
		}

		if policy.ListPolicy.AllValues != policyType {
			return fmt.Errorf("Expected the list policy to '%s' all values, got '%s'", policyType, policy.ListPolicy.AllValues)
		}

		return nil
	}
}

func testAccCheckGoogleOrganizationListPolicyAllowedValues(n string, values []string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		policy, err := getGoogleOrganizationPolicyTestResource(s, n)
		if err != nil {
			return err
		}

		if !reflect.DeepEqual(policy.ListPolicy.AllowedValues, values) {
			return fmt.Errorf("Expected the list policy to allow '%s', instead allowed '%s'", values, policy.ListPolicy.AllowedValues)
		}

		return nil
	}
}

func testAccCheckGoogleOrganizationListPolicyDeniedValues(n string, values []string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		policy, err := getGoogleOrganizationPolicyTestResource(s, n)
		if err != nil {
			return err
		}

		if !reflect.DeepEqual(policy.ListPolicy.DeniedValues, values) {
			return fmt.Errorf("Expected the list policy to deny '%s', instead denied '%s'", values, policy.ListPolicy.DeniedValues)
		}

		return nil
	}
}

func getGoogleOrganizationPolicyTestResource(s *terraform.State, n string) (*cloudresourcemanager.OrgPolicy, error) {
	rn := "google_organization_policy." + n
	rs, ok := s.RootModule().Resources[rn]
	if !ok {
		return nil, fmt.Errorf("Not found: %s", rn)
	}

	if rs.Primary.ID == "" {
		return nil, fmt.Errorf("No ID is set")
	}

	config := testAccProvider.Meta().(*Config)

	return config.clientResourceManager.Organizations.GetOrgPolicy("organizations/"+rs.Primary.Attributes["org_id"], &cloudresourcemanager.GetOrgPolicyRequest{
		Constraint: rs.Primary.Attributes["constraint"],
	}).Do()
}

func testAccGoogleOrganizationPolicy_boolean(org string, enforced bool) string {
	return fmt.Sprintf(`
resource "google_organization_policy" "bool" {
	org_id = "%s"
	constraint = "constraints/compute.disableSerialPortAccess"

	boolean_policy {
		enforced = %t
	}
}
`, org, enforced)
}

func testAccGoogleOrganizationPolicy_list_allowAll(org string) string {
	return fmt.Sprintf(`
resource "google_organization_policy" "list" {
	org_id = "%s"
	constraint = "constraints/serviceuser.services"

	list_policy {
		allow {
			all = true
		}
	}
}
`, org)
}

func testAccGoogleOrganizationPolicy_list_allowSome(org, project string) string {
	return fmt.Sprintf(`
resource "google_organization_policy" "list" {
	org_id = "%s"
	constraint = "constraints/compute.trustedImageProjects"

	list_policy {
		allow {
			values = [
				"%s",
			]
		}
  }
}
`, org, project)
}

func testAccGoogleOrganizationPolicy_list_denySome(org string) string {
	return fmt.Sprintf(`
resource "google_organization_policy" "list" {
	org_id = "%s"
 	constraint = "serviceuser.services"

  	list_policy {
		deny {
			values = [
				"maps-ios-backend.googleapis.com",
				"placesios.googleapis.com",
			]
		}
	}
}
`, org)
}
<|endoftext|>


<commit_before>package main

import (
	"ApacheLog2DB/core"
	"database/sql"
	"flag"
	"fmt"
	"github.com/davecheney/profile"
	_ "github.com/mattn/go-sqlite3"
	"os"
)

func usage() {
	fmt.Println("USAGE: log2db [OPTION]... SOURCE DEST")
	flag.PrintDefaults()
}

func main() {
	flag.Usage = func() { usage() }
	export := flag.Bool("e", false, "Export from SOURCE to DEST")
	flag.Parse()

	args := flag.Args()

	if len(args) != 2 {
		usage()
		os.Exit(1)
	}

	var db *sql.DB
	var err error
	reader := os.Stdin
	writer := os.Stdout

	if !*export {
		if !(args[0] == "-" || args[0] == "--") {
			reader, err = os.Open(args[0])
			defer reader.Close()
		}
	} else {
		if args[0] == "-" || args[0] == "--" {
			fmt.Fprintf(os.Stderr, "Input file must be a db string")
			os.Exit(1)
		}
		db, err = sql.Open("sqlite3", args[0])
		defer db.Close()
	}

	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}

	if !*export {
		if args[1] == "-" || args[1] == "--" {
			fmt.Fprintf(os.Stderr, "Output file must be a db string")
			os.Exit(1)
		}
		db, err = sql.Open("sqlite3", args[1])
		defer db.Close()
	} else {
		if !(args[1] == "-" || args[1] == "--") {
			writer, err = os.Create(args[1])
			defer writer.Close()
		}
	}

	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}

	if *export {
		core.ExportLog(db, writer)
	} else {
		core.ImportLog(reader, db)
	}
	os.Exit(0)
}
<commit_msg>disable profiling<commit_after>package main

import (
	"ApacheLog2DB/core"
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"os"
)

func usage() {
	fmt.Println("USAGE: log2db [OPTION]... SOURCE DEST")
	flag.PrintDefaults()
}

func main() {
	flag.Usage = func() { usage() }
	export := flag.Bool("e", false, "Export from SOURCE to DEST")
	flag.Parse()

	args := flag.Args()

	if len(args) != 2 {
		usage()
		os.Exit(1)
	}

	var db *sql.DB
	var err error
	reader := os.Stdin
	writer := os.Stdout

	if !*export {
		if !(args[0] == "-" || args[0] == "--") {
			reader, err = os.Open(args[0])
			defer reader.Close()
		}
	} else {
		if args[0] == "-" || args[0] == "--" {
			fmt.Fprintf(os.Stderr, "Input file must be a db string")
			os.Exit(1)
		}
		db, err = sql.Open("sqlite3", args[0])
		defer db.Close()
	}

	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}

	if !*export {
		if args[1] == "-" || args[1] == "--" {
			fmt.Fprintf(os.Stderr, "Output file must be a db string")
			os.Exit(1)
		}
		db, err = sql.Open("sqlite3", args[1])
		defer db.Close()
	} else {
		if !(args[1] == "-" || args[1] == "--") {
			writer, err = os.Create(args[1])
			defer writer.Close()
		}
	}

	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}

	if *export {
		core.ExportLog(db, writer)
	} else {
		core.ImportLog(reader, db)
	}
	os.Exit(0)
}
<|endoftext|>