<commit_before>// Copyright 2017-2021 The Usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sshkey

import (
	"github.com/sacloud/usacloud/pkg/cli"
	"github.com/sacloud/usacloud/pkg/cmd/cflag"
	"github.com/sacloud/usacloud/pkg/cmd/core"
	"github.com/sacloud/usacloud/pkg/cmd/examples"
)

var createCommand = &core.Command{
	Name:     "create",
	Category: "basic",
	Order:    20,

	ColumnDefs: defaultColumnDefs,

	ParameterInitializer: func() interface{} {
		return newCreateParameter()
	},
}

type createParameter struct {
	cflag.CommonParameter  `cli:",squash" mapconv:"-"`
	cflag.ConfirmParameter `cli:",squash" mapconv:"-"`
	cflag.OutputParameter  `cli:",squash" mapconv:"-"`

	cflag.NameParameter `cli:",squash" mapconv:",squash"`
	cflag.DescParameter `cli:",squash" mapconv:",squash"`
	PublicKey           string `validate:"required"`
}

func newCreateParameter() *createParameter {
	return &createParameter{}
}

func init() {
	Resource.AddCommand(createCommand)
}

func (p *createParameter) ExampleParameters(ctx cli.Context) interface{} {
	return &createParameter{
		NameParameter: examples.Name,
		DescParameter: examples.Description,
		PublicKey:     "/path/to/your/public/key | ssh-rsa ...",
	}
}
<commit_msg>Added path_or_content filter to sshkey create parameter<commit_after>// Copyright 2017-2021 The Usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sshkey

import (
	"github.com/sacloud/usacloud/pkg/cli"
	"github.com/sacloud/usacloud/pkg/cmd/cflag"
	"github.com/sacloud/usacloud/pkg/cmd/core"
	"github.com/sacloud/usacloud/pkg/cmd/examples"
)

var createCommand = &core.Command{
	Name:     "create",
	Category: "basic",
	Order:    20,

	ColumnDefs: defaultColumnDefs,

	ParameterInitializer: func() interface{} {
		return newCreateParameter()
	},
}

type createParameter struct {
	cflag.CommonParameter  `cli:",squash" mapconv:"-"`
	cflag.ConfirmParameter `cli:",squash" mapconv:"-"`
	cflag.OutputParameter  `cli:",squash" mapconv:"-"`

	cflag.NameParameter `cli:",squash" mapconv:",squash"`
	cflag.DescParameter `cli:",squash" mapconv:",squash"`
	PublicKey           string `mapconv:",filters=path_or_content" validate:"required"`
}

func newCreateParameter() *createParameter {
	return &createParameter{}
}

func init() {
	Resource.AddCommand(createCommand)
}

func (p *createParameter) ExampleParameters(ctx cli.Context) interface{} {
	return &createParameter{
		NameParameter: examples.Name,
		DescParameter: examples.Description,
		PublicKey:     "/path/to/your/public/key | ssh-rsa ...",
	}
}
<|endoftext|>


<commit_before>package jwtmanager

import (
	"testing"

	"github.com/vouch/vouch-proxy/pkg/cfg"
	"github.com/vouch/vouch-proxy/pkg/structs"

	// log "github.com/Sirupsen/logrus"
	log "github.com/Sirupsen/logrus"
	"github.com/stretchr/testify/assert"
)

var (
	u1 = structs.User{
		Username:      "test@testing.com",
		EmailVerified: true,
		Name:          "Test Name",
	}

	lc VouchClaims
)

func init() {
	// log.SetLevel(log.DebugLevel)

	lc = VouchClaims{
		u1.Username,
		Sites,
		StandardClaims,
	}
}

func TestCreateUserTokenStringAndParseToUsername(t *testing.T) {

	uts := CreateUserTokenString(u1)
	assert.NotEmpty(t, uts)

	utsParsed, err := ParseTokenString(uts)
	if utsParsed == nil || err != nil {
		t.Error(err)
	} else {
		log.Debugf("test parsed token string %v", utsParsed)
		ptUsername, _ := PTokenToUsername(utsParsed)
		assert.Equal(t, u1.Username, ptUsername)
	}

}

func TestClaims(t *testing.T) {
	cfg.ParseConfig()

	log.Debugf("jwt config %s %d", string(cfg.Cfg.JWT.Secret), cfg.Cfg.JWT.MaxAge)
	assert.NotEmpty(t, cfg.Cfg.JWT.Secret)
	assert.NotEmpty(t, cfg.Cfg.JWT.MaxAge)

	// now := time.Now()
	// d := time.Duration(ExpiresAtMinutes) * time.Minute
	// log.Infof("lc d %s", d.String())
	// lc.StandardClaims.ExpiresAt = now.Add(time.Duration(ExpiresAtMinutes) * time.Minute).Unix()
	// log.Infof("lc expiresAt %d", now.Unix()-lc.StandardClaims.ExpiresAt)
	uts := CreateUserTokenString(u1)
	utsParsed, _ := ParseTokenString(uts)
	assert.True(t, SiteInToken("naga.bnf.net", utsParsed))

}
<commit_msg>Fix test by removing EmailVerified since it got moved to Google User struct<commit_after>package jwtmanager

import (
	"testing"

	"github.com/vouch/vouch-proxy/pkg/cfg"
	"github.com/vouch/vouch-proxy/pkg/structs"

	// log "github.com/Sirupsen/logrus"
	log "github.com/Sirupsen/logrus"
	"github.com/stretchr/testify/assert"
)

var (
	u1 = structs.User{
		Username: "test@testing.com",
		Name:     "Test Name",
	}

	lc VouchClaims
)

func init() {
	// log.SetLevel(log.DebugLevel)

	lc = VouchClaims{
		u1.Username,
		Sites,
		StandardClaims,
	}
}

func TestCreateUserTokenStringAndParseToUsername(t *testing.T) {

	uts := CreateUserTokenString(u1)
	assert.NotEmpty(t, uts)

	utsParsed, err := ParseTokenString(uts)
	if utsParsed == nil || err != nil {
		t.Error(err)
	} else {
		log.Debugf("test parsed token string %v", utsParsed)
		ptUsername, _ := PTokenToUsername(utsParsed)
		assert.Equal(t, u1.Username, ptUsername)
	}

}

func TestClaims(t *testing.T) {
	cfg.ParseConfig()

	log.Debugf("jwt config %s %d", string(cfg.Cfg.JWT.Secret), cfg.Cfg.JWT.MaxAge)
	assert.NotEmpty(t, cfg.Cfg.JWT.Secret)
	assert.NotEmpty(t, cfg.Cfg.JWT.MaxAge)

	// now := time.Now()
	// d := time.Duration(ExpiresAtMinutes) * time.Minute
	// log.Infof("lc d %s", d.String())
	// lc.StandardClaims.ExpiresAt = now.Add(time.Duration(ExpiresAtMinutes) * time.Minute).Unix()
	// log.Infof("lc expiresAt %d", now.Unix()-lc.StandardClaims.ExpiresAt)
	uts := CreateUserTokenString(u1)
	utsParsed, _ := ParseTokenString(uts)
	assert.True(t, SiteInToken("naga.bnf.net", utsParsed))

}
<|endoftext|>


<commit_before>package siri

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type SOAPClient struct {
	url string
}

func NewSOAPClient(url string) *SOAPClient {
	return &SOAPClient{url: url}
}

// Temp
func WrapSoap(s string) string {
	soap := strings.Join([]string{
		"<S:Envelope xmlns:S=\"http://schemas.xmlsoap.org/soap/envelope/\" xmlns:SOAP-ENV=\"http://schemas.xmlsoap.org/soap/envelope/\">\n\t<S:Body>\n}",
		s,
		"\n\t</S:Body>\n</S:Envelope>"}, "")
	return soap
}

func (client *SOAPClient) CheckStatus(request *SIRICheckStatusRequest) (*XMLCheckStatusResponse, error) {
	// Wrap the request XML
	soapRequest := WrapSoap(request.BuildXML())

	// Create http request
	httpRequest, err := http.NewRequest("POST", client.url, strings.NewReader(soapRequest))
	if err != nil {
		return nil, err
	}
	httpRequest.Header.Set("Accept-Encoding", "gzip, deflate")
	httpRequest.Header.Set("Content-Type", "text/xml")

	// Send http request
	response, err := http.DefaultClient.Do(httpRequest)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()

	// Check response status
	if response.StatusCode != http.StatusOK {
		return nil, errors.New(strings.Join([]string{"Request error, response status code: ", strconv.Itoa(response.StatusCode)}, ""))
	}

	// Create XMLCheckStatusResponse
	responseContent, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}
	xmlResponse, err := NewXMLCheckStatusResponseFromContent(responseContent)
	if err != nil {
		return nil, err
	}

	return xmlResponse, nil
}

func CheckStatusHandler(w http.ResponseWriter, r *http.Request) {
	requestContent, err := ioutil.ReadAll(r.Body)
	if err != nil {
		//Handle error
	}
	xmlRequest, err := NewXMLCheckStatusRequestFromContent(requestContent)
	if err != nil {
		// Handle error
	}

	response := new(SIRICheckStatusResponse)
	response.Address = strings.Join([]string{r.URL.Host, r.URL.Path}, "")
	response.ProducerRef = "Edwig"
	response.RequestMessageRef = xmlRequest.MessageIdentifier()
	response.ResponseMessageIdentifier = "c464f588-5128-46c8-ac3f-8b8a465692ab" // uuid - Temp
	response.Status = true                                                      // Temp
	response.ResponseTimestamp = time.Now()
	response.ServiceStartedTime = time.Now() //Temp

	fmt.Fprintf(w, response.BuildXML())
}
<commit_msg>Remove typo in SOAP envelope. Refs #1694<commit_after>package siri

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type SOAPClient struct {
	url string
}

func NewSOAPClient(url string) *SOAPClient {
	return &SOAPClient{url: url}
}

// Temp
func WrapSoap(s string) string {
	soap := strings.Join([]string{
		"<S:Envelope xmlns:S=\"http://schemas.xmlsoap.org/soap/envelope/\" xmlns:SOAP-ENV=\"http://schemas.xmlsoap.org/soap/envelope/\">\n\t<S:Body>\n",
		s,
		"\n\t</S:Body>\n</S:Envelope>"}, "")
	return soap
}

func (client *SOAPClient) CheckStatus(request *SIRICheckStatusRequest) (*XMLCheckStatusResponse, error) {
	// Wrap the request XML
	soapRequest := WrapSoap(request.BuildXML())

	// Create http request
	httpRequest, err := http.NewRequest("POST", client.url, strings.NewReader(soapRequest))
	if err != nil {
		return nil, err
	}
	httpRequest.Header.Set("Accept-Encoding", "gzip, deflate")
	httpRequest.Header.Set("Content-Type", "text/xml")

	// Send http request
	response, err := http.DefaultClient.Do(httpRequest)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()

	// Check response status
	if response.StatusCode != http.StatusOK {
		return nil, errors.New(strings.Join([]string{"Request error, response status code: ", strconv.Itoa(response.StatusCode)}, ""))
	}

	// Create XMLCheckStatusResponse
	responseContent, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}
	xmlResponse, err := NewXMLCheckStatusResponseFromContent(responseContent)
	if err != nil {
		return nil, err
	}

	return xmlResponse, nil
}

func CheckStatusHandler(w http.ResponseWriter, r *http.Request) {
	requestContent, err := ioutil.ReadAll(r.Body)
	if err != nil {
		//Handle error
	}
	xmlRequest, err := NewXMLCheckStatusRequestFromContent(requestContent)
	if err != nil {
		// Handle error
	}

	response := new(SIRICheckStatusResponse)
	response.Address = strings.Join([]string{r.URL.Host, r.URL.Path}, "")
	response.ProducerRef = "Edwig"
	response.RequestMessageRef = xmlRequest.MessageIdentifier()
	response.ResponseMessageIdentifier = "c464f588-5128-46c8-ac3f-8b8a465692ab" // uuid - Temp
	response.Status = true                                                      // Temp
	response.ResponseTimestamp = time.Now()
	response.ServiceStartedTime = time.Now() //Temp

	fmt.Fprintf(w, response.BuildXML())
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/user"
	"strings"
)

// Conf represents Slack configuration data
type Conf struct {
	Token    string `json:"token"`
	Channel  string `json:"channel"`
	Username string `json:"username"`
}

var conf Conf

func main() {

	usr, err := user.Current()
	if err != nil {
		os.Stderr.WriteString("Can't get current user info... Are you an alien?\n")
		os.Exit(1)
	}
	confFile := usr.HomeDir + "/.slack-cli.json"

	file, err := ioutil.ReadFile(confFile)
	if err != nil {
		os.Stderr.WriteString("Can't find ~/.slack-cli.json.\n")
		os.Exit(1)
	}

	err = json.Unmarshal(file, &conf)
	if err != nil {
		os.Stderr.WriteString("Can't parse ~/.slack-cli.json. Is the format correct?\n")
		os.Exit(1)
	}

	if len(os.Args) < 2 {
		os.Stderr.WriteString("Usage: slack-cli MESSAGE-TO-SEND\n")
		os.Exit(1)
	}

	query := fmt.Sprintf("token=%s&channel=%s&username=%s&text=%s",
		conf.Token,
		conf.Channel,
		conf.Username,
		strings.Join(os.Args[1:], " "))

	body := strings.NewReader(query)
	req, err := http.NewRequest("POST", "https://slack.com/api/chat.postMessage", body)
	if err != nil {
		os.Stderr.WriteString("Can't post to Slack API.\n")
		os.Exit(1)
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		os.Stderr.WriteString("Can't post to Slack API.\n")
		os.Exit(1)
	}
	defer resp.Body.Close()
}
<commit_msg>Create loadConf function<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/user"
	"strings"
)

// Conf represents Slack configuration data
type Conf struct {
	Token    string `json:"token"`
	Channel  string `json:"channel"`
	Username string `json:"username"`
}

var conf Conf

func loadConf() {
	usr, err := user.Current()
	if err != nil {
		os.Stderr.WriteString("Can't get current user info... Are you an alien?\n")
		os.Exit(1)
	}
	confFile := usr.HomeDir + "/.slack-cli.json"

	file, err := ioutil.ReadFile(confFile)
	if err != nil {
		os.Stderr.WriteString("Can't find ~/.slack-cli.json.\n")
		os.Exit(1)
	}

	err = json.Unmarshal(file, &conf)
	if err != nil {
		os.Stderr.WriteString("Can't parse ~/.slack-cli.json. Is the format correct?\n")
		os.Exit(1)
	}

	if len(os.Args) < 2 {
		os.Stderr.WriteString("Usage: slack-cli MESSAGE-TO-SEND\n")
		os.Exit(1)
	}
}

func main() {
	loadConf()

	query := fmt.Sprintf("token=%s&channel=%s&username=%s&text=%s",
		conf.Token,
		conf.Channel,
		conf.Username,
		strings.Join(os.Args[1:], " "))

	body := strings.NewReader(query)
	req, err := http.NewRequest("POST", "https://slack.com/api/chat.postMessage", body)
	if err != nil {
		os.Stderr.WriteString("Can't post to Slack API.\n")
		os.Exit(1)
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		os.Stderr.WriteString("Can't post to Slack API.\n")
		os.Exit(1)
	}
	defer resp.Body.Close()
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	"time"

	kapi "k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/registry/service"
	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
	utilwait "k8s.io/kubernetes/pkg/util/wait"

	"github.com/openshift/origin/pkg/security"
	"github.com/openshift/origin/pkg/security/uid"
	"github.com/openshift/origin/pkg/security/uidallocator"
)

// Repair is a controller loop that periodically examines all UID allocations
// and logs any errors, and then sets the compacted and accurate list of both
//
// Can be run at infrequent intervals, and is best performed on startup of the master.
// Is level driven and idempotent - all claimed UIDs will be updated into the allocator
// map at the end of a single execution loop if no race is encountered.
//
type Repair struct {
	interval time.Duration
	client   client.NamespaceInterface
	alloc    service.RangeRegistry
	uidRange *uid.Range
}

// NewRepair creates a controller that periodically ensures that all UIDs labels that are allocated in the cluster
// are claimed.
func NewRepair(interval time.Duration, client client.NamespaceInterface, uidRange *uid.Range, alloc service.RangeRegistry) *Repair {
	return &Repair{
		interval: interval,
		client:   client,
		uidRange: uidRange,
		alloc:    alloc,
	}
}

// RunUntil starts the controller until the provided ch is closed.
func (c *Repair) RunUntil(ch chan struct{}) {
	utilwait.Until(func() {
		if err := c.RunOnce(); err != nil {
			utilruntime.HandleError(err)
		}
	}, c.interval, ch)
}

// RunOnce verifies the state of allocations and returns an error if an unrecoverable problem occurs.
func (c *Repair) RunOnce() error {
	// TODO: (per smarterclayton) if Get() or List() is a weak consistency read,
	// or if they are executed against different leaders,
	// the ordering guarantee required to ensure no item is allocated twice is violated.
	// List must return a ResourceVersion higher than the etcd index Get,
	// and the release code must not release items that have allocated but not yet been created
	// See #8295

	latest, err := c.alloc.Get()
	if err != nil {
		return fmt.Errorf("unable to refresh the security allocation UID blocks: %v", err)
	}

	list, err := c.client.List(kapi.ListOptions{})
	if err != nil {
		return fmt.Errorf("unable to refresh the security allocation UID blocks: %v", err)
	}

	uids := uidallocator.NewInMemory(c.uidRange)

	for _, ns := range list.Items {
		value, ok := ns.Annotations[security.UIDRangeAnnotation]
		if !ok {
			continue
		}
		block, err := uid.ParseBlock(value)
		if err != nil {
			continue
		}
		switch err := uids.Allocate(block); err {
		case nil:
		case uidallocator.ErrNotInRange, uidallocator.ErrAllocated:
			continue
		case uidallocator.ErrFull:
			// TODO: send event
			return fmt.Errorf("the UID range %s is full; you must widen the range in order to allocate more UIDs", c.uidRange)
		default:
			return fmt.Errorf("unable to allocate UID block %s for namespace %s due to an unknown error, exiting: %v", block, ns.Name, err)
		}
	}

	err = uids.Snapshot(latest)
	if err != nil {
		return fmt.Errorf("unable to persist the updated namespace UID allocations: %v", err)
	}

	if err := c.alloc.CreateOrUpdate(latest); err != nil {
		return fmt.Errorf("unable to persist the updated namespace UID allocations: %v", err)
	}
	return nil
}
<commit_msg>Repair package changed<commit_after>package controller

import (
	"fmt"
	"time"

	kapi "k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/registry/rangeallocation"
	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
	utilwait "k8s.io/kubernetes/pkg/util/wait"

	"github.com/openshift/origin/pkg/security"
	"github.com/openshift/origin/pkg/security/uid"
	"github.com/openshift/origin/pkg/security/uidallocator"
)

// Repair is a controller loop that periodically examines all UID allocations
// and logs any errors, and then sets the compacted and accurate list of both
//
// Can be run at infrequent intervals, and is best performed on startup of the master.
// Is level driven and idempotent - all claimed UIDs will be updated into the allocator
// map at the end of a single execution loop if no race is encountered.
//
type Repair struct {
	interval time.Duration
	client   client.NamespaceInterface
	alloc    rangeallocation.RangeRegistry
	uidRange *uid.Range
}

// NewRepair creates a controller that periodically ensures that all UIDs labels that are allocated in the cluster
// are claimed.
func NewRepair(interval time.Duration, client client.NamespaceInterface, uidRange *uid.Range, alloc rangeallocation.RangeRegistry) *Repair {
	return &Repair{
		interval: interval,
		client:   client,
		uidRange: uidRange,
		alloc:    alloc,
	}
}

// RunUntil starts the controller until the provided ch is closed.
func (c *Repair) RunUntil(ch chan struct{}) {
	utilwait.Until(func() {
		if err := c.RunOnce(); err != nil {
			utilruntime.HandleError(err)
		}
	}, c.interval, ch)
}

// RunOnce verifies the state of allocations and returns an error if an unrecoverable problem occurs.
func (c *Repair) RunOnce() error {
	// TODO: (per smarterclayton) if Get() or List() is a weak consistency read,
	// or if they are executed against different leaders,
	// the ordering guarantee required to ensure no item is allocated twice is violated.
	// List must return a ResourceVersion higher than the etcd index Get,
	// and the release code must not release items that have allocated but not yet been created
	// See #8295

	latest, err := c.alloc.Get()
	if err != nil {
		return fmt.Errorf("unable to refresh the security allocation UID blocks: %v", err)
	}

	list, err := c.client.List(kapi.ListOptions{})
	if err != nil {
		return fmt.Errorf("unable to refresh the security allocation UID blocks: %v", err)
	}

	uids := uidallocator.NewInMemory(c.uidRange)

	for _, ns := range list.Items {
		value, ok := ns.Annotations[security.UIDRangeAnnotation]
		if !ok {
			continue
		}
		block, err := uid.ParseBlock(value)
		if err != nil {
			continue
		}
		switch err := uids.Allocate(block); err {
		case nil:
		case uidallocator.ErrNotInRange, uidallocator.ErrAllocated:
			continue
		case uidallocator.ErrFull:
			// TODO: send event
			return fmt.Errorf("the UID range %s is full; you must widen the range in order to allocate more UIDs", c.uidRange)
		default:
			return fmt.Errorf("unable to allocate UID block %s for namespace %s due to an unknown error, exiting: %v", block, ns.Name, err)
		}
	}

	err = uids.Snapshot(latest)
	if err != nil {
		return fmt.Errorf("unable to persist the updated namespace UID allocations: %v", err)
	}

	if err := c.alloc.CreateOrUpdate(latest); err != nil {
		return fmt.Errorf("unable to persist the updated namespace UID allocations: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"bytes"
	"context"
	"fmt"
	"io"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/color"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/logfile"
	latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/custom"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/structure"
)

type Config interface {
	docker.Config

	TestCases() []*latestV1.TestCase
	Muted() config.Muted
}

// NewTester parses the provided test cases from the Skaffold config,
// and returns a Tester instance with all the necessary test runners
// to run all specified tests.
func NewTester(cfg Config, imagesAreLocal func(imageName string) (bool, error)) (Tester, error) {
	testers, err := getImageTesters(cfg, imagesAreLocal, cfg.TestCases())
	if err != nil {
		return nil, err
	}

	return FullTester{
		Testers: testers,
		muted:   cfg.Muted(),
	}, nil
}

// TestDependencies returns the watch dependencies for the target artifact to the runner.
func (t FullTester) TestDependencies(artifact *latestV1.Artifact) ([]string, error) {
	var deps []string
	for _, tester := range t.Testers[artifact.ImageName] {
		result, err := tester.TestDependencies()
		if err != nil {
			return nil, err
		}
		deps = append(deps, result...)
	}
	return deps, nil
}

// Test is the top level testing execution call. It serves as the
// entrypoint to all individual tests.
func (t FullTester) Test(ctx context.Context, out io.Writer, bRes []graph.Artifact) error {
	if len(t.Testers) == 0 {
		return nil
	}

	color.Default.Fprintln(out, "Testing images...")

	if t.muted.MuteTest() {
		file, err := logfile.Create("test.log")
		if err != nil {
			return fmt.Errorf("unable to create log file for tests: %w", err)
		}
		fmt.Fprintln(out, " - writing logs to", file.Name())

		// Print logs to a memory buffer and to a file.
		var buf bytes.Buffer
		w := io.MultiWriter(file, &buf)

		// Run the tests.
		err = t.runTests(ctx, w, bRes)

		// After the test finish, close the log file. If the tests failed, print the full log to the console.
		file.Close()
		if err != nil {
			buf.WriteTo(out)
		}

		return err
	}

	return t.runTests(ctx, out, bRes)
}

func (t FullTester) runTests(ctx context.Context, out io.Writer, bRes []graph.Artifact) error {
	for _, b := range bRes {
		for _, tester := range t.Testers[b.ImageName] {
			if err := tester.Test(ctx, out, b.Tag); err != nil {
				return fmt.Errorf("running tests: %w", err)
			}
		}
	}
	return nil
}

func getImageTesters(cfg docker.Config, imagesAreLocal func(imageName string) (bool, error), tcs []*latestV1.TestCase) (ImageTesters, error) {
	runners := make(map[string][]ImageTester)
	for _, tc := range tcs {
		isLocal, err := imagesAreLocal(tc.ImageName)
		if err != nil {
			return nil, err
		}

		if len(tc.StructureTests) != 0 {
			structureRunner, err := structure.New(cfg, tc, isLocal)
			if err != nil {
				return nil, err
			}
			runners[tc.ImageName] = append(runners[tc.ImageName], structureRunner)
		}

		for _, customTest := range tc.CustomTests {
			customRunner, err := custom.New(cfg, tc.ImageName, tc.Workspace, customTest)
			if err != nil {
				return nil, err
			}
			runners[tc.ImageName] = append(runners[tc.ImageName], customRunner)
		}
	}
	return runners, nil
}
<commit_msg>add emission of TaskEvents for Test phase (#5814)<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"bytes"
	"context"
	"fmt"
	"io"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/color"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	eventV2 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/event/v2"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/logfile"
	latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/custom"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/structure"
)

type Config interface {
	docker.Config

	TestCases() []*latestV1.TestCase
	Muted() config.Muted
}

// NewTester parses the provided test cases from the Skaffold config,
// and returns a Tester instance with all the necessary test runners
// to run all specified tests.
func NewTester(cfg Config, imagesAreLocal func(imageName string) (bool, error)) (Tester, error) {
	testers, err := getImageTesters(cfg, imagesAreLocal, cfg.TestCases())
	if err != nil {
		return nil, err
	}

	return FullTester{
		Testers: testers,
		muted:   cfg.Muted(),
	}, nil
}

// TestDependencies returns the watch dependencies for the target artifact to the runner.
func (t FullTester) TestDependencies(artifact *latestV1.Artifact) ([]string, error) {
	var deps []string
	for _, tester := range t.Testers[artifact.ImageName] {
		result, err := tester.TestDependencies()
		if err != nil {
			return nil, err
		}
		deps = append(deps, result...)
	}
	return deps, nil
}

// Test is the top level testing execution call. It serves as the
// entrypoint to all individual tests.
func (t FullTester) Test(ctx context.Context, out io.Writer, bRes []graph.Artifact) error {
	if len(t.Testers) == 0 {
		return nil
	}

	eventV2.TaskInProgress(constants.Test)
	color.Default.Fprintln(out, "Testing images...")

	if t.muted.MuteTest() {
		file, err := logfile.Create("test.log")
		if err != nil {
			return fmt.Errorf("unable to create log file for tests: %w", err)
		}
		fmt.Fprintln(out, " - writing logs to", file.Name())

		// Print logs to a memory buffer and to a file.
		var buf bytes.Buffer
		w := io.MultiWriter(file, &buf)

		// Run the tests.
		err = t.runTests(ctx, w, bRes)

		// After the test finish, close the log file. If the tests failed, print the full log to the console.
		file.Close()
		if err != nil {
			buf.WriteTo(out)
		}

		return err
	}

	if err := t.runTests(ctx, out, bRes); err != nil {
		eventV2.TaskFailed(constants.Test, err)
		return err
	}

	eventV2.TaskSucceeded(constants.Test)
	return nil
}

func (t FullTester) runTests(ctx context.Context, out io.Writer, bRes []graph.Artifact) error {
	for _, b := range bRes {
		for _, tester := range t.Testers[b.ImageName] {
			if err := tester.Test(ctx, out, b.Tag); err != nil {
				return fmt.Errorf("running tests: %w", err)
			}
		}
	}
	return nil
}

func getImageTesters(cfg docker.Config, imagesAreLocal func(imageName string) (bool, error), tcs []*latestV1.TestCase) (ImageTesters, error) {
	runners := make(map[string][]ImageTester)
	for _, tc := range tcs {
		isLocal, err := imagesAreLocal(tc.ImageName)
		if err != nil {
			return nil, err
		}

		if len(tc.StructureTests) != 0 {
			structureRunner, err := structure.New(cfg, tc, isLocal)
			if err != nil {
				return nil, err
			}
			runners[tc.ImageName] = append(runners[tc.ImageName], structureRunner)
		}

		for _, customTest := range tc.CustomTests {
			customRunner, err := custom.New(cfg, tc.ImageName, tc.Workspace, customTest)
			if err != nil {
				return nil, err
			}
			runners[tc.ImageName] = append(runners[tc.ImageName], customRunner)
		}
	}
	return runners, nil
}
<|endoftext|>


<commit_before>package cloudwatch

import (
	"context"
	"errors"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/tsdb"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/grafana/grafana/pkg/components/null"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/metrics"
)

type CloudWatchExecutor struct {
	*models.DataSource
}

type DatasourceInfo struct {
	Profile       string
	Region        string
	AuthType      string
	AssumeRoleArn string
	Namespace     string

	AccessKey string
	SecretKey string
}

func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	return &CloudWatchExecutor{}, nil
}

var (
	plog               log.Logger
	standardStatistics map[string]bool
	aliasFormat        *regexp.Regexp
)

func init() {
	plog = log.New("tsdb.cloudwatch")
	tsdb.RegisterTsdbQueryEndpoint("cloudwatch", NewCloudWatchExecutor)
	standardStatistics = map[string]bool{
		"Average":     true,
		"Maximum":     true,
		"Minimum":     true,
		"Sum":         true,
		"SampleCount": true,
	}
	aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
}

func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	var result *tsdb.Response
	e.DataSource = dsInfo
	queryType := queryContext.Queries[0].Model.Get("type").MustString("")
	var err error

	switch queryType {
	case "metricFindQuery":
		result, err = e.executeMetricFindQuery(ctx, queryContext)
		break
	case "annotationQuery":
		result, err = e.executeAnnotationQuery(ctx, queryContext)
		break
	case "timeSeriesQuery":
		fallthrough
	default:
		result, err = e.executeTimeSeriesQuery(ctx, queryContext)
		break
	}

	return result, err
}

func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	result := &tsdb.Response{
		Results: make(map[string]*tsdb.QueryResult),
	}

	errCh := make(chan error, 1)
	resCh := make(chan *tsdb.QueryResult, 1)

	currentlyExecuting := 0
	for i, model := range queryContext.Queries {
		queryType := model.Model.Get("type").MustString()
		if queryType != "timeSeriesQuery" {
			continue
		}
		currentlyExecuting++
		go func(refId string, index int) {
			queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext)
			currentlyExecuting--
			if err != nil {
				errCh <- err
			} else {
				queryRes.RefId = refId
				resCh <- queryRes
			}
		}(model.RefId, i)
	}

	for currentlyExecuting != 0 {
		select {
		case res := <-resCh:
			result.Results[res.RefId] = res
		case err := <-errCh:
			return result, err
		case <-ctx.Done():
			return result, ctx.Err()
		}
	}

	return result, nil
}

func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
	query, err := parseQuery(parameters)
	if err != nil {
		return nil, err
	}

	client, err := e.getClient(query.Region)
	if err != nil {
		return nil, err
	}

	startTime, err := queryContext.TimeRange.ParseFrom()
	if err != nil {
		return nil, err
	}

	endTime, err := queryContext.TimeRange.ParseTo()
	if err != nil {
		return nil, err
	}

	params := &cloudwatch.GetMetricStatisticsInput{
		Namespace:  aws.String(query.Namespace),
		MetricName: aws.String(query.MetricName),
		Dimensions: query.Dimensions,
		Period:     aws.Int64(int64(query.Period)),
		StartTime:  aws.Time(startTime),
		EndTime:    aws.Time(endTime),
	}
	if len(query.Statistics) > 0 {
		params.Statistics = query.Statistics
	}
	if len(query.ExtendedStatistics) > 0 {
		params.ExtendedStatistics = query.ExtendedStatistics
	}

	if setting.Env == setting.DEV {
		plog.Debug("CloudWatch query", "raw query", params)
	}

	resp, err := client.GetMetricStatisticsWithContext(ctx, params, request.WithResponseReadTimeout(10*time.Second))
	if err != nil {
		return nil, err
	}
	metrics.M_Aws_CloudWatch_GetMetricStatistics.Inc()

	queryRes, err := parseResponse(resp, query)
	if err != nil {
		return nil, err
	}

	return queryRes, nil
}

func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) {
	var result []*cloudwatch.Dimension

	for k, v := range model.Get("dimensions").MustMap() {
		kk := k
		if vv, ok := v.(string); ok {
			result = append(result, &cloudwatch.Dimension{
				Name:  &kk,
				Value: &vv,
			})
		} else {
			return nil, errors.New("failed to parse")
		}
	}

	sort.Slice(result, func(i, j int) bool {
		return *result[i].Name < *result[j].Name
	})
	return result, nil
}

func parseStatistics(model *simplejson.Json) ([]string, []string, error) {
	var statistics []string
	var extendedStatistics []string

	for _, s := range model.Get("statistics").MustArray() {
		if ss, ok := s.(string); ok {
			if _, isStandard := standardStatistics[ss]; isStandard {
				statistics = append(statistics, ss)
			} else {
				extendedStatistics = append(extendedStatistics, ss)
			}
		} else {
			return nil, nil, errors.New("failed to parse")
		}
	}

	return statistics, extendedStatistics, nil
}

func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
	region, err := model.Get("region").String()
	if err != nil {
		return nil, err
	}

	namespace, err := model.Get("namespace").String()
	if err != nil {
		return nil, err
	}

	metricName, err := model.Get("metricName").String()
	if err != nil {
		return nil, err
	}

	dimensions, err := parseDimensions(model)
	if err != nil {
		return nil, err
	}

	statistics, extendedStatistics, err := parseStatistics(model)
	if err != nil {
		return nil, err
	}

	p := model.Get("period").MustString("")
	if p == "" {
		if namespace == "AWS/EC2" {
			p = "300"
		} else {
			p = "60"
		}
	}

	period := 300
	if regexp.MustCompile(`^\d+$`).Match([]byte(p)) {
		period, err = strconv.Atoi(p)
		if err != nil {
			return nil, err
		}
	} else {
		d, err := time.ParseDuration(p)
		if err != nil {
			return nil, err
		}
		period = int(d.Seconds())
	}

	alias := model.Get("alias").MustString("{{metric}}_{{stat}}")

	return &CloudWatchQuery{
		Region:             region,
		Namespace:          namespace,
		MetricName:         metricName,
		Dimensions:         dimensions,
		Statistics:         aws.StringSlice(statistics),
		ExtendedStatistics: aws.StringSlice(extendedStatistics),
		Period:             period,
		Alias:              alias,
	}, nil
}

func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string {
	data := map[string]string{}
	data["region"] = query.Region
	data["namespace"] = query.Namespace
	data["metric"] = query.MetricName
	data["stat"] = stat
	for k, v := range dimensions {
		data[k] = v
	}

	result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
		labelName := strings.Replace(string(in), "{{", "", 1)
		labelName = strings.Replace(labelName, "}}", "", 1)
		labelName = strings.TrimSpace(labelName)
		if val, exists := data[labelName]; exists {
			return []byte(val)
		}

		return in
	})

	return string(result)
}

func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) {
	queryRes := tsdb.NewQueryResult()

	var value float64
	for _, s := range append(query.Statistics, query.ExtendedStatistics...) {
		series := tsdb.TimeSeries{
			Tags: map[string]string{},
		}
		for _, d := range query.Dimensions {
			series.Tags[*d.Name] = *d.Value
		}
		series.Name = formatAlias(query, *s, series.Tags)

		lastTimestamp := make(map[string]time.Time)
		sort.Slice(resp.Datapoints, func(i, j int) bool {
			return (*resp.Datapoints[i].Timestamp).Before(*resp.Datapoints[j].Timestamp)
		})
		for _, v := range resp.Datapoints {
			switch *s {
			case "Average":
				value = *v.Average
			case "Maximum":
				value = *v.Maximum
			case "Minimum":
				value = *v.Minimum
			case "Sum":
				value = *v.Sum
			case "SampleCount":
				value = *v.SampleCount
			default:
				if strings.Index(*s, "p") == 0 && v.ExtendedStatistics[*s] != nil {
					value = *v.ExtendedStatistics[*s]
				}
			}

			// terminate gap of data points
			timestamp := *v.Timestamp
			if _, ok := lastTimestamp[*s]; ok {
				nextTimestampFromLast := lastTimestamp[*s].Add(time.Duration(query.Period) * time.Second)
				for timestamp.After(nextTimestampFromLast) {
					series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(nextTimestampFromLast.Unix()*1000)))
					nextTimestampFromLast = nextTimestampFromLast.Add(time.Duration(query.Period) * time.Second)
				}
			}
			lastTimestamp[*s] = timestamp

			series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(value), float64(timestamp.Unix()*1000)))
		}

		queryRes.Series = append(queryRes.Series, &series)
	}

	return queryRes, nil
}
<commit_msg>fix cloudwatch alert bug<commit_after>package cloudwatch

import (
	"context"
	"errors"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/tsdb"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/grafana/grafana/pkg/components/null"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/metrics"
)

type CloudWatchExecutor struct {
	*models.DataSource
}

type DatasourceInfo struct {
	Profile       string
	Region        string
	AuthType      string
	AssumeRoleArn string
	Namespace     string

	AccessKey string
	SecretKey string
}

func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	return &CloudWatchExecutor{}, nil
}

var (
	plog               log.Logger
	standardStatistics map[string]bool
	aliasFormat        *regexp.Regexp
)

func init() {
	plog = log.New("tsdb.cloudwatch")
	tsdb.RegisterTsdbQueryEndpoint("cloudwatch", NewCloudWatchExecutor)
	standardStatistics = map[string]bool{
		"Average":     true,
		"Maximum":     true,
		"Minimum":     true,
		"Sum":         true,
		"SampleCount": true,
	}
	aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
}

func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	var result *tsdb.Response
	e.DataSource = dsInfo
	queryType := queryContext.Queries[0].Model.Get("type").MustString("")
	var err error

	switch queryType {
	case "metricFindQuery":
		result, err = e.executeMetricFindQuery(ctx, queryContext)
		break
	case "annotationQuery":
		result, err = e.executeAnnotationQuery(ctx, queryContext)
		break
	case "timeSeriesQuery":
		fallthrough
	default:
		result, err = e.executeTimeSeriesQuery(ctx, queryContext)
		break
	}

	return result, err
}

func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	result := &tsdb.Response{
		Results: make(map[string]*tsdb.QueryResult),
	}

	errCh := make(chan error, 1)
	resCh := make(chan *tsdb.QueryResult, 1)

	currentlyExecuting := 0
	for i, model := range queryContext.Queries {
		queryType := model.Model.Get("type").MustString()
		if queryType != "timeSeriesQuery" && queryType != "" {
			continue
		}
		currentlyExecuting++
		go func(refId string, index int) {
			queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext)
			currentlyExecuting--
			if err != nil {
				errCh <- err
			} else {
				queryRes.RefId = refId
				resCh <- queryRes
			}
		}(model.RefId, i)
	}

	for currentlyExecuting != 0 {
		select {
		case res := <-resCh:
			result.Results[res.RefId] = res
		case err := <-errCh:
			return result, err
		case <-ctx.Done():
			return result, ctx.Err()
		}
	}

	return result, nil
}

func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
	query, err := parseQuery(parameters)
	if err != nil {
		return nil, err
	}

	client, err := e.getClient(query.Region)
	if err != nil {
		return nil, err
	}

	startTime, err := queryContext.TimeRange.ParseFrom()
	if err != nil {
		return nil, err
	}

	endTime, err := queryContext.TimeRange.ParseTo()
	if err != nil {
		return nil, err
	}

	params := &cloudwatch.GetMetricStatisticsInput{
		Namespace:  aws.String(query.Namespace),
		MetricName: aws.String(query.MetricName),
		Dimensions: query.Dimensions,
		Period:     aws.Int64(int64(query.Period)),
		StartTime:  aws.Time(startTime),
		EndTime:    aws.Time(endTime),
	}
	if len(query.Statistics) > 0 {
		params.Statistics = query.Statistics
	}
	if len(query.ExtendedStatistics) > 0 {
		params.ExtendedStatistics = query.ExtendedStatistics
	}

	if setting.Env == setting.DEV {
		plog.Debug("CloudWatch query", "raw query", params)
	}

	resp, err := client.GetMetricStatisticsWithContext(ctx, params, request.WithResponseReadTimeout(10*time.Second))
	if err != nil {
		return nil, err
	}
	metrics.M_Aws_CloudWatch_GetMetricStatistics.Inc()

	queryRes, err := parseResponse(resp, query)
	if err != nil {
		return nil, err
	}

	return queryRes, nil
}

func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) {
	var result []*cloudwatch.Dimension

	for k, v := range model.Get("dimensions").MustMap() {
		kk := k
		if vv, ok := v.(string); ok {
			result = append(result, &cloudwatch.Dimension{
				Name:  &kk,
				Value: &vv,
			})
		} else {
			return nil, errors.New("failed to parse")
		}
	}

	sort.Slice(result, func(i, j int) bool {
		return *result[i].Name < *result[j].Name
	})
	return result, nil
}

func parseStatistics(model *simplejson.Json) ([]string, []string, error) {
	var statistics []string
	var extendedStatistics []string

	for _, s := range model.Get("statistics").MustArray() {
		if ss, ok := s.(string); ok {
			if _, isStandard := standardStatistics[ss]; isStandard {
				statistics = append(statistics, ss)
			} else {
				extendedStatistics = append(extendedStatistics, ss)
			}
		} else {
			return nil, nil, errors.New("failed to parse")
		}
	}

	return statistics, extendedStatistics, nil
}

func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
	region, err := model.Get("region").String()
	if err != nil {
		return nil, err
	}

	namespace, err := model.Get("namespace").String()
	if err != nil {
		return nil, err
	}

	metricName, err := model.Get("metricName").String()
	if err != nil {
		return nil, err
	}

	dimensions, err := parseDimensions(model)
	if err != nil {
		return nil, err
	}

	statistics, extendedStatistics, err := parseStatistics(model)
	if err != nil {
		return nil, err
	}

	p := model.Get("period").MustString("")
	if p == "" {
		if namespace == "AWS/EC2" {
			p = "300"
		} else {
			p = "60"
		}
	}

	period := 300
	if regexp.MustCompile(`^\d+$`).Match([]byte(p)) {
		period, err = strconv.Atoi(p)
		if err != nil {
			return nil, err
		}
	} else {
		d, err := time.ParseDuration(p)
		if err != nil {
			return nil, err
		}
		period = int(d.Seconds())
	}

	alias := model.Get("alias").MustString("{{metric}}_{{stat}}")

	return &CloudWatchQuery{
		Region:             region,
		Namespace:          namespace,
		MetricName:         metricName,
		Dimensions:         dimensions,
		Statistics:         aws.StringSlice(statistics),
		ExtendedStatistics: aws.StringSlice(extendedStatistics),
		Period:             period,
		Alias:              alias,
	}, nil
}

func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string {
	data := map[string]string{}
	data["region"] = query.Region
	data["namespace"] = query.Namespace
	data["metric"] = query.MetricName
	data["stat"] = stat
	for k, v := range dimensions {
		data[k] = v
	}

	result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
		labelName := strings.Replace(string(in), "{{", "", 1)
		labelName = strings.Replace(labelName, "}}", "", 1)
		labelName = strings.TrimSpace(labelName)
		if val, exists := data[labelName]; exists {
			return []byte(val)
		}

		return in
	})

	return string(result)
}

func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) {
	queryRes := tsdb.NewQueryResult()

	var value float64
	for _, s := range append(query.Statistics, query.ExtendedStatistics...) {
		series := tsdb.TimeSeries{
			Tags: map[string]string{},
		}
		for _, d := range query.Dimensions {
			series.Tags[*d.Name] = *d.Value
		}
		series.Name = formatAlias(query, *s, series.Tags)

		lastTimestamp := make(map[string]time.Time)
		sort.Slice(resp.Datapoints, func(i, j int) bool {
			return (*resp.Datapoints[i].Timestamp).Before(*resp.Datapoints[j].Timestamp)
		})
		for _, v := range resp.Datapoints {
			switch *s {
			case "Average":
				value = *v.Average
			case "Maximum":
				value = *v.Maximum
			case "Minimum":
				value = *v.Minimum
			case "Sum":
				value = *v.Sum
			case "SampleCount":
				value = *v.SampleCount
			default:
				if strings.Index(*s, "p") == 0 && v.ExtendedStatistics[*s] != nil {
					value = *v.ExtendedStatistics[*s]
				}
			}

			// terminate gap of data points
			timestamp := *v.Timestamp
			if _, ok := lastTimestamp[*s]; ok {
				nextTimestampFromLast := lastTimestamp[*s].Add(time.Duration(query.Period) * time.Second)
				for timestamp.After(nextTimestampFromLast) {
					series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(nextTimestampFromLast.Unix()*1000)))
					nextTimestampFromLast = nextTimestampFromLast.Add(time.Duration(query.Period) * time.Second)
				}
			}
			lastTimestamp[*s] = timestamp

			series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(value), float64(timestamp.Unix()*1000)))
		}

		queryRes.Series = append(queryRes.Series, &series)
	}

	return queryRes, nil
}
<|endoftext|>


<commit_before>package server

import (
	"net"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/koding/tunnel"
	"k8s.io/apimachinery/pkg/util/wait"
	informers "k8s.io/client-go/informers/core/v1"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
)

type Controller struct {
	nodes  informers.NodeInformer
	tunnel *tunnel.Server
	queue  workqueue.RateLimitingInterface
	store  map[string]net.Listener
}

func NewController(informer informers.NodeInformer, tunnel *tunnel.Server) *Controller {
	c := &Controller{
		nodes:  informer,
		tunnel: tunnel,
		queue:  workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(5*time.Second, 300*time.Second)),
		store:  make(map[string]net.Listener),
	}

	c.nodes.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(obj)
			if err == nil {
				c.queue.Add(key)
			}
		},
		DeleteFunc: func(obj interface{}) {
			key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
			if err == nil {
				c.queue.Add(key)
			}
		},
	})

	return c
}

func (c *Controller) Run(threadiness int, stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer c.queue.ShutDown()
	defer wg.Done()
	wg.Add(1)
	glog.Infof(`Starting WormholeGenerator with %d workers`, threadiness)

	for i := 0; i < threadiness; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	ticker := time.NewTicker(5 * time.Minute)
	go func() {
		for {
			select {
			case <-ticker.C:
				glog.V(5).Infof("Running periodic recheck. Queuing all known nodes...")
				for key, _ := range c.store {
					c.queue.Add(key)
				}
			case <-stopCh:
				ticker.Stop()
				return
			}
		}
	}()

	<-stopCh
}

func (c *Controller) runWorker() {
	for c.processNextWorkItem() {
	}
}

func (c *Controller) processNextWorkItem() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)

	// Invoke the method containing the business logic
	err := c.reconcile(key.(string))
	c.handleErr(err, key)
	return true
}

func (c *Controller) handleErr(err error, key interface{}) {
	if err == nil {
		// Forget about the #AddRateLimited history of the key on every successful synchronization.
		// This ensures that future processing of updates for this key is not delayed because of
		// an outdated error history.
		c.queue.Forget(key)
		return
	}
	glog.Errorf("Requeuing %v: %v", key, err)

	// This controller retries 5 times if something goes wrong. After that, it stops trying.
	if c.queue.NumRequeues(key) < 5 {
		// Re-enqueue the key rate limited. Based on the rate limiter on the
		// queue and the re-enqueue history, the key will be processed later again.
		c.queue.AddRateLimited(key)
		return
	}

	glog.Infof("Dropping %v. Too many errors", key)
	c.queue.Forget(key)
}

func (c *Controller) reconcile(key string) error {
	obj, exists, err := c.nodes.Informer().GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}

	if !exists {
		return c.delNode(key)
	}

	return c.addNode(key, obj.(*v1.Node))
}

func (c *Controller) addNode(key string, node *v1.Node) error {
	if c.store[key] == nil {

		listener, err := net.Listen("tcp", "127.0.0.1:")
		if err != nil {
			return err
		}

		glog.Infof("Listening to node %v on %v", key, listener.Addr())

		c.store[key] = listener
		c.tunnel.AddAddr(listener, nil, node.Spec.ExternalID)
	} else {
		glog.V(5).Infof("Already listening on this node... Skipping %v", key)
	}
	return nil
}

func (c *Controller) delNode(key string) error {
	listener := c.store[key]
	if listener != nil {
		glog.Infof("Deleting node %v", key)
		c.tunnel.DeleteAddr(listener, nil)
		listener.Close()
		c.store[key] = nil
	} else {
		glog.V(5).Infof("Not listening on this node... Skipping %v", key)
	}
	return nil
}
<commit_msg>added iptables monkeying<commit_after>package server

import (
	"bytes"
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/koding/tunnel"
	"github.com/sapcc/kubernikus/pkg/util/iptables"
	"k8s.io/apimachinery/pkg/util/wait"
	informers "k8s.io/client-go/informers/core/v1"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
	utilexec "k8s.io/utils/exec"
)

const (
	KUBERNIKUS_TUNNELS iptables.Chain = "KUBERNIKUS-TUNNELS"
)

type Controller struct {
	nodes    informers.NodeInformer
	tunnel   *tunnel.Server
	queue    workqueue.RateLimitingInterface
	store    map[string]net.Listener
	iptables iptables.Interface
}

func NewController(informer informers.NodeInformer, tunnel *tunnel.Server) *Controller {
	c := &Controller{
		nodes:    informer,
		tunnel:   tunnel,
		queue:    workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(5*time.Second, 300*time.Second)),
		store:    make(map[string]net.Listener),
		iptables: iptables.New(utilexec.New(), iptables.ProtocolIpv4),
	}

	c.nodes.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(obj)
			if err == nil {
				c.queue.Add(key)
			}
		},
		DeleteFunc: func(obj interface{}) {
			key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
			if err == nil {
				c.queue.Add(key)
			}
		},
	})

	return c
}

func (c *Controller) Run(threadiness int, stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer c.queue.ShutDown()
	defer wg.Done()
	wg.Add(1)
	glog.Infof(`Starting WormholeGenerator with %d workers`, threadiness)

	for i := 0; i < threadiness; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	ticker := time.NewTicker(5 * time.Minute)
	go func() {
		for {
			select {
			case <-ticker.C:
				glog.V(5).Infof("Running periodic recheck. Queuing all known nodes...")
				for key, _ := range c.store {
					c.queue.Add(key)
				}
			case <-stopCh:
				ticker.Stop()
				return
			}
		}
	}()

	<-stopCh
}

func (c *Controller) runWorker() {
	for c.processNextWorkItem() {
	}
}

func (c *Controller) processNextWorkItem() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)

	// Invoke the method containing the business logic
	err := c.reconcile(key.(string))
	c.handleErr(err, key)
	return true
}

func (c *Controller) handleErr(err error, key interface{}) {
	if err == nil {
		// Forget about the #AddRateLimited history of the key on every successful synchronization.
		// This ensures that future processing of updates for this key is not delayed because of
		// an outdated error history.
		c.queue.Forget(key)
		return
	}
	glog.Errorf("Requeuing %v: %v", key, err)

	// This controller retries 5 times if something goes wrong. After that, it stops trying.
	if c.queue.NumRequeues(key) < 5 {
		// Re-enqueue the key rate limited. Based on the rate limiter on the
		// queue and the re-enqueue history, the key will be processed later again.
		c.queue.AddRateLimited(key)
		return
	}

	glog.Infof("Dropping %v. Too many errors", key)
	c.queue.Forget(key)
}

func (c *Controller) reconcile(key string) error {
	obj, exists, err := c.nodes.Informer().GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}

	if !exists {
		return c.delNode(key)
	}

	return c.addNode(key, obj.(*v1.Node))
}

func (c *Controller) addNode(key string, node *v1.Node) error {
	if c.store[key] == nil {

		listener, err := net.Listen("tcp", "127.0.0.1:")
		if err != nil {
			return err
		}

		glog.Infof("Listening to node %v on %v", key, listener.Addr())

		c.store[key] = listener
		c.tunnel.AddAddr(listener, nil, node.Spec.ExternalID)

		if err := c.redoIPTablesSpratz(); err != nil {
			return err
		}
	} else {
		glog.V(5).Infof("Already listening on this node... Skipping %v", key)
	}
	return nil
}

func (c *Controller) delNode(key string) error {
	listener := c.store[key]
	if listener != nil {
		glog.Infof("Deleting node %v", key)
		c.tunnel.DeleteAddr(listener, nil)
		listener.Close()
		c.store[key] = nil

		if err := c.redoIPTablesSpratz(); err != nil {
			return err
		}
	} else {
		glog.V(5).Infof("Not listening on this node... Skipping %v", key)
	}
	return nil
}

func (c *Controller) redoIPTablesSpratz() error {
	table := iptables.TableFilter

	if _, err := c.iptables.EnsureChain(table, KUBERNIKUS_TUNNELS); err != nil {
		glog.Errorf("Failed to ensure that %s chain %s exists: %v", table, KUBERNIKUS_TUNNELS, err)
		return err
	}

	args := []string{"-m", "comment", "--comment", "kubernikus tunnels", "-j", string(KUBERNIKUS_TUNNELS)}
	if _, err := c.iptables.EnsureRule(iptables.Append, table, iptables.ChainInput, args...); err != nil {
		glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", table, iptables.ChainInput, KUBERNIKUS_TUNNELS, err)
		return err
	}

	iptablesSaveRaw := bytes.NewBuffer(nil)
	existingFilterChains := make(map[iptables.Chain]string)
	err := c.iptables.SaveInto(table, iptablesSaveRaw)
	if err != nil {
		glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err)
	} else {
		existingFilterChains = iptables.GetChainLines(table, iptablesSaveRaw.Bytes())
	}

	filterChains := bytes.NewBuffer(nil)
	filterRules := bytes.NewBuffer(nil)
	writeLine(filterChains, "*filter")
	if chain, ok := existingFilterChains[KUBERNIKUS_TUNNELS]; ok {
		writeLine(filterChains, chain)
	} else {
		writeLine(filterChains, iptables.MakeChainLine(KUBERNIKUS_TUNNELS))
	}

	for key, _ := range c.store {
		err := c.writeTunnelRedirect(key, filterRules)
		if err != nil {
			return err
		}
	}

	writeLine(filterRules, "COMMIT")

	lines := append(filterChains.Bytes(), filterRules.Bytes()...)
	glog.V(6).Infof("Restoring iptables rules: %s", lines)
	err = c.iptables.RestoreAll(lines, iptables.NoFlushTables, iptables.RestoreCounters)
	if err != nil {
		glog.Errorf("Failed to execute iptables-restore: %v", err)
		return err
	}

	return nil
}

func (c *Controller) writeTunnelRedirect(key string, filterRules *bytes.Buffer) error {
	obj, exists, err := c.nodes.Informer().GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}

	if !exists {
		return nil
	}

	node := obj.(*v1.Node)
	ip, err := GetNodeHostIP(node)
	if err != nil {
		return err
	}

	port := c.store[key].Addr().(*net.TCPAddr).Port

	writeLine(filterRules,
		"-A", string(KUBERNIKUS_TUNNELS),
		"-m", "comment", "--comment", fmt.Sprintf(`"tunnel to %v"`, key),
		"-t", "nat",
		"-I", "PREROUTING",
		"-p", "tcp",
		"--dst", ip.String(),
		"--dport", "22",
		"--to-ports", fmt.Sprintf("%v", port),
		"-j", "REDIRECT",
	)

	return nil
}

func writeLine(buf *bytes.Buffer, words ...string) {
	buf.WriteString(strings.Join(words, " ") + "\n")
}

func GetNodeHostIP(node *v1.Node) (net.IP, error) {
	addresses := node.Status.Addresses
	addressMap := make(map[v1.NodeAddressType][]v1.NodeAddress)
	for i := range addresses {
		addressMap[addresses[i].Type] = append(addressMap[addresses[i].Type], addresses[i])
	}
	if addresses, ok := addressMap[v1.NodeInternalIP]; ok {
		return net.ParseIP(addresses[0].Address), nil
	}
	if addresses, ok := addressMap[v1.NodeExternalIP]; ok {
		return net.ParseIP(addresses[0].Address), nil
	}
	return nil, fmt.Errorf("host IP unknown; known addresses: %v", addresses)
}
<|endoftext|>


<commit_before>package build

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/drone/drone-cli/drone/internal"
	"github.com/urfave/cli"
)

var buildStartCmd = cli.Command{
	Name:      "start",
	Usage:     "start a build",
	ArgsUsage: "<repo/name> [build]",
	Action:    buildStart,
	Flags: []cli.Flag{
		cli.StringSliceFlag{
			Name:  "param, p",
			Usage: "custom parameters to be injected into the job environment. Format: KEY=value",
		},
	},
}

func buildStart(c *cli.Context) (err error) {
	repo := c.Args().First()
	owner, name, err := internal.ParseRepo(repo)
	if err != nil {
		return err
	}

	client, err := internal.NewClient(c)
	if err != nil {
		return err
	}

	buildArg := c.Args().Get(1)
	var number int
	if buildArg == "last" {
		// Fetch the build number from the last build
		build, err := client.BuildLast(owner, name, "")
		if err != nil {
			return err
		}
		number = build.Number
	} else {
		if len(buildArg) == 0 {
			return errors.New("Missing job number")
		}
		number, err = strconv.Atoi(buildArg)
		if err != nil {
			return err
		}
	}

	params := internal.ParseKeyPair(c.StringSlice("param"))

	build, err := client.BuildStart(owner, name, number, params)
	if err != nil {
		return err
	}

	fmt.Printf("Starting build %s/%s#%d\n", owner, name, build.Number)
	return nil
}
<commit_msg>Revert build_start.go<commit_after>package build

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/drone/drone-cli/drone/internal"
	"github.com/urfave/cli"
)

var buildStartCmd = cli.Command{
	Name:      "start",
	Usage:     "start a build",
	ArgsUsage: "<repo/name> [build]",
	Action:    buildStart,
	Flags: []cli.Flag{
		cli.StringSliceFlag{
			Name:  "param, p",
			Usage: "custom parameters to be injected into the job environment. Format: KEY=value",
		},
	},
}

func buildStart(c *cli.Context) (err error) {
	repo := c.Args().First()
	owner, name, err := internal.ParseRepo(repo)
	if err != nil {
		return err
	}

	client, err := internal.NewClient(c)
	if err != nil {
		return err
	}

	buildArg := c.Args().Get(1)
	var number int
	if buildArg == "last" {
		// Fetch the build number from the last build
		build, err := client.BuildLast(owner, name, "")
		if err != nil {
			return err
		}
		number = build.Number
	} else {
		if len(buildArg) == 0 {
			return errors.New("missing job number")
		}
		number, err = strconv.Atoi(buildArg)
		if err != nil {
			return err
		}
	}

	params := internal.ParseKeyPair(c.StringSlice("param"))

	build, err := client.BuildStart(owner, name, number, params)
	if err != nil {
		return err
	}

	fmt.Printf("Starting build %s/%s#%d\n", owner, name, build.Number)
	return nil
}
<|endoftext|>


<commit_before>package drouter

import (
	"fmt"
	"net"
	"testing"
	"time"

	"github.com/TrilliumIT/iputil"
	dockerTypes "github.com/docker/engine-api/types"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/vishvananda/netlink"
)

type simulation struct {
	dr         *distributedRouter
	c          []*container
	n          []*dockerTypes.NetworkResource
	hns        *netlink.Handle
	cb         *simCallbacks
	assert     *assert.Assertions
	require    *require.Assertions
	c0routes   []netlink.Route
	c1routes   []netlink.Route
	hostRoutes []netlink.Route
}

type simCallbacks struct {
	assertInit     func()
	assertC2Start  func()
	assertN3Add    func()
	assertC2Stop   func()
	assertN3Remove func()
	assertDeinit   func()
}

func newSimCallbacks() *simCallbacks {
	return &simCallbacks{
		assertInit:     func() {},
		assertC2Start:  func() {},
		assertN3Add:    func() {},
		assertC2Stop:   func() {},
		assertN3Remove: func() {},
		assertDeinit:   func() {},
	}
}

func newSimulation(opts *DistributedRouterOptions, t *testing.T) (*simulation, error) {
	assert := assert.New(t)
	require := require.New(t)

	hns, err := netlinkHandleFromPid(1)
	if err != nil {
		return nil, err
	}

	dr, err := newDistributedRouter(opts)
	if err != nil {
		return nil, err
	}

	return &simulation{
		dr:      dr,
		c:       make([]*container, 4),
		n:       make([]*dockerTypes.NetworkResource, 4),
		assert:  assert,
		require: require,
		hns:     hns,
		cb:      newSimCallbacks(),
	}, nil
}

func (st *simulation) runV4() error {
	var err error

	//capture host routes before anything
	st.hostRoutes, err = st.hns.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get host initial routes.")
	for _, r := range st.hostRoutes {
		fmt.Printf("%+v\n", r)
	}

	//create first 3 networks
	fmt.Println("Creating networks 0, 1, and 2.")

	for i := 0; i < 3; i++ {
		st.n[i], err = createNetwork(i, i != 0)
		st.require.NoError(err, "Failed to create n%v.", i)
		defer func(n *dockerTypes.NetworkResource) {
			st.require.NoError(dc.NetworkRemove(bg, n.ID), "Failed to remove %v.", n.Name)
		}(st.n[i])
	}

	//create first 2 containers
	fmt.Println("Creating containers 0-1.")
	for i := 0; i < 2; i++ {
		st.c[i], err = createContainer(i, st.n[i].Name)
		st.require.NoError(err, "Failed to get container object for c%v.", i)
		defer func(c *container) { st.assert.NoError(c.remove(), "Failed to remove %v.", c.id) }(st.c[i])
	}

	st.c0routes, err = st.c[0].handle.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get c0 initial routes.")

	st.c1routes, err = st.c[1].handle.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get c1 initial routes.")

	//Get DRouter going
	quit := make(chan struct{})
	stopChan = quit

	ech := make(chan error)
	go func() {
		fmt.Println("Starting DRouter.")
		ech <- st.dr.start()
	}()

	startDelay := time.NewTimer(10 * time.Second)
	select {
	case <-startDelay.C:
		err = nil
	case err = <-ech:
	}
	fmt.Println("DRouter started.")
	st.require.NoError(err, "Run() returned an error.")

	//check c0 routes
	st.checkC0Routes()

	//global initial assertions
	if drn, ok := st.dr.getNetwork(st.n[0].ID); ok {
		st.assert.False(drn.isConnected(), "drouter should not be connected to n0.")
	}

	drn, ok := st.dr.getNetwork(st.n[1].ID)
	st.assert.True(ok, "should have learned n1 by now.")
	st.assert.True(drn.isConnected(), "drouter should be connected to n1.")

	if drn, ok = st.dr.getNetwork(st.n[2].ID); ok {
		st.assert.Equal(aggressive, drn.isConnected(), "drouter should be connected to n2 in aggressive mode.")
	}

	//DRouter init callback assertions
	st.cb.assertInit()
	checkLogs(st.assert)

	//EVENT: create c2
	st.c[2], err = createContainer(2, st.n[2].Name)
	st.require.NoError(err, "Failed to create c2.")
	time.Sleep(5 * time.Second)

	//check c0 routes
	st.checkC0Routes()

	//global c2start assertions
	drn, ok = st.dr.getNetwork(st.n[1].ID)
	st.assert.True(ok, "should have learned n1 by now.")
	st.assert.True(drn.isConnected(), "drouter should be connected to n1.")

	drn, ok = st.dr.getNetwork(st.n[2].ID)
	st.assert.True(ok, "should have learned n2 by now.")
	st.assert.True(drn.isConnected(), "drouter should be connected to n2.")

	//c2 start callback assertions
	st.cb.assertC2Start()
	checkLogs(st.assert)

	//EVENT: create n3
	st.n[3], err = createNetwork(3, true)
	st.assert.NoError(err, "Failed to create n3.")
	//sleep to give aggressive time to connect to n3
	time.Sleep(10 * time.Second)

	//check c0 routes
	st.checkC0Routes()

	//global n3add assertions
	if drn, ok = st.dr.getNetwork(st.n[3].ID); ok {
		st.assert.Equal(aggressive, drn.isConnected(), "drouter should be connected to n3 in aggressive mode.")
	}

	//n3add callback assertions
	st.cb.assertN3Add()
	checkLogs(st.assert)

	//EVENT: disconnect from n3, then remove it
	//we have to disconnect first, because so would an admin
	if drn, ok = st.dr.getNetwork(st.n[3].ID); ok && drn.isConnected() {
		st.require.NoError(dc.NetworkDisconnect(bg, st.n[3].ID, selfContainerID, false), "Failed to disconnect drouter from n3.")
		time.Sleep(5 * time.Second)
		st.assert.True(drn.adminDown, "The adminDown flag should be true after a manual disconnect.")
	}

	//admin now deletes the network
	st.assert.NoError(dc.NetworkRemove(bg, st.n[3].ID))
	time.Sleep(5 * time.Second)

	//check c0 routes
	st.checkC0Routes()

	//global n3remove assertions
	drn, ok = st.dr.getNetwork(st.n[1].ID)
	st.assert.True(ok, "should have learned n1 by now.")
	st.assert.True(drn.isConnected(), "drouter should still be connected to n1.")

	drn, ok = st.dr.getNetwork(st.n[2].ID)
	st.assert.True(ok, "should have learned n2 by now.")
	st.assert.True(drn.isConnected(), "drouter should still be connected to n2.")

	drn, ok = st.dr.getNetwork(st.n[3].ID)
	if ok {
		st.assert.False(drn.isConnected(), "drouter should not be connected to n3.")
	}

	//n3remove callbacks
	st.cb.assertN3Remove()
	checkLogs(st.assert)

	//EVENT: stop c2
	st.assert.NoError(st.c[2].remove(), "Failed to remove c2.")
	time.Sleep(5 * time.Second)

	//check c0 routes
	st.checkC0Routes()

	if drn, ok = st.dr.getNetwork(st.n[2].ID); ok {
		st.assert.Equal(aggressive, drn.isConnected(), "drouter should still be connected to n2 in aggressive mode.")
	}

	//c2 stop callback
	st.cb.assertC2Stop()
	checkLogs(st.assert)

	st.assert.Equal(aggressive, st.handleContainsRoute(st.c[1].handle, testNets[2], nil), "c1 should have a route to n2 in aggressive mode.")

	//EVENT: Now test quitting
	fmt.Println("Stopping DRouter.")
	close(quit)

	st.assert.NoError(<-ech, "Error during drouter shutdown.")
	fmt.Println("DRouter stopped.")
	time.Sleep(5 * time.Second)

	//check host routes
	hostNewRoutes, err := st.hns.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get host routes after dr stop.")
	for _, r := range hostNewRoutes {
		fmt.Printf("%+v\n", r)
	}
	st.assert.EqualValues(st.hostRoutes, hostNewRoutes, "Host routes should be returned to original state.")

	//check c0 routes
	st.checkC0Routes()

	//check c1 routes
	c1newRoutes, err := st.c[1].handle.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get c1 routes after dr start.")
	st.assert.EqualValues(st.c1routes, c1newRoutes, "c1 routes should be returned to original state.")

	//TODO: more global quit assertions here

	//Deinit callback
	st.cb.assertDeinit()
	checkLogs(st.assert)

	return nil
}

func (st *simulation) checkC0Routes() {
	c0newRoutes, err := st.c[0].handle.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get c0 routes after dr start.")
	st.assert.EqualValues(st.c0routes, c0newRoutes, "Should not modify c0 routes.")
}

func (st *simulation) handleContainsRoute(h *netlink.Handle, to *net.IPNet, via *net.IP) bool {
	routes, err := h.RouteList(nil, netlink.FAMILY_ALL)
	st.assert.NoError(err, "Failed to get routes from handle.")

	for _, r := range routes {
		if r.Dst == nil || r.Gw == nil {
			continue
		}

		if iputil.SubnetEqualSubnet(r.Dst, to) && (via == nil || r.Gw.Equal(*via)) {
			return true
		}
	}
	return false
}
<commit_msg>remove debug prints<commit_after>package drouter

import (
	"fmt"
	"net"
	"testing"
	"time"

	"github.com/TrilliumIT/iputil"
	dockerTypes "github.com/docker/engine-api/types"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/vishvananda/netlink"
)

type simulation struct {
	dr         *distributedRouter
	c          []*container
	n          []*dockerTypes.NetworkResource
	hns        *netlink.Handle
	cb         *simCallbacks
	assert     *assert.Assertions
	require    *require.Assertions
	c0routes   []netlink.Route
	c1routes   []netlink.Route
	hostRoutes []netlink.Route
}

type simCallbacks struct {
	assertInit     func()
	assertC2Start  func()
	assertN3Add    func()
	assertC2Stop   func()
	assertN3Remove func()
	assertDeinit   func()
}

func newSimCallbacks() *simCallbacks {
	return &simCallbacks{
		assertInit:     func() {},
		assertC2Start:  func() {},
		assertN3Add:    func() {},
		assertC2Stop:   func() {},
		assertN3Remove: func() {},
		assertDeinit:   func() {},
	}
}

func newSimulation(opts *DistributedRouterOptions, t *testing.T) (*simulation, error) {
	assert := assert.New(t)
	require := require.New(t)

	hns, err := netlinkHandleFromPid(1)
	if err != nil {
		return nil, err
	}

	dr, err := newDistributedRouter(opts)
	if err != nil {
		return nil, err
	}

	return &simulation{
		dr:      dr,
		c:       make([]*container, 4),
		n:       make([]*dockerTypes.NetworkResource, 4),
		assert:  assert,
		require: require,
		hns:     hns,
		cb:      newSimCallbacks(),
	}, nil
}

func (st *simulation) runV4() error {
	var err error

	//capture host routes before anything
	st.hostRoutes, err = st.hns.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get host initial routes.")

	//create first 3 networks
	fmt.Println("Creating networks 0, 1, and 2.")

	for i := 0; i < 3; i++ {
		st.n[i], err = createNetwork(i, i != 0)
		st.require.NoError(err, "Failed to create n%v.", i)
		defer func(n *dockerTypes.NetworkResource) {
			st.require.NoError(dc.NetworkRemove(bg, n.ID), "Failed to remove %v.", n.Name)
		}(st.n[i])
	}

	//create first 2 containers
	fmt.Println("Creating containers 0-1.")
	for i := 0; i < 2; i++ {
		st.c[i], err = createContainer(i, st.n[i].Name)
		st.require.NoError(err, "Failed to get container object for c%v.", i)
		defer func(c *container) { st.assert.NoError(c.remove(), "Failed to remove %v.", c.id) }(st.c[i])
	}

	st.c0routes, err = st.c[0].handle.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get c0 initial routes.")

	st.c1routes, err = st.c[1].handle.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get c1 initial routes.")

	//Get DRouter going
	quit := make(chan struct{})
	stopChan = quit

	ech := make(chan error)
	go func() {
		fmt.Println("Starting DRouter.")
		ech <- st.dr.start()
	}()

	startDelay := time.NewTimer(10 * time.Second)
	select {
	case <-startDelay.C:
		err = nil
	case err = <-ech:
	}
	fmt.Println("DRouter started.")
	st.require.NoError(err, "Run() returned an error.")

	//check c0 routes
	st.checkC0Routes()

	//global initial assertions
	if drn, ok := st.dr.getNetwork(st.n[0].ID); ok {
		st.assert.False(drn.isConnected(), "drouter should not be connected to n0.")
	}

	drn, ok := st.dr.getNetwork(st.n[1].ID)
	st.assert.True(ok, "should have learned n1 by now.")
	st.assert.True(drn.isConnected(), "drouter should be connected to n1.")

	if drn, ok = st.dr.getNetwork(st.n[2].ID); ok {
		st.assert.Equal(aggressive, drn.isConnected(), "drouter should be connected to n2 in aggressive mode.")
	}

	//DRouter init callback assertions
	st.cb.assertInit()
	checkLogs(st.assert)

	//EVENT: create c2
	st.c[2], err = createContainer(2, st.n[2].Name)
	st.require.NoError(err, "Failed to create c2.")
	time.Sleep(5 * time.Second)

	//check c0 routes
	st.checkC0Routes()

	//global c2start assertions
	drn, ok = st.dr.getNetwork(st.n[1].ID)
	st.assert.True(ok, "should have learned n1 by now.")
	st.assert.True(drn.isConnected(), "drouter should be connected to n1.")

	drn, ok = st.dr.getNetwork(st.n[2].ID)
	st.assert.True(ok, "should have learned n2 by now.")
	st.assert.True(drn.isConnected(), "drouter should be connected to n2.")

	//c2 start callback assertions
	st.cb.assertC2Start()
	checkLogs(st.assert)

	//EVENT: create n3
	st.n[3], err = createNetwork(3, true)
	st.assert.NoError(err, "Failed to create n3.")
	//sleep to give aggressive time to connect to n3
	time.Sleep(10 * time.Second)

	//check c0 routes
	st.checkC0Routes()

	//global n3add assertions
	if drn, ok = st.dr.getNetwork(st.n[3].ID); ok {
		st.assert.Equal(aggressive, drn.isConnected(), "drouter should be connected to n3 in aggressive mode.")
	}

	//n3add callback assertions
	st.cb.assertN3Add()
	checkLogs(st.assert)

	//EVENT: disconnect from n3, then remove it
	//we have to disconnect first, because so would an admin
	if drn, ok = st.dr.getNetwork(st.n[3].ID); ok && drn.isConnected() {
		st.require.NoError(dc.NetworkDisconnect(bg, st.n[3].ID, selfContainerID, false), "Failed to disconnect drouter from n3.")
		time.Sleep(5 * time.Second)
		st.assert.True(drn.adminDown, "The adminDown flag should be true after a manual disconnect.")
	}

	//admin now deletes the network
	st.assert.NoError(dc.NetworkRemove(bg, st.n[3].ID))
	time.Sleep(5 * time.Second)

	//check c0 routes
	st.checkC0Routes()

	//global n3remove assertions
	drn, ok = st.dr.getNetwork(st.n[1].ID)
	st.assert.True(ok, "should have learned n1 by now.")
	st.assert.True(drn.isConnected(), "drouter should still be connected to n1.")

	drn, ok = st.dr.getNetwork(st.n[2].ID)
	st.assert.True(ok, "should have learned n2 by now.")
	st.assert.True(drn.isConnected(), "drouter should still be connected to n2.")

	drn, ok = st.dr.getNetwork(st.n[3].ID)
	if ok {
		st.assert.False(drn.isConnected(), "drouter should not be connected to n3.")
	}

	//n3remove callbacks
	st.cb.assertN3Remove()
	checkLogs(st.assert)

	//EVENT: stop c2
	st.assert.NoError(st.c[2].remove(), "Failed to remove c2.")
	time.Sleep(5 * time.Second)

	//check c0 routes
	st.checkC0Routes()

	if drn, ok = st.dr.getNetwork(st.n[2].ID); ok {
		st.assert.Equal(aggressive, drn.isConnected(), "drouter should still be connected to n2 in aggressive mode.")
	}

	//c2 stop callback
	st.cb.assertC2Stop()
	checkLogs(st.assert)

	st.assert.Equal(aggressive, st.handleContainsRoute(st.c[1].handle, testNets[2], nil), "c1 should have a route to n2 in aggressive mode.")

	//EVENT: Now test quitting
	fmt.Println("Stopping DRouter.")
	close(quit)

	st.assert.NoError(<-ech, "Error during drouter shutdown.")
	fmt.Println("DRouter stopped.")
	time.Sleep(5 * time.Second)

	//check host routes
	hostNewRoutes, err := st.hns.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get host routes after dr stop.")
	for _, r := range hostNewRoutes {
		fmt.Printf("%+v\n", r)
	}
	st.assert.EqualValues(st.hostRoutes, hostNewRoutes, "Host routes should be returned to original state.")

	//check c0 routes
	st.checkC0Routes()

	//check c1 routes
	c1newRoutes, err := st.c[1].handle.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get c1 routes after dr start.")
	st.assert.EqualValues(st.c1routes, c1newRoutes, "c1 routes should be returned to original state.")

	//TODO: more global quit assertions here

	//Deinit callback
	st.cb.assertDeinit()
	checkLogs(st.assert)

	return nil
}

func (st *simulation) checkC0Routes() {
	c0newRoutes, err := st.c[0].handle.RouteList(nil, netlink.FAMILY_V4)
	st.require.NoError(err, "Failed to get c0 routes after dr start.")
	st.assert.EqualValues(st.c0routes, c0newRoutes, "Should not modify c0 routes.")
}

func (st *simulation) handleContainsRoute(h *netlink.Handle, to *net.IPNet, via *net.IP) bool {
	routes, err := h.RouteList(nil, netlink.FAMILY_ALL)
	st.assert.NoError(err, "Failed to get routes from handle.")

	for _, r := range routes {
		if r.Dst == nil || r.Gw == nil {
			continue
		}

		if iputil.SubnetEqualSubnet(r.Dst, to) && (via == nil || r.Gw.Equal(*via)) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package chow

import (
	"../../primitives/number"
	"../saes"
)

type Construction struct {
	TBox [10][16][256]byte
}

func GenerateTables(key [16]byte) (table [10][16][256]byte) {
	constr := saes.Construction{key}
	roundKeys := constr.StretchedKey()

	// Apply ShiftRows to round keys 0 to 9.
	for k := 0; k < 10; k++ {
		roundKeys[k] = constr.ShiftRows(roundKeys[k])
	}

	// Build T-Boxes 1 to 9
	for round := 0; round < 9; round++ {
		table[round] = [16][256]byte{}

		for place := 0; place < 16; place++ {
			table[round][place] = [256]byte{}

			for x := 0; x < 256; x++ {
				table[round][place][x] = constr.SubByte(byte(x) ^ roundKeys[round][place])
			}
		}
	}

	// 10th T-Box
	for place := 0; place < 16; place++ {
		table[9][place] = [256]byte{}

		for x := 0; x < 256; x++ {
			table[9][place][x] = constr.SubByte(byte(x)^roundKeys[9][place]) ^ roundKeys[10][place]
		}
	}

	return
}

func (constr *Construction) Encrypt(block [16]byte) [16]byte {
	for i := 0; i < 9; i++ {
		block = constr.shiftRows(block)

		for j := 0; j < 16; j++ {
			block[j] = constr.TBox[i][j][block[j]]
		}

		block = constr.mixColumns(block)
	}

	block = constr.shiftRows(block)

	for j := 0; j < 16; j++ {
		block[j] = constr.TBox[9][j][block[j]]
	}

	return block
}

func (constr *Construction) shiftRows(block [16]byte) [16]byte {
	return [16]byte{
		block[0], block[5], block[10], block[15], block[4], block[9], block[14], block[3], block[8], block[13], block[2],
		block[7], block[12], block[1], block[6], block[11],
	}
}

func (constr *Construction) mixColumns(block [16]byte) (out [16]byte) {
	for i := 0; i < 4; i++ {
		copy(out[4*i:4*(i+1)], constr.mixColumn(block[4*i:4*(i+1)]))
	}

	return out
}

func (constr *Construction) mixColumn(slice []byte) []byte {
	column := number.ArrayFieldElem{}
	for i := 0; i < 4; i++ {
		column = append(column, number.ByteFieldElem(slice[i]))
	}

	column = column.Mul(number.ArrayFieldElem{
		number.ByteFieldElem(0x02), number.ByteFieldElem(0x01),
		number.ByteFieldElem(0x01), number.ByteFieldElem(0x03),
	})

	out := make([]byte, 4)
	for i := 0; i < len(column); i++ {
		out[i] = byte(column[i])
	}

	return out
}
<commit_msg>Represent TBoxes as ByteTables.<commit_after>package chow

import (
	"../../primitives/number"
	"../../primitives/table"
	"../saes"
)

type TBox struct {
	Constr   saes.Construction
	KeyByte1 byte
	KeyByte2 byte
}

func (tbox TBox) Get(i byte) byte {
	return tbox.Constr.SubByte(i^tbox.KeyByte1) ^ tbox.KeyByte2
}

type Construction struct {
	TBox [10][16]table.ByteTable
}

func GenerateTables(key [16]byte) (table [10][16]table.ByteTable) {
	constr := saes.Construction{key}
	roundKeys := constr.StretchedKey()

	// Apply ShiftRows to round keys 0 to 9.
	for k := 0; k < 10; k++ {
		roundKeys[k] = constr.ShiftRows(roundKeys[k])
	}

	// Build T-Boxes 1 to 9
	for round := 0; round < 9; round++ {
		for place := 0; place < 16; place++ {
			table[round][place] = TBox{constr, roundKeys[round][place], 0}
		}
	}

	// 10th T-Box
	for place := 0; place < 16; place++ {
		table[9][place] = TBox{constr, roundKeys[9][place], roundKeys[10][place]}
	}

	return
}

func (constr *Construction) Encrypt(block [16]byte) [16]byte {
	for i := 0; i < 9; i++ {
		block = constr.shiftRows(block)

		for j := 0; j < 16; j++ {
			block[j] = constr.TBox[i][j].Get(block[j])
		}

		block = constr.mixColumns(block)
	}

	block = constr.shiftRows(block)

	for j := 0; j < 16; j++ {
		block[j] = constr.TBox[9][j].Get(block[j])
	}

	return block
}

func (constr *Construction) shiftRows(block [16]byte) [16]byte {
	return [16]byte{
		block[0], block[5], block[10], block[15], block[4], block[9], block[14], block[3], block[8], block[13], block[2],
		block[7], block[12], block[1], block[6], block[11],
	}
}

func (constr *Construction) mixColumns(block [16]byte) (out [16]byte) {
	for i := 0; i < 4; i++ {
		copy(out[4*i:4*(i+1)], constr.mixColumn(block[4*i:4*(i+1)]))
	}

	return out
}

func (constr *Construction) mixColumn(slice []byte) []byte {
	column := number.ArrayFieldElem{}
	for i := 0; i < 4; i++ {
		column = append(column, number.ByteFieldElem(slice[i]))
	}

	column = column.Mul(number.ArrayFieldElem{
		number.ByteFieldElem(0x02), number.ByteFieldElem(0x01),
		number.ByteFieldElem(0x01), number.ByteFieldElem(0x03),
	})

	out := make([]byte, 4)
	for i := 0; i < len(column); i++ {
		out[i] = byte(column[i])
	}

	return out
}
<|endoftext|>