<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package webhook

import (
	"context"
	"fmt"
	"net"
	"testing"

	"golang.org/x/sync/errgroup"
	"knative.dev/pkg/controller"

	// Make system.Namespace() work in tests.
	_ "knative.dev/pkg/system/testing"

	. "knative.dev/pkg/reconciler/testing"
)

func newDefaultOptions() Options {
	return Options{
		ServiceName: "webhook",
		Port:        443,
		SecretName:  "webhook-certs",
	}
}

const (
	testResourceName = "test-resource"
	user1            = "brutto@knative.dev"
)

func newNonRunningTestWebhook(t *testing.T, options Options, acs ...interface{}) (
	ctx context.Context, ac *Webhook, cancel context.CancelFunc) {
	t.Helper()

	// Create fake clients
	ctx, ctxCancel, informers := SetupFakeContextWithCancel(t)
	ctx = WithOptions(ctx, options)

	stopCb, err := controller.RunInformers(ctx.Done(), informers...)
	if err != nil {
		t.Fatalf("StartInformers() = %v", err)
	}
	cancel = func() {
		ctxCancel()
		stopCb()
	}

	ac, err = New(ctx, acs)
	if err != nil {
		t.Fatalf("Failed to create new admission controller: %v", err)
	}
	return
}

func TestRegistrationStopChanFire(t *testing.T) {
	opts := newDefaultOptions()
	_, ac, cancel := newNonRunningTestWebhook(t, opts)
	defer cancel()

	stopCh := make(chan struct{})

	var g errgroup.Group
	g.Go(func() error {
		return ac.Run(stopCh)
	})
	close(stopCh)

	if err := g.Wait(); err != nil {
		t.Fatal("Error during run: ", err)
	}
	conn, err := net.Dial("tcp", fmt.Sprintf(":%d", opts.Port))
	if err == nil {
		conn.Close()
		t.Errorf("Unexpected success to dial to port %d", opts.Port)
	}
}
<commit_msg>Change default port in webhook test to 8443. (#1242)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package webhook

import (
	"context"
	"fmt"
	"net"
	"testing"

	"golang.org/x/sync/errgroup"
	"knative.dev/pkg/controller"

	// Make system.Namespace() work in tests.
	_ "knative.dev/pkg/system/testing"

	. "knative.dev/pkg/reconciler/testing"
)

func newDefaultOptions() Options {
	return Options{
		ServiceName: "webhook",
		Port:        8443,
		SecretName:  "webhook-certs",
	}
}

const (
	testResourceName = "test-resource"
	user1            = "brutto@knative.dev"
)

func newNonRunningTestWebhook(t *testing.T, options Options, acs ...interface{}) (
	ctx context.Context, ac *Webhook, cancel context.CancelFunc) {
	t.Helper()

	// Create fake clients
	ctx, ctxCancel, informers := SetupFakeContextWithCancel(t)
	ctx = WithOptions(ctx, options)

	stopCb, err := controller.RunInformers(ctx.Done(), informers...)
	if err != nil {
		t.Fatalf("StartInformers() = %v", err)
	}
	cancel = func() {
		ctxCancel()
		stopCb()
	}

	ac, err = New(ctx, acs)
	if err != nil {
		t.Fatalf("Failed to create new admission controller: %v", err)
	}
	return
}

func TestRegistrationStopChanFire(t *testing.T) {
	opts := newDefaultOptions()
	_, ac, cancel := newNonRunningTestWebhook(t, opts)
	defer cancel()

	stopCh := make(chan struct{})

	var g errgroup.Group
	g.Go(func() error {
		return ac.Run(stopCh)
	})
	close(stopCh)

	if err := g.Wait(); err != nil {
		t.Fatal("Error during run: ", err)
	}
	conn, err := net.Dial("tcp", fmt.Sprintf(":%d", opts.Port))
	if err == nil {
		conn.Close()
		t.Errorf("Unexpected success to dial to port %d", opts.Port)
	}
}
<|endoftext|>


<commit_before>//go:build integ
// +build integ

// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helm

import (
	"fmt"
	"os"
	"path/filepath"
	"testing"

	"istio.io/istio/pkg/test/framework"
	kubecluster "istio.io/istio/pkg/test/framework/components/cluster/kube"
	"istio.io/istio/pkg/test/helm"
	"istio.io/istio/tests/util/sanitycheck"
)

// TestDefaultInstall tests Istio installation using Helm with default options
func TestDefaultInstall(t *testing.T) {
	overrideValuesStr := `
global:
  hub: %s
  tag: %s
`
	framework.
		NewTest(t).
		Features("installation.helm.default.install").
		Run(setupInstallation(overrideValuesStr))
}

// TestInstallWithFirstPartyJwt tests Istio installation using Helm
// with first-party-jwt enabled
// (TODO) remove this test when Istio no longer supports first-party-jwt
func TestInstallWithFirstPartyJwt(t *testing.T) {
	overrideValuesStr := `
global:
  hub: %s
  tag: %s
  jwtPolicy: first-party-jwt
`

	framework.
		NewTest(t).
		Features("installation.helm.firstpartyjwt.install").
		Run(func(t framework.TestContext) {
			setupInstallation(overrideValuesStr)(t)
		})
}

func setupInstallation(overrideValuesStr string) func(t framework.TestContext) {
	return func(t framework.TestContext) {
		workDir, err := t.CreateTmpDirectory("helm-install-test")
		if err != nil {
			t.Fatal("failed to create test directory")
		}
		cs := t.Clusters().Default().(*kubecluster.Cluster)
		h := helm.New(cs.Filename())

		s := t.Settings()
		overrideValues := fmt.Sprintf(overrideValuesStr, s.Image.Hub, s.Image.Tag)
		overrideValuesFile := filepath.Join(workDir, "values.yaml")
		if err := os.WriteFile(overrideValuesFile, []byte(overrideValues), os.ModePerm); err != nil {
			t.Fatalf("failed to write iop cr file: %v", err)
		}
		InstallIstio(t, cs, h, "", overrideValuesFile, ManifestsChartPath, "", true)

		VerifyInstallation(t, cs, true)
		VerifyValidation(t)

		sanitycheck.RunTrafficTest(t, t)
		t.Cleanup(func() {
			deleteIstio(t, h, cs)
		})
	}
}
<commit_msg>Dump helm failures (#41721)<commit_after>//go:build integ
// +build integ

// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helm

import (
	"fmt"
	"os"
	"path/filepath"
	"testing"

	"istio.io/istio/pkg/test/framework"
	kubecluster "istio.io/istio/pkg/test/framework/components/cluster/kube"
	"istio.io/istio/pkg/test/framework/components/namespace"
	"istio.io/istio/pkg/test/helm"
	"istio.io/istio/tests/util/sanitycheck"
)

// TestDefaultInstall tests Istio installation using Helm with default options
func TestDefaultInstall(t *testing.T) {
	overrideValuesStr := `
global:
  hub: %s
  tag: %s
`
	framework.
		NewTest(t).
		Features("installation.helm.default.install").
		Run(setupInstallation(overrideValuesStr))
}

// TestInstallWithFirstPartyJwt tests Istio installation using Helm
// with first-party-jwt enabled
// (TODO) remove this test when Istio no longer supports first-party-jwt
func TestInstallWithFirstPartyJwt(t *testing.T) {
	overrideValuesStr := `
global:
  hub: %s
  tag: %s
  jwtPolicy: first-party-jwt
`

	framework.
		NewTest(t).
		Features("installation.helm.firstpartyjwt.install").
		Run(func(t framework.TestContext) {
			setupInstallation(overrideValuesStr)(t)
		})
}

func setupInstallation(overrideValuesStr string) func(t framework.TestContext) {
	return func(t framework.TestContext) {
		workDir, err := t.CreateTmpDirectory("helm-install-test")
		if err != nil {
			t.Fatal("failed to create test directory")
		}
		cs := t.Clusters().Default().(*kubecluster.Cluster)
		h := helm.New(cs.Filename())

		s := t.Settings()
		overrideValues := fmt.Sprintf(overrideValuesStr, s.Image.Hub, s.Image.Tag)
		overrideValuesFile := filepath.Join(workDir, "values.yaml")
		if err := os.WriteFile(overrideValuesFile, []byte(overrideValues), os.ModePerm); err != nil {
			t.Fatalf("failed to write iop cr file: %v", err)
		}
		t.Cleanup(func() {
			if !t.Failed() {
				return
			}
			if t.Settings().CIMode {
				namespace.Dump(t, IstioNamespace)
			}
		})
		InstallIstio(t, cs, h, "", overrideValuesFile, ManifestsChartPath, "", true)

		VerifyInstallation(t, cs, true)
		VerifyValidation(t)

		sanitycheck.RunTrafficTest(t, t)
		t.Cleanup(func() {
			deleteIstio(t, h, cs)
		})
	}
}
<|endoftext|>


<commit_before>package worker

import (
	"fmt"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
	"strings"
	"time"
)

type eventsWorker struct {
	sm           state.Manager
	qm           queue.Manager
	conf         config.Config
	log          flotillaLog.Logger
	pollInterval time.Duration
	t            tomb.Tomb
	queue        string
	engine       *string
	s3Client     *s3.S3
}

func (ew *eventsWorker) Initialize(conf config.Config, sm state.Manager, ee engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, engine *string, qm queue.Manager) error {
	ew.pollInterval = pollInterval
	ew.conf = conf
	ew.sm = sm
	ew.qm = qm
	ew.log = log
	ew.engine = engine
	eventsQueue, err := ew.qm.QurlFor("eks.events_queue", false)

	if err != nil {
		_ = ew.log.Log("message", "Error receiving Kubernetes Event queue", "error", fmt.Sprintf("%+v", err))
		return nil
	}

	ew.queue = eventsQueue
	_ = ew.qm.Initialize(ew.conf, "eks")

	return nil
}

func (ew *eventsWorker) GetTomb() *tomb.Tomb {
	return &ew.t
}

func (ew *eventsWorker) Run() error {
	for {
		select {
		case <-ew.t.Dying():
			_ = ew.log.Log("message", "A CloudTrail worker was terminated")
			return nil
		default:
			ew.runOnce()
			time.Sleep(ew.pollInterval)
		}
	}
}

func (ew *eventsWorker) runOnce() {
	kubernetesEvent, err := ew.qm.ReceiveKubernetesEvent(ew.queue)
	if err != nil {
		_ = ew.log.Log("message", "Error receiving Kubernetes Events", "error", fmt.Sprintf("%+v", err))
		return
	}
	ew.processEvent(kubernetesEvent)
}

func (ew *eventsWorker) processEvent(kubernetesEvent state.KubernetesEvent) {
	runId := kubernetesEvent.InvolvedObject.Labels.JobName
	if !strings.HasPrefix(runId, "eks") {
		return
	}

	layout := "2020-08-31T17:27:50Z"
	timestamp, err := time.Parse(layout, kubernetesEvent.FirstTimestamp)

	if err != nil {
		timestamp = time.Now()
	}

	run, err := ew.sm.GetRun(runId)
	if err == nil {
		event := state.PodEvent{
			Timestamp:    &timestamp,
			EventType:    kubernetesEvent.Type,
			Reason:       kubernetesEvent.Reason,
			SourceObject: kubernetesEvent.InvolvedObject.Name,
			Message:      kubernetesEvent.Message,
		}

		var events state.PodEvents
		if run.PodEvents != nil {
			events = append(*run.PodEvents, event)
		} else {
			events = state.PodEvents{event}
		}
		run.PodEvents = &events
		run, err = ew.sm.UpdateRun(runId, run)
		if err != nil {
			_ = ew.log.Log("message", "error saving kubernetes events", "run", runId, "error", fmt.Sprintf("%+v", err))
		}
	}
}
<commit_msg>fixing typo<commit_after>package worker

import (
	"fmt"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
	"strings"
	"time"
)

type eventsWorker struct {
	sm           state.Manager
	qm           queue.Manager
	conf         config.Config
	log          flotillaLog.Logger
	pollInterval time.Duration
	t            tomb.Tomb
	queue        string
	engine       *string
	s3Client     *s3.S3
}

func (ew *eventsWorker) Initialize(conf config.Config, sm state.Manager, ee engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, engine *string, qm queue.Manager) error {
	ew.pollInterval = pollInterval
	ew.conf = conf
	ew.sm = sm
	ew.qm = qm
	ew.log = log
	ew.engine = engine
	eventsQueue, err := ew.qm.QurlFor(conf.GetString("eks.events_queue"), false)

	if err != nil {
		_ = ew.log.Log("message", "Error receiving Kubernetes Event queue", "error", fmt.Sprintf("%+v", err))
		return nil
	}

	ew.queue = eventsQueue
	_ = ew.qm.Initialize(ew.conf, "eks")

	return nil
}

func (ew *eventsWorker) GetTomb() *tomb.Tomb {
	return &ew.t
}

func (ew *eventsWorker) Run() error {
	for {
		select {
		case <-ew.t.Dying():
			_ = ew.log.Log("message", "A CloudTrail worker was terminated")
			return nil
		default:
			ew.runOnce()
			time.Sleep(ew.pollInterval)
		}
	}
}

func (ew *eventsWorker) runOnce() {
	kubernetesEvent, err := ew.qm.ReceiveKubernetesEvent(ew.queue)
	if err != nil {
		_ = ew.log.Log("message", "Error receiving Kubernetes Events", "error", fmt.Sprintf("%+v", err))
		return
	}
	ew.processEvent(kubernetesEvent)
}

func (ew *eventsWorker) processEvent(kubernetesEvent state.KubernetesEvent) {
	runId := kubernetesEvent.InvolvedObject.Labels.JobName
	if !strings.HasPrefix(runId, "eks") {
		return
	}

	layout := "2020-08-31T17:27:50Z"
	timestamp, err := time.Parse(layout, kubernetesEvent.FirstTimestamp)

	if err != nil {
		timestamp = time.Now()
	}

	run, err := ew.sm.GetRun(runId)
	if err == nil {
		event := state.PodEvent{
			Timestamp:    &timestamp,
			EventType:    kubernetesEvent.Type,
			Reason:       kubernetesEvent.Reason,
			SourceObject: kubernetesEvent.InvolvedObject.Name,
			Message:      kubernetesEvent.Message,
		}

		var events state.PodEvents
		if run.PodEvents != nil {
			events = append(*run.PodEvents, event)
		} else {
			events = state.PodEvents{event}
		}
		run.PodEvents = &events
		run, err = ew.sm.UpdateRun(runId, run)
		if err != nil {
			_ = ew.log.Log("message", "error saving kubernetes events", "run", runId, "error", fmt.Sprintf("%+v", err))
		}
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015, Derek Marcotte
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 * 1. Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

/* stdlib includes */
import (
	"flag"
	"os"
	"runtime"
	"time"
)

/* external includes */
import "github.com/op/go-logging"

/* definitions */

/* meat */
/* dependancy injection is for another day */
var log = logging.MustGetLogger(os.Args[0])

func main() {
	var configPath string
	var config Configuration

	flag.StringVar(&configPath, "config", "etc/hfm.conf", "Configuration file path")
	flag.Parse()

	if e := config.LoadConfiguration(configPath); e != nil {
		log.Error("Could not load configuration file %v: %+v", configPath, e)
		panic(e)
	}

	ruleDone := make(chan *RuleDriver)

	/* close enough for most applications */
	appInstance := time.Now().UnixNano()

	log.Info("Loaded %d rules.", len(config.Rules))
	log.Debug("%d goroutines - before main dispatch loop.", runtime.NumGoroutine())
	for _, rule := range config.Rules {
		log.Debug("Dispatching rule '%s'", rule.Name)
		log.Debug("%s details: %+v", rule.Name, rule)

		// driver gets its own copy of the rule, safe from
		// side effects later
		driver := RuleDriver{Rule: *rule, Done: ruleDone, AppInstance: appInstance}
		go driver.Run()
	}
	log.Debug("%d goroutines - after dispatch loop.", runtime.NumGoroutine())

	for i := 0; i < len(config.Rules); i++ {
		driver := <-ruleDone
		log.Info("'%s' completed execution.  Ran for: %v\n\n", driver.Rule.Name, driver.Last.ExecDuration)
	}

	log.Debug("%d goroutines - at the end.", runtime.NumGoroutine())
}
<commit_msg>enable syslog logging from command line<commit_after>/*
 * Copyright (c) 2015, Derek Marcotte
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 * 1. Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

/* stdlib includes */
import (
	"flag"
	"fmt"
	"log/syslog"
	"os"
	"path"
	"runtime"
	"strings"
	"time"
)

/* external includes */
import "github.com/op/go-logging"

/* definitions */

type LogConfiguration struct {
	Where    string
	Facility string
}

/* meat */
/* dependancy injection is for another day */
var log = logging.MustGetLogger(path.Base(os.Args[0]))

func configureLogging(conf LogConfiguration) error {
	conf.Where = strings.ToLower(conf.Where)
	switch conf.Where {
	case "syslog":
	case "stderr":
		return nil
	default:
		return fmt.Errorf("Invalid log location, must be one of {stderr, syslog}\n")
	}

	facilityList := map[string]syslog.Priority{
		"kern":     syslog.LOG_KERN,
		"user":     syslog.LOG_USER,
		"mail":     syslog.LOG_MAIL,
		"daemon":   syslog.LOG_DAEMON,
		"auth":     syslog.LOG_AUTH,
		"syslog":   syslog.LOG_SYSLOG,
		"lpr":      syslog.LOG_LPR,
		"news":     syslog.LOG_NEWS,
		"uucp":     syslog.LOG_UUCP,
		"cron":     syslog.LOG_CRON,
		"authpriv": syslog.LOG_AUTHPRIV,
		"ftp":      syslog.LOG_FTP,
		"local0":   syslog.LOG_LOCAL0,
		"local1":   syslog.LOG_LOCAL1,
		"local2":   syslog.LOG_LOCAL2,
		"local3":   syslog.LOG_LOCAL3,
		"local4":   syslog.LOG_LOCAL4,
		"local5":   syslog.LOG_LOCAL5,
		"local6":   syslog.LOG_LOCAL6,
		"local7":   syslog.LOG_LOCAL7,
	}

	conf.Facility = strings.ToLower(conf.Facility)

	f, ok := facilityList[conf.Facility]
	if !ok {
		return fmt.Errorf("Invalid syslog facility")
	}

	be, _ := logging.NewSyslogBackendPriority(path.Base(os.Args[0]), f)
	log.SetBackend(logging.AddModuleLevel(be))

	return nil
}

func main() {
	var configPath string
	var config Configuration

	var lc LogConfiguration

	flag.StringVar(&configPath, "config", "etc/hfm.conf", "Configuration file path")
	flag.StringVar(&lc.Where, "log", "stderr", "Where to log {stderr, syslog}")
	flag.StringVar(&lc.Facility, "facility", "local0", "Log facility (when -log set to syslog) {local0-9, user, etc}")
	flag.Parse()

	if e := configureLogging(lc); e != nil {
		fmt.Printf("Could not configure logging: %v", e)
		panic(e)
	}

	if e := config.LoadConfiguration(configPath); e != nil {
		fmt.Printf("Could not load configuration file %v: %+v", configPath, e)
		panic(e)
	}

	ruleDone := make(chan *RuleDriver)

	/* close enough for most applications */
	appInstance := time.Now().UnixNano()

	log.Info("Loaded %d rules.", len(config.Rules))
	log.Debug("%d goroutines - before main dispatch loop.", runtime.NumGoroutine())
	for _, rule := range config.Rules {
		log.Debug("Dispatching rule '%s'", rule.Name)
		log.Debug("%s details: %+v", rule.Name, rule)

		// driver gets its own copy of the rule, safe from
		// side effects later
		driver := RuleDriver{Rule: *rule, Done: ruleDone, AppInstance: appInstance}
		go driver.Run()
	}
	log.Debug("%d goroutines - after dispatch loop.", runtime.NumGoroutine())

	for i := 0; i < len(config.Rules); i++ {
		driver := <-ruleDone
		log.Info("'%s' completed execution.  Ran for: %v\n\n", driver.Rule.Name, driver.Last.ExecDuration)
	}

	log.Debug("%d goroutines - at the end.", runtime.NumGoroutine())
}
<|endoftext|>


<commit_before>package commands

import (
	"time"

	"github.com/AlexanderThaller/lablog/src/data"
	"github.com/AlexanderThaller/lablog/src/helper"
	"github.com/juju/errgo"

	"github.com/AlexanderThaller/cobra"
)

var flagAddTimeStamp time.Time
var flagAddTimeStampRaw string

func init() {
	flagAddTimeStamp = time.Now()

	cmdAdd.PersistentFlags().StringVarP(&flagAddTimeStampRaw, "timestamp", "t",
		flagAddTimeStamp.String(), "The timestamp for which to record the note.")
}

var cmdAdd = &cobra.Command{
	Use:   "add [command]",
	Short: "Add a new entry to the log.",
	Long:  `Add a new entry like a note or a todo to the log. You have to specify a project for which we want to record the log for.`,
	Run:   runCmdAdd,
}

func runCmdAdd(cmd *cobra.Command, args []string) {
}

var cmdAddNote = &cobra.Command{
	Use:   "note",
	Short: "Add a new note to the log.",
	Long:  `Add a new note to the log which can have a timestamp and an free form value for text.`,
	Run:   runCmdAddNote,
}

func runCmdAddNote(cmd *cobra.Command, args []string) {
	project, timestamp, value, err := helper.ArgsToEntryValues(args, flagAddTimeStamp, flagAddTimeStampRaw)
	helper.ErrExit(errgo.Notef(err, "can not convert args to entry usable values"))

	note := data.Note{
		Value:     value,
		TimeStamp: timestamp,
	}

	err = helper.RecordEntry(flagDataDir, project, note)
	helper.ErrExit(errgo.Notef(err, "can not record note to store"))
}

var cmdAddTodo = &cobra.Command{
	Use:   "todo [command]",
	Short: "Add a new todo to the log.",
	Long:  `Add a new todo to the log which can have a timestamp, a toggle state (if its active or not) and an free form value for text.`,
	Run:   runCmdAddTodo,
}

func runCmdAddTodo(cmd *cobra.Command, args []string) {
}

var cmdAddTodoActive = &cobra.Command{
	Use:   "active",
	Short: "Add a new todo to the log and mark it as active.",
	Long:  `Add a new todo to the log which can have a timestamp, is marked as active and an free form value for text.`,
	Run:   runCmdAddTodoActive,
}

func runCmdAddTodoActive(cmd *cobra.Command, args []string) {
	project, todo, err := helper.ArgsToTodo(args, flagAddTimeStamp, flagAddTimeStampRaw)
	helper.ErrExit(errgo.Notef(err, "can not convert args to todo"))

	todo.Active = true

	err = helper.RecordEntry(flagDataDir, project, todo)
	helper.ErrExit(errgo.Notef(err, "can not record todo to store"))
}

var cmdAddTodoInActive = &cobra.Command{
	Use:   "inactive",
	Short: "Add a new todo to the log and mark it as inactive.",
	Long:  `Add a new todo to the log which can have a timestamp, is marked as inactive and an free form value for text.`,
	Run:   runCmdAddTodoInActive,
}

func runCmdAddTodoInActive(cmd *cobra.Command, args []string) {
	project, todo, err := helper.ArgsToTodo(args, flagAddTimeStamp, flagAddTimeStampRaw)
	helper.ErrExit(errgo.Notef(err, "can not convert args to todo"))

	todo.Active = false

	err = helper.RecordEntry(flagDataDir, project, todo)
	helper.ErrExit(errgo.Notef(err, "can not record todo to store"))
}
<commit_msg>Will now show help for commands that dont do anything.<commit_after>package commands

import (
	"time"

	"github.com/AlexanderThaller/lablog/src/data"
	"github.com/AlexanderThaller/lablog/src/helper"
	"github.com/juju/errgo"

	"github.com/AlexanderThaller/cobra"
)

var flagAddTimeStamp time.Time
var flagAddTimeStampRaw string

func init() {
	flagAddTimeStamp = time.Now()

	cmdAdd.PersistentFlags().StringVarP(&flagAddTimeStampRaw, "timestamp", "t",
		flagAddTimeStamp.String(), "The timestamp for which to record the note.")
}

var cmdAdd = &cobra.Command{
	Use:   "add [command]",
	Short: "Add a new entry to the log.",
	Long:  `Add a new entry like a note or a todo to the log. You have to specify a project for which we want to record the log for.`,
	Run:   runCmdAdd,
}

func runCmdAdd(cmd *cobra.Command, args []string) {
	cmd.Help()
}

var cmdAddNote = &cobra.Command{
	Use:   "note",
	Short: "Add a new note to the log.",
	Long:  `Add a new note to the log which can have a timestamp and an free form value for text.`,
	Run:   runCmdAddNote,
}

func runCmdAddNote(cmd *cobra.Command, args []string) {
	project, timestamp, value, err := helper.ArgsToEntryValues(args, flagAddTimeStamp, flagAddTimeStampRaw)
	helper.ErrExit(errgo.Notef(err, "can not convert args to entry usable values"))

	note := data.Note{
		Value:     value,
		TimeStamp: timestamp,
	}

	err = helper.RecordEntry(flagDataDir, project, note)
	helper.ErrExit(errgo.Notef(err, "can not record note to store"))
}

var cmdAddTodo = &cobra.Command{
	Use:   "todo [command]",
	Short: "Add a new todo to the log.",
	Long:  `Add a new todo to the log which can have a timestamp, a toggle state (if its active or not) and an free form value for text.`,
	Run:   runCmdAddTodo,
}

func runCmdAddTodo(cmd *cobra.Command, args []string) {
	cmd.Help()
}

var cmdAddTodoActive = &cobra.Command{
	Use:   "active",
	Short: "Add a new todo to the log and mark it as active.",
	Long:  `Add a new todo to the log which can have a timestamp, is marked as active and an free form value for text.`,
	Run:   runCmdAddTodoActive,
}

func runCmdAddTodoActive(cmd *cobra.Command, args []string) {
	project, todo, err := helper.ArgsToTodo(args, flagAddTimeStamp, flagAddTimeStampRaw)
	helper.ErrExit(errgo.Notef(err, "can not convert args to todo"))

	todo.Active = true

	err = helper.RecordEntry(flagDataDir, project, todo)
	helper.ErrExit(errgo.Notef(err, "can not record todo to store"))
}

var cmdAddTodoInActive = &cobra.Command{
	Use:   "inactive",
	Short: "Add a new todo to the log and mark it as inactive.",
	Long:  `Add a new todo to the log which can have a timestamp, is marked as inactive and an free form value for text.`,
	Run:   runCmdAddTodoInActive,
}

func runCmdAddTodoInActive(cmd *cobra.Command, args []string) {
	project, todo, err := helper.ArgsToTodo(args, flagAddTimeStamp, flagAddTimeStampRaw)
	helper.ErrExit(errgo.Notef(err, "can not convert args to todo"))

	todo.Active = false

	err = helper.RecordEntry(flagDataDir, project, todo)
	helper.ErrExit(errgo.Notef(err, "can not record todo to store"))
}
<|endoftext|>


<commit_before>package tcp

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestTCP(t *testing.T) {
	timeToRoomDeletion = 100 * time.Millisecond
	go Run("debug", "8281", "8282")
	time.Sleep(100 * time.Millisecond)
	c1, banner, _, err := ConnectToTCPServer("localhost:8281", "testRoom", 1*time.Minute)
	assert.Equal(t, banner, "8282")
	assert.Nil(t, err)
	c2, _, _, err := ConnectToTCPServer("localhost:8281", "testRoom")
	assert.Nil(t, err)
	_, _, _, err = ConnectToTCPServer("localhost:8281", "testRoom")
	assert.NotNil(t, err)
	_, _, _, err = ConnectToTCPServer("localhost:8281", "testRoom", 1*time.Nanosecond)
	assert.NotNil(t, err)

	// try sending data
	assert.Nil(t, c1.Send([]byte("hello, c2")))
	data, err := c2.Receive()
	assert.Nil(t, err)
	assert.Equal(t, []byte("hello, c2"), data)

	assert.Nil(t, c2.Send([]byte("hello, c1")))
	data, err = c1.Receive()
	assert.Nil(t, err)
	assert.Equal(t, []byte("hello, c1"), data)

	c1.Close()
	time.Sleep(300 * time.Millisecond)
}
<commit_msg>fix tests<commit_after>package tcp

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestTCP(t *testing.T) {
	timeToRoomDeletion = 100 * time.Millisecond
	go Run("debug", "8281", "pass123", "8282")
	time.Sleep(100 * time.Millisecond)
	c1, banner, _, err := ConnectToTCPServer("localhost:8281", "pass123", "testRoom", 1*time.Minute)
	assert.Equal(t, banner, "8282")
	assert.Nil(t, err)
	c2, _, _, err := ConnectToTCPServer("localhost:8281", "pass123", "testRoom")
	assert.Nil(t, err)
	_, _, _, err = ConnectToTCPServer("localhost:8281", "pass123", "testRoom")
	assert.NotNil(t, err)
	_, _, _, err = ConnectToTCPServer("localhost:8281", "pass123", "testRoom", 1*time.Nanosecond)
	assert.NotNil(t, err)

	// try sending data
	assert.Nil(t, c1.Send([]byte("hello, c2")))
	data, err := c2.Receive()
	assert.Nil(t, err)
	assert.Equal(t, []byte("hello, c2"), data)

	assert.Nil(t, c2.Send([]byte("hello, c1")))
	data, err = c1.Receive()
	assert.Nil(t, err)
	assert.Equal(t, []byte("hello, c1"), data)

	c1.Close()
	time.Sleep(300 * time.Millisecond)
}
<|endoftext|>


<commit_before>package store

import (
    "log"

    "github.com/garyburd/redigo/redis"
    "github.com/spf13/viper"
)

func Save(url, content string) {
    viper.SetDefault("redis.server", "localhost")
    viper.SetDefault("redis.port", "6379")
    viper.SetDefault("redis.protocol", "tcp")

    err := viper.ReadInConfig()
    server := viper.GetString("redis.server")
    port := viper.GetString("redis.port")
    protocol := viper.GetString("redis.protocol")

    log.Println("redis server: " + protocol + "://" + server + ":" + port)

    c, err := redis.Dial(protocol, server + ":" + port)
    if err != nil {
        panic(err)
    }
    defer c.Close()
    c.Do("SET", url, content)
}
<commit_msg>md5 url and store in redis<commit_after>package store

import (
	"crypto/md5"
	"encoding/hex"
	"log"

	"github.com/garyburd/redigo/redis"
	"github.com/spf13/viper"
)

func Save(url, content string) {
	connection, _ := getConnection()
	defer connection.Close()
	connection.Do("SET", url, content)
	connection.Do("SET", getMD5Str(url), url)
}

func getConnection() (connection redis.Conn, err error) {
	viper.SetDefault("redis.server", "localhost")
	viper.SetDefault("redis.port", "6379")
	viper.SetDefault("redis.protocol", "tcp")

	server := viper.GetString("redis.server")
	port := viper.GetString("redis.port")
	protocol := viper.GetString("redis.protocol")

	connection, err = redis.Dial(protocol, server+":"+port)
	if err != nil {
		panic(err)
	}

	log.Println("redis server: " + protocol + "://" + server + ":" + port)

	return
}

func getMD5Str(s string) string {
	md5Ctx := md5.New()
	md5Ctx.Write([]byte(s))
	cipherStr := md5Ctx.Sum(nil)
	return hex.EncodeToString(cipherStr)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package spotinstmodel

import (
	"fmt"
	"strings"

	"github.com/golang/glog"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/model"
	"k8s.io/kops/pkg/model/awsmodel"
	"k8s.io/kops/pkg/model/defaults"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
	"k8s.io/kops/upup/pkg/fi/cloudup/spotinsttasks"
)

const (
	// InstanceGroupLabelOrientation is the metadata label used on the
	// instance group to specify which orientation should be used.
	InstanceGroupLabelOrientation = "spotinst.io/orientation"

	// InstanceGroupLabelUtilizeReservedInstances is the metadata label used
	// on the instance group to specify whether reserved instances should be
	// utilized.
	InstanceGroupLabelUtilizeReservedInstances = "spotinst.io/utilize-reserved-instances"

	// InstanceGroupLabelFallbackToOnDemand is the metadata label used on the
	// instance group to specify whether fallback to on-demand instances should
	// be enabled.
	InstanceGroupLabelFallbackToOnDemand = "spotinst.io/fallback-to-ondemand"

	// InstanceGroupLabelAutoScalerDisabled is the metadata label used on the
	// instance group to specify whether the auto-scaler should be enabled.
	InstanceGroupLabelAutoScalerDisabled = "spotinst.io/autoscaler-disabled"

	// InstanceGroupLabelAutoScalerNodeLabels is the metadata label used on the
	// instance group to specify whether default node labels should be set for
	// the auto-scaler.
	InstanceGroupLabelAutoScalerNodeLabels = "spotinst.io/autoscaler-node-labels"
)

// ElastigroupModelBuilder configures Elastigroup objects
type ElastigroupModelBuilder struct {
	*awsmodel.AWSModelContext

	BootstrapScript   *model.BootstrapScript
	Lifecycle         *fi.Lifecycle
	SecurityLifecycle *fi.Lifecycle
}

var _ fi.ModelBuilder = &ElastigroupModelBuilder{}

func (b *ElastigroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
	for _, ig := range b.InstanceGroups {
		glog.V(2).Infof("Building instance group %q", b.AutoscalingGroupName(ig))

		group := &spotinsttasks.Elastigroup{
			Lifecycle:            b.Lifecycle,
			Name:                 fi.String(b.AutoscalingGroupName(ig)),
			ImageID:              fi.String(ig.Spec.Image),
			Monitoring:           fi.Bool(false),
			OnDemandInstanceType: fi.String(strings.Split(ig.Spec.MachineType, ",")[0]),
			SpotInstanceTypes:    strings.Split(ig.Spec.MachineType, ","),
			SecurityGroups: []*awstasks.SecurityGroup{
				b.LinkToSecurityGroup(ig.Spec.Role),
			},
		}

		// Cloud config.
		if cfg := b.Cluster.Spec.CloudConfig; cfg != nil {
			// Product.
			if cfg.SpotinstProduct != nil {
				group.Product = cfg.SpotinstProduct
			}

			// Orientation.
			if cfg.SpotinstOrientation != nil {
				group.Orientation = cfg.SpotinstOrientation
			}
		}

		// Strategy.
		for k, v := range ig.ObjectMeta.Labels {
			switch k {
			case InstanceGroupLabelOrientation:
				group.Orientation = fi.String(v)
				break

			case InstanceGroupLabelUtilizeReservedInstances:
				if v == "true" {
					group.UtilizeReservedInstances = fi.Bool(true)
				} else if v == "false" {
					group.UtilizeReservedInstances = fi.Bool(false)
				}
				break

			case InstanceGroupLabelFallbackToOnDemand:
				if v == "true" {
					group.FallbackToOnDemand = fi.Bool(true)
				} else if v == "false" {
					group.FallbackToOnDemand = fi.Bool(false)
				}
				break
			}
		}

		// Instance profile.
		iprof, err := b.LinkToIAMInstanceProfile(ig)
		if err != nil {
			return err
		}
		group.IAMInstanceProfile = iprof

		// Root volume.
		volumeSize := fi.Int32Value(ig.Spec.RootVolumeSize)
		if volumeSize == 0 {
			var err error
			volumeSize, err = defaults.DefaultInstanceGroupVolumeSize(ig.Spec.Role)
			if err != nil {
				return err
			}
		}

		volumeType := fi.StringValue(ig.Spec.RootVolumeType)
		if volumeType == "" {
			volumeType = awsmodel.DefaultVolumeType
		}

		group.RootVolumeSize = fi.Int64(int64(volumeSize))
		group.RootVolumeType = fi.String(volumeType)
		group.RootVolumeOptimization = ig.Spec.RootVolumeOptimization

		// Tenancy.
		if ig.Spec.Tenancy != "" {
			group.Tenancy = fi.String(ig.Spec.Tenancy)
		}

		// Risk.
		var risk float64
		switch ig.Spec.Role {
		case kops.InstanceGroupRoleMaster:
			risk = 0
		case kops.InstanceGroupRoleNode:
			risk = 100
		case kops.InstanceGroupRoleBastion:
			risk = 0
		default:
			return fmt.Errorf("spotinst: kops.Role not found %s", ig.Spec.Role)
		}
		group.Risk = &risk

		// Security groups.
		for _, id := range ig.Spec.AdditionalSecurityGroups {
			sgTask := &awstasks.SecurityGroup{
				Name:   fi.String(id),
				ID:     fi.String(id),
				Shared: fi.Bool(true),
			}
			if err := c.EnsureTask(sgTask); err != nil {
				return err
			}
			group.SecurityGroups = append(group.SecurityGroups, sgTask)
		}

		// SSH Key.
		sshKey, err := b.LinkToSSHKey()
		if err != nil {
			return err
		}
		group.SSHKey = sshKey

		// Load balancer.
		var lb *awstasks.LoadBalancer
		switch ig.Spec.Role {
		case kops.InstanceGroupRoleMaster:
			if b.UseLoadBalancerForAPI() {
				lb = b.LinkToELB("api")
			}
		case kops.InstanceGroupRoleBastion:
			lb = b.LinkToELB(model.BastionELBSecurityGroupPrefix)
		}
		if lb != nil {
			group.LoadBalancer = lb
		}

		// User data.
		userData, err := b.BootstrapScript.ResourceNodeUp(ig, b.Cluster)
		if err != nil {
			return err
		}
		group.UserData = userData

		// Public IP.
		subnetMap := make(map[string]*kops.ClusterSubnetSpec)
		for i := range b.Cluster.Spec.Subnets {
			subnet := &b.Cluster.Spec.Subnets[i]
			subnetMap[subnet.Name] = subnet
		}

		var subnetType kops.SubnetType
		for _, subnetName := range ig.Spec.Subnets {
			subnet := subnetMap[subnetName]
			if subnet == nil {
				return fmt.Errorf("spotinst: InstanceGroup %q uses subnet %q that does not exist", ig.ObjectMeta.Name, subnetName)
			}
			if subnetType != "" && subnetType != subnet.Type {
				return fmt.Errorf("spotinst: InstanceGroup %q cannot be in subnets of different Type", ig.ObjectMeta.Name)
			}
			subnetType = subnet.Type
		}

		associatePublicIP := true
		switch subnetType {
		case kops.SubnetTypePublic, kops.SubnetTypeUtility:
			associatePublicIP = true
			if ig.Spec.AssociatePublicIP != nil {
				associatePublicIP = *ig.Spec.AssociatePublicIP
			}
		case kops.SubnetTypePrivate:
			associatePublicIP = false
			if ig.Spec.AssociatePublicIP != nil {
				if *ig.Spec.AssociatePublicIP {
					glog.Warningf("Ignoring AssociatePublicIP=true for private InstanceGroup %q", ig.ObjectMeta.Name)
				}
			}
		default:
			return fmt.Errorf("spotinst: unknown subnet type %q", subnetType)
		}
		group.AssociatePublicIP = &associatePublicIP

		// Subnets.
		subnets, err := b.GatherSubnets(ig)
		if err != nil {
			return err
		}
		if len(subnets) == 0 {
			return fmt.Errorf("spotinst: could not determine any subnets for InstanceGroup %q; subnets was %s", ig.ObjectMeta.Name, ig.Spec.Subnets)
		}
		for _, subnet := range subnets {
			group.Subnets = append(group.Subnets, b.LinkToSubnet(subnet))
		}

		// Capacity.
		minSize := int32(1)
		if ig.Spec.MinSize != nil {
			minSize = fi.Int32Value(ig.Spec.MinSize)
		} else if ig.Spec.Role == kops.InstanceGroupRoleNode {
			minSize = 2
		}

		maxSize := int32(1)
		if ig.Spec.MaxSize != nil {
			maxSize = *ig.Spec.MaxSize
		} else if ig.Spec.Role == kops.InstanceGroupRoleNode {
			maxSize = 10
		}

		group.MinSize = fi.Int64(int64(minSize))
		group.MaxSize = fi.Int64(int64(maxSize))

		// Tags.
		tags, err := b.CloudTagsForInstanceGroup(ig)
		if err != nil {
			return fmt.Errorf("spotinst: error building cloud tags: %v", err)
		}
		tags[awsup.TagClusterName] = b.ClusterName()
		tags["Name"] = b.AutoscalingGroupName(ig)
		group.Tags = tags

		// Auto Scaler.
		if ig.Spec.Role != kops.InstanceGroupRoleBastion {
			group.ClusterIdentifier = fi.String(b.ClusterName())

			// Toggle auto scaler's features.
			var autoScalerDisabled bool
			var autoScalerNodeLabels bool
			{
				for k, v := range ig.ObjectMeta.Labels {
					switch k {
					case InstanceGroupLabelAutoScalerDisabled:
						if v == "true" {
							autoScalerDisabled = true
						} else if v == "false" {
							autoScalerDisabled = false
						}
						break

					case InstanceGroupLabelAutoScalerNodeLabels:
						if v == "true" {
							autoScalerNodeLabels = true
						} else if v == "false" {
							autoScalerNodeLabels = false
						}
						break
					}
				}
			}

			// Toggle the auto scaler.
			group.AutoScalerEnabled = fi.Bool(!autoScalerDisabled)

			// Set the node labels.
			if ig.Spec.Role == kops.InstanceGroupRoleNode {
				nodeLabels := make(map[string]string)
				for k, v := range ig.Spec.NodeLabels {
					if strings.HasPrefix(k, kops.NodeLabelInstanceGroup) && !autoScalerNodeLabels {
						continue
					}
					nodeLabels[k] = v
				}
				if len(nodeLabels) > 0 {
					group.AutoScalerNodeLabels = nodeLabels
				}
			}
		}

		c.AddTask(group)
	}

	return nil
}
<commit_msg>fix: max size defaults to 2<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package spotinstmodel

import (
	"fmt"
	"strings"

	"github.com/golang/glog"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/model"
	"k8s.io/kops/pkg/model/awsmodel"
	"k8s.io/kops/pkg/model/defaults"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
	"k8s.io/kops/upup/pkg/fi/cloudup/spotinsttasks"
)

const (
	// InstanceGroupLabelOrientation is the metadata label used on the
	// instance group to specify which orientation should be used.
	InstanceGroupLabelOrientation = "spotinst.io/orientation"

	// InstanceGroupLabelUtilizeReservedInstances is the metadata label used
	// on the instance group to specify whether reserved instances should be
	// utilized.
	InstanceGroupLabelUtilizeReservedInstances = "spotinst.io/utilize-reserved-instances"

	// InstanceGroupLabelFallbackToOnDemand is the metadata label used on the
	// instance group to specify whether fallback to on-demand instances should
	// be enabled.
	InstanceGroupLabelFallbackToOnDemand = "spotinst.io/fallback-to-ondemand"

	// InstanceGroupLabelAutoScalerDisabled is the metadata label used on the
	// instance group to specify whether the auto-scaler should be enabled.
	InstanceGroupLabelAutoScalerDisabled = "spotinst.io/autoscaler-disabled"

	// InstanceGroupLabelAutoScalerNodeLabels is the metadata label used on the
	// instance group to specify whether default node labels should be set for
	// the auto-scaler.
	InstanceGroupLabelAutoScalerNodeLabels = "spotinst.io/autoscaler-node-labels"
)

// ElastigroupModelBuilder configures Elastigroup objects
type ElastigroupModelBuilder struct {
	*awsmodel.AWSModelContext

	BootstrapScript   *model.BootstrapScript
	Lifecycle         *fi.Lifecycle
	SecurityLifecycle *fi.Lifecycle
}

var _ fi.ModelBuilder = &ElastigroupModelBuilder{}

func (b *ElastigroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
	for _, ig := range b.InstanceGroups {
		glog.V(2).Infof("Building instance group %q", b.AutoscalingGroupName(ig))

		group := &spotinsttasks.Elastigroup{
			Lifecycle:            b.Lifecycle,
			Name:                 fi.String(b.AutoscalingGroupName(ig)),
			ImageID:              fi.String(ig.Spec.Image),
			Monitoring:           fi.Bool(false),
			OnDemandInstanceType: fi.String(strings.Split(ig.Spec.MachineType, ",")[0]),
			SpotInstanceTypes:    strings.Split(ig.Spec.MachineType, ","),
			SecurityGroups: []*awstasks.SecurityGroup{
				b.LinkToSecurityGroup(ig.Spec.Role),
			},
		}

		// Cloud config.
		if cfg := b.Cluster.Spec.CloudConfig; cfg != nil {
			// Product.
			if cfg.SpotinstProduct != nil {
				group.Product = cfg.SpotinstProduct
			}

			// Orientation.
			if cfg.SpotinstOrientation != nil {
				group.Orientation = cfg.SpotinstOrientation
			}
		}

		// Strategy.
		for k, v := range ig.ObjectMeta.Labels {
			switch k {
			case InstanceGroupLabelOrientation:
				group.Orientation = fi.String(v)
				break

			case InstanceGroupLabelUtilizeReservedInstances:
				if v == "true" {
					group.UtilizeReservedInstances = fi.Bool(true)
				} else if v == "false" {
					group.UtilizeReservedInstances = fi.Bool(false)
				}
				break

			case InstanceGroupLabelFallbackToOnDemand:
				if v == "true" {
					group.FallbackToOnDemand = fi.Bool(true)
				} else if v == "false" {
					group.FallbackToOnDemand = fi.Bool(false)
				}
				break
			}
		}

		// Instance profile.
		iprof, err := b.LinkToIAMInstanceProfile(ig)
		if err != nil {
			return err
		}
		group.IAMInstanceProfile = iprof

		// Root volume.
		volumeSize := fi.Int32Value(ig.Spec.RootVolumeSize)
		if volumeSize == 0 {
			var err error
			volumeSize, err = defaults.DefaultInstanceGroupVolumeSize(ig.Spec.Role)
			if err != nil {
				return err
			}
		}

		volumeType := fi.StringValue(ig.Spec.RootVolumeType)
		if volumeType == "" {
			volumeType = awsmodel.DefaultVolumeType
		}

		group.RootVolumeSize = fi.Int64(int64(volumeSize))
		group.RootVolumeType = fi.String(volumeType)
		group.RootVolumeOptimization = ig.Spec.RootVolumeOptimization

		// Tenancy.
		if ig.Spec.Tenancy != "" {
			group.Tenancy = fi.String(ig.Spec.Tenancy)
		}

		// Risk.
		var risk float64
		switch ig.Spec.Role {
		case kops.InstanceGroupRoleMaster:
			risk = 0
		case kops.InstanceGroupRoleNode:
			risk = 100
		case kops.InstanceGroupRoleBastion:
			risk = 0
		default:
			return fmt.Errorf("spotinst: kops.Role not found %s", ig.Spec.Role)
		}
		group.Risk = &risk

		// Security groups.
		for _, id := range ig.Spec.AdditionalSecurityGroups {
			sgTask := &awstasks.SecurityGroup{
				Name:   fi.String(id),
				ID:     fi.String(id),
				Shared: fi.Bool(true),
			}
			if err := c.EnsureTask(sgTask); err != nil {
				return err
			}
			group.SecurityGroups = append(group.SecurityGroups, sgTask)
		}

		// SSH Key.
		sshKey, err := b.LinkToSSHKey()
		if err != nil {
			return err
		}
		group.SSHKey = sshKey

		// Load balancer.
		var lb *awstasks.LoadBalancer
		switch ig.Spec.Role {
		case kops.InstanceGroupRoleMaster:
			if b.UseLoadBalancerForAPI() {
				lb = b.LinkToELB("api")
			}
		case kops.InstanceGroupRoleBastion:
			lb = b.LinkToELB(model.BastionELBSecurityGroupPrefix)
		}
		if lb != nil {
			group.LoadBalancer = lb
		}

		// User data.
		userData, err := b.BootstrapScript.ResourceNodeUp(ig, b.Cluster)
		if err != nil {
			return err
		}
		group.UserData = userData

		// Public IP.
		subnetMap := make(map[string]*kops.ClusterSubnetSpec)
		for i := range b.Cluster.Spec.Subnets {
			subnet := &b.Cluster.Spec.Subnets[i]
			subnetMap[subnet.Name] = subnet
		}

		var subnetType kops.SubnetType
		for _, subnetName := range ig.Spec.Subnets {
			subnet := subnetMap[subnetName]
			if subnet == nil {
				return fmt.Errorf("spotinst: InstanceGroup %q uses subnet %q that does not exist", ig.ObjectMeta.Name, subnetName)
			}
			if subnetType != "" && subnetType != subnet.Type {
				return fmt.Errorf("spotinst: InstanceGroup %q cannot be in subnets of different Type", ig.ObjectMeta.Name)
			}
			subnetType = subnet.Type
		}

		associatePublicIP := true
		switch subnetType {
		case kops.SubnetTypePublic, kops.SubnetTypeUtility:
			associatePublicIP = true
			if ig.Spec.AssociatePublicIP != nil {
				associatePublicIP = *ig.Spec.AssociatePublicIP
			}
		case kops.SubnetTypePrivate:
			associatePublicIP = false
			if ig.Spec.AssociatePublicIP != nil {
				if *ig.Spec.AssociatePublicIP {
					glog.Warningf("Ignoring AssociatePublicIP=true for private InstanceGroup %q", ig.ObjectMeta.Name)
				}
			}
		default:
			return fmt.Errorf("spotinst: unknown subnet type %q", subnetType)
		}
		group.AssociatePublicIP = &associatePublicIP

		// Subnets.
		subnets, err := b.GatherSubnets(ig)
		if err != nil {
			return err
		}
		if len(subnets) == 0 {
			return fmt.Errorf("spotinst: could not determine any subnets for InstanceGroup %q; subnets was %s", ig.ObjectMeta.Name, ig.Spec.Subnets)
		}
		for _, subnet := range subnets {
			group.Subnets = append(group.Subnets, b.LinkToSubnet(subnet))
		}

		// Capacity.
		minSize := int32(1)
		if ig.Spec.MinSize != nil {
			minSize = fi.Int32Value(ig.Spec.MinSize)
		} else if ig.Spec.Role == kops.InstanceGroupRoleNode {
			minSize = 2
		}

		maxSize := int32(1)
		if ig.Spec.MaxSize != nil {
			maxSize = *ig.Spec.MaxSize
		} else if ig.Spec.Role == kops.InstanceGroupRoleNode {
			maxSize = 2
		}

		group.MinSize = fi.Int64(int64(minSize))
		group.MaxSize = fi.Int64(int64(maxSize))

		// Tags.
		tags, err := b.CloudTagsForInstanceGroup(ig)
		if err != nil {
			return fmt.Errorf("spotinst: error building cloud tags: %v", err)
		}
		tags[awsup.TagClusterName] = b.ClusterName()
		tags["Name"] = b.AutoscalingGroupName(ig)
		group.Tags = tags

		// Auto Scaler.
		if ig.Spec.Role != kops.InstanceGroupRoleBastion {
			group.ClusterIdentifier = fi.String(b.ClusterName())

			// Toggle auto scaler's features.
			var autoScalerDisabled bool
			var autoScalerNodeLabels bool
			{
				for k, v := range ig.ObjectMeta.Labels {
					switch k {
					case InstanceGroupLabelAutoScalerDisabled:
						if v == "true" {
							autoScalerDisabled = true
						} else if v == "false" {
							autoScalerDisabled = false
						}
						break

					case InstanceGroupLabelAutoScalerNodeLabels:
						if v == "true" {
							autoScalerNodeLabels = true
						} else if v == "false" {
							autoScalerNodeLabels = false
						}
						break
					}
				}
			}

			// Toggle the auto scaler.
			group.AutoScalerEnabled = fi.Bool(!autoScalerDisabled)

			// Set the node labels.
			if ig.Spec.Role == kops.InstanceGroupRoleNode {
				nodeLabels := make(map[string]string)
				for k, v := range ig.Spec.NodeLabels {
					if strings.HasPrefix(k, kops.NodeLabelInstanceGroup) && !autoScalerNodeLabels {
						continue
					}
					nodeLabels[k] = v
				}
				if len(nodeLabels) > 0 {
					group.AutoScalerNodeLabels = nodeLabels
				}
			}
		}

		c.AddTask(group)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A tool to measure the upload throughput of GCS.
package main

import (
	"crypto/rand"
	"flag"
	"fmt"
	"io"
	"log"

	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

var fBucket = flag.String("bucket", "", "Name of bucket.")
var fKeyFile = flag.String("key_file", "", "Path to JSON key file.")
var fSize = flag.Int64("size", 1<<26, "Size of content to write.")

func createBucket() (bucket gcs.Bucket, err error)

func run() (err error) {
	bucket, err := createBucket()
	if err != nil {
		err = fmt.Errorf("createBucket: %v", err)
		return
	}

	// Create a temporary file to hold random contents.
	f, err := fsutil.AnonymousFile("")
	if err != nil {
		err = fmt.Errorf("AnonymousFile: %v", err)
		return
	}

	// Copy a bunch of random data into the file.
	_, err = io.Copy(f, io.LimitReader(rand.Reader, *fSize))
	if err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	// Seek back to the start for consumption below.
	_, err = f.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	// Create an object using the contents of the file.
	req := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: "foo",
		},
		Contents: f,
	}

	_, err = bucket.CreateObject(context.Background(), req)
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	return
}

func main() {
	err := run()
	if err != nil {
		log.Fatalln(err)
	}
}
<commit_msg>createBucket<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A tool to measure the upload throughput of GCS.
package main

import (
	"crypto/rand"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"

	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/oauthutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

var fBucket = flag.String("bucket", "", "Name of bucket.")
var fKeyFile = flag.String("key_file", "", "Path to JSON key file.")
var fSize = flag.Int64("size", 1<<26, "Size of content to write.")

func createBucket() (bucket gcs.Bucket, err error) {
	// Create an authenticated HTTP client.
	if *fKeyFile == "" {
		err = errors.New("You must set --key_file.")
		return
	}

	httpClient, err := oauthutil.NewJWTHttpClient(
		*fKeyFile,
		[]string{storage.ScopeFullControl})

	if err != nil {
		err = fmt.Errorf("NewJWTHttpClient: %v", err)
		return
	}

	// Use that to create a connection.
	conn, err := gcs.NewConn("", httpClient)
	if err != nil {
		err = fmt.Errorf("NewConn: %v", err)
		return
	}

	// Extract the bucket.
	if *fBucket == "" {
		err = errors.New("You must set --bucket.")
		return
	}

	bucket = conn.GetBucket(*fBucket)

	return
}

func run() (err error) {
	bucket, err := createBucket()
	if err != nil {
		err = fmt.Errorf("createBucket: %v", err)
		return
	}

	// Create a temporary file to hold random contents.
	f, err := fsutil.AnonymousFile("")
	if err != nil {
		err = fmt.Errorf("AnonymousFile: %v", err)
		return
	}

	// Copy a bunch of random data into the file.
	_, err = io.Copy(f, io.LimitReader(rand.Reader, *fSize))
	if err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	// Seek back to the start for consumption below.
	_, err = f.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	// Create an object using the contents of the file.
	req := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: "foo",
		},
		Contents: f,
	}

	_, err = bucket.CreateObject(context.Background(), req)
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	return
}

func main() {
	err := run()
	if err != nil {
		log.Fatalln(err)
	}
}
<|endoftext|>


<commit_before>package WatchDog

import (
	"math"
	"sync"
	"sync/atomic"
	"time"

	"github.com/6xiao/go/Common"
)

type WatchDog struct {
	sync.Mutex

	wait time.Duration
	hung func()
	meat *int64
}

func NewDog(duration time.Duration, hung func()) *WatchDog {
	d := WatchDog{sync.Mutex{}, duration, hung, new(int64)}
	d.Feed(math.MaxInt32)
	go d.eat()
	return &d
}

func (this *WatchDog) eat() {
	defer Common.CheckPanic()

	for this.hung != nil {
		time.Sleep(this.wait)

		m := atomic.LoadInt64(this.meat)
		if m < 0 {
			return
		}

		if m == 0 {
			this.hung()
		} else {
			atomic.StoreInt64(this.meat, m/2)
		}
	}
}

func (this *WatchDog) Feed(meat int64) bool {
	defer Common.CheckPanic()

	if meat > math.MaxInt32 {
		return false
	}

	return atomic.AddInt64(this.meat, meat) > 0
}

func (this *WatchDog) Kill() {
	defer Common.CheckPanic()
	atomic.StoreInt64(this.meat, math.MinInt64)
}
<commit_msg>remove lock<commit_after>package WatchDog

import (
	"math"
	"sync/atomic"
	"time"

	"github.com/6xiao/go/Common"
)

type WatchDog struct {
	wait time.Duration
	hung func()
	meat int64
}

func NewDog(duration time.Duration, hung func()) *WatchDog {
	d := new(WatchDog)
	d.wait = duration
	d.hung = hung
	d.meat = math.MaxInt32

	go d.eat()
	return d
}

func (this *WatchDog) eat() {
	defer Common.CheckPanic()

	for this.hung != nil {
		time.Sleep(this.wait)

		m := atomic.LoadInt64(&this.meat)
		if m < 0 {
			return
		}

		if m == 0 {
			this.hung()
		} else {
			atomic.StoreInt64(&this.meat, m/2)
		}
	}
}

func (this *WatchDog) Feed(meat int64) bool {
	defer Common.CheckPanic()

	if meat > math.MaxInt32 {
		return false
	}
	return atomic.AddInt64(&this.meat, meat) > 0
}

func (this *WatchDog) Kill() {
	defer Common.CheckPanic()
	atomic.StoreInt64(&this.meat, math.MinInt32)
}
<|endoftext|>


<commit_before>package kloud

import (
	"errors"
	_ "expvar"
	"fmt"
	"io/ioutil"
	"log"
	_ "net/http/pprof"
	"net/url"
	"strings"
	"time"

	"golang.org/x/net/context"

	"koding/artifact"
	"koding/db/mongodb/modelhelper"
	"koding/httputil"
	"koding/kites/common"
	"koding/kites/keygen"
	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/contexthelper/publickeys"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/dnsstorage"
	"koding/kites/kloud/keycreator"
	"koding/kites/kloud/pkg/dnsclient"
	"koding/kites/kloud/queue"
	"koding/kites/kloud/stack"
	"koding/kites/kloud/stackplan"
	"koding/kites/kloud/stackplan/stackcred"
	"koding/kites/kloud/terraformer"
	"koding/kites/kloud/userdata"

	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/koding/kite"
	kiteconfig "github.com/koding/kite/config"
	"github.com/koding/logging"
)

//go:generate go run genimport.go -o import.go
//go:generate go fmt import.go

// Name holds kite name
var Name = "kloud"

// Kloud represents a configured kloud kite.
type Kloud struct {
	Kite   *kite.Kite
	Stack  *stack.Kloud
	Keygen *keygen.Server

	// Queue is responsible for executing checks and actions on user
	// machines. Given the interval they are queued and processed,
	// thus the naming. For example queue is responsible for
	// shutting down a non-always-on vm when it idles for more
	// than 1h.
	Queue *queue.Queue
}

// Config defines the configuration that Kloud needs to operate.
type Config struct {
	// ---  KLOUD SPECIFIC ---
	IP          string
	Port        int
	Region      string
	Environment string

	// Connect to Koding mongodb
	MongoURL string `required:"true"`

	// CredentialEndpoint is an API for managing stack credentials.
	CredentialEndpoint string

	// --- DEVELOPMENT CONFIG ---
	// Show version and exit if enabled
	Version bool

	// Enable debug log mode
	DebugMode bool

	// Enable production mode, operates on production channel
	ProdMode bool

	// Enable test mode, disabled some authentication checks
	TestMode bool

	// Defines the base domain for domain creation
	HostedZone string `required:"true"`

	// MaxResults limits the max items fetched per page for each
	// AWS Describe* API calls.
	MaxResults int `default:"500"`

	// --- KLIENT DEVELOPMENT ---
	// KontrolURL to connect and to de deployed with klient
	KontrolURL string `required:"true"`

	// KlientURL overwrites the Klient deb url returned by userdata.GetLatestDeb
	// method.
	KlientURL string

	// TunnelURL overwrites default tunnelserver url. Used by vagrant provider.
	TunnelURL string

	// Private key to create kite.key
	PrivateKey string `required:"true"`

	// Public key to create kite.key
	PublicKey string `required:"true"`

	// Private and public key to put a ssh key into the users VM's so we can
	// have access to it. Note that these are different then from the Kontrol
	// keys.
	UserPublicKey  string `required:"true"`
	UserPrivateKey string `required:"true"`

	// Keygen configuration.
	KeygenAccessKey string
	KeygenSecretKey string
	KeygenBucket    string
	KeygenRegion    string        `default:"us-east-1"`
	KeygenTokenTTL  time.Duration `default:"3h"`

	// --- KONTROL CONFIGURATION ---
	Public      bool   // Try to register with a public ip
	RegisterURL string // Explicitly register with this given url

	AWSAccessKeyId     string
	AWSSecretAccessKey string

	SLUsername string
	SLAPIKey   string

	JanitorSecretKey     string
	VmwatcherSecretKey   string
	KloudSecretKey       string
	TerraformerSecretKey string
}

// New gives new, registered kloud kite.
//
// If conf contains invalid or missing configuration, it return non-nil error.
func New(conf *Config) (*Kloud, error) {
	k := kite.New(stack.NAME, stack.VERSION)
	k.Config = kiteconfig.MustGet()
	k.Config.Port = conf.Port

	k.ClientFunc = httputil.ClientFunc(conf.DebugMode)

	if conf.DebugMode {
		k.SetLogLevel(kite.DEBUG)
	}

	if conf.Region != "" {
		k.Config.Region = conf.Region
	}

	if conf.Environment != "" {
		k.Config.Environment = conf.Environment
	}

	// TODO(rjeczalik): refactor modelhelper methods to not use global DB
	modelhelper.Initialize(conf.MongoURL)

	sess, err := newSession(conf, k)
	if err != nil {
		return nil, err
	}

	authUsers := map[string]string{
		"kloudctl":  conf.KloudSecretKey,
		"janitor":   conf.JanitorSecretKey,
		"vmwatcher": conf.VmwatcherSecretKey,
	}

	var credURL *url.URL

	if conf.CredentialEndpoint != "" {
		if u, err := url.Parse(conf.CredentialEndpoint); err == nil {
			credURL = u
		}
	}

	if credURL == nil {
		sess.Log.Warning(`disabling "Sneaker" for storing stack credential data`)
	}

	storeOpts := &stackcred.StoreOptions{
		MongoDB: sess.DB,
		Log:     sess.Log.New("stackcred"),
		CredURL: credURL,
		Client:  httputil.DefaultRestClient(conf.DebugMode),
	}

	stacker := &stackplan.Stacker{
		DB:             sess.DB,
		Log:            sess.Log,
		Kite:           sess.Kite,
		Userdata:       sess.Userdata,
		Debug:          conf.DebugMode,
		KloudSecretKey: conf.KloudSecretKey,
		CredStore:      stackcred.NewStore(storeOpts),
		TunnelURL:      conf.TunnelURL,
	}

	stats := common.MustInitMetrics(Name)

	kloud := &Kloud{
		Stack: stack.New(),
		Queue: &queue.Queue{
			Log:      sess.Log.New("queue"),
			Interval: 5 * time.Second,
			MongoDB:  sess.DB,
		},
	}

	kloud.Stack.ContextCreator = func(ctx context.Context) context.Context {
		return session.NewContext(ctx, sess)
	}

	kloud.Stack.Metrics = stats
	userPrivateKey, userPublicKey := userMachinesKeys(conf.UserPublicKey, conf.UserPrivateKey)

	// RSA key pair that we add to the newly created machine for
	// provisioning.
	kloud.Stack.PublicKeys = &publickeys.Keys{
		KeyName:    publickeys.DeployKeyName,
		PrivateKey: userPrivateKey,
		PublicKey:  userPublicKey,
	}
	kloud.Stack.DomainStorage = sess.DNSStorage
	kloud.Stack.Domainer = sess.DNSClient
	kloud.Stack.Locker = stacker
	kloud.Stack.Log = sess.Log
	kloud.Stack.SecretKey = conf.KloudSecretKey

	for _, p := range stackplan.All() {
		s := stacker.New(p)

		if err = kloud.Stack.AddProvider(p.Name, s); err != nil {
			return nil, err
		}

		kloud.Queue.Register(s)
	}

	go kloud.Queue.Run()

	if conf.KeygenAccessKey != "" && conf.KeygenSecretKey != "" {
		cfg := &keygen.Config{
			AccessKey:  conf.KeygenAccessKey,
			SecretKey:  conf.KeygenSecretKey,
			Region:     conf.KeygenRegion,
			Bucket:     conf.KeygenBucket,
			AuthExpire: conf.KeygenTokenTTL,
			AuthFunc:   kloud.Stack.ValidateUser,
			Kite:       k,
		}

		kloud.Keygen = keygen.NewServer(cfg)
	} else {
		k.Log.Warning(`disabling "keygen" methods due to missing S3/STS credentials`)
	}

	// Teams/stack handling methods
	k.HandleFunc("plan", kloud.Stack.Plan)
	k.HandleFunc("apply", kloud.Stack.Apply)
	k.HandleFunc("describeStack", kloud.Stack.Status)
	k.HandleFunc("authenticate", kloud.Stack.Authenticate)
	k.HandleFunc("bootstrap", kloud.Stack.Bootstrap)

	// Single machine handling
	k.HandleFunc("stop", kloud.Stack.Stop)
	k.HandleFunc("start", kloud.Stack.Start)
	k.HandleFunc("info", kloud.Stack.Info)
	k.HandleFunc("event", kloud.Stack.Event)

	// Klient proxy methods
	k.HandleFunc("admin.add", kloud.Stack.AdminAdd)
	k.HandleFunc("admin.remove", kloud.Stack.AdminRemove)

	k.HandleHTTPFunc("/healthCheck", artifact.HealthCheckHandler(Name))
	k.HandleHTTPFunc("/version", artifact.VersionHandler())

	for worker, key := range authUsers {
		worker, key := worker, key
		k.Authenticators[worker] = func(r *kite.Request) error {
			if r.Auth.Key != key {
				return errors.New("wrong secret key passed, you are not authenticated")
			}
			return nil
		}
	}

	if conf.DebugMode {
		// This should be actually debug level 2. It outputs every single Kite
		// message and enables the kite debugging system. So enable it only if
		// you need it.
		// k.SetLogLevel(kite.DEBUG)
		k.Log.Info("Debug mode enabled")
	}

	if conf.TestMode {
		k.Log.Info("Test mode enabled")
	}

	registerURL := k.RegisterURL(!conf.Public)
	if conf.RegisterURL != "" {
		u, err := url.Parse(conf.RegisterURL)
		if err != nil {
			return nil, fmt.Errorf("Couldn't parse register url: %s", err)
		}

		registerURL = u
	}

	if err := k.RegisterForever(registerURL); err != nil {
		return nil, err
	}

	return kloud, nil
}

func newSession(conf *Config, k *kite.Kite) (*session.Session, error) {
	c := credentials.NewStaticCredentials(conf.AWSAccessKeyId, conf.AWSSecretAccessKey, "")

	kontrolPrivateKey, kontrolPublicKey := kontrolKeys(conf)

	klientFolder := "development/latest"
	if conf.ProdMode {
		k.Log.Info("Prod mode enabled")
		klientFolder = "production/latest"
	}

	k.Log.Info("Klient distribution channel is: %s", klientFolder)

	// Credential belongs to the `koding-kloud` user in AWS IAM's
	sess := &session.Session{
		DB:   modelhelper.Mongo,
		Kite: k,
		Userdata: &userdata.Userdata{
			Keycreator: &keycreator.Key{
				KontrolURL:        getKontrolURL(conf.KontrolURL),
				KontrolPrivateKey: kontrolPrivateKey,
				KontrolPublicKey:  kontrolPublicKey,
			},
			KlientURL: conf.KlientURL,
			Bucket:    userdata.NewBucket("koding-klient", klientFolder, c),
		},
		Terraformer: &terraformer.Options{
			Endpoint:  "http://127.0.0.1:2300/kite",
			SecretKey: conf.TerraformerSecretKey,
			Kite:      k,
		},
		Log: logging.NewCustom("kloud", conf.DebugMode),
	}

	sess.DNSStorage = dnsstorage.NewMongodbStorage(sess.DB)

	if conf.AWSAccessKeyId != "" && conf.AWSSecretAccessKey != "" {

		dnsOpts := &dnsclient.Options{
			Creds:      c,
			HostedZone: conf.HostedZone,
			Log:        logging.NewCustom("kloud-dns", conf.DebugMode),
			Debug:      conf.DebugMode,
		}

		dns, err := dnsclient.NewRoute53Client(dnsOpts)
		if err != nil {
			return nil, err
		}

		sess.DNSClient = dns

		opts := &amazon.ClientOptions{
			Credentials: c,
			Regions:     amazon.ProductionRegions,
			Log:         logging.NewCustom("kloud-koding", conf.DebugMode),
			MaxResults:  int64(conf.MaxResults),
			Debug:       conf.DebugMode,
		}

		ec2clients, err := amazon.NewClients(opts)
		if err != nil {
			return nil, err
		}

		sess.AWSClients = ec2clients
	}

	return sess, nil
}

func userMachinesKeys(publicPath, privatePath string) (string, string) {
	pubKey, err := ioutil.ReadFile(publicPath)
	if err != nil {
		log.Fatalln(err)
	}
	publicKey := string(pubKey)

	privKey, err := ioutil.ReadFile(privatePath)
	if err != nil {
		log.Fatalln(err)
	}
	privateKey := string(privKey)

	return strings.TrimSpace(privateKey), strings.TrimSpace(publicKey)
}

func kontrolKeys(conf *Config) (string, string) {
	pubKey, err := ioutil.ReadFile(conf.PublicKey)
	if err != nil {
		log.Fatalln(err)
	}
	publicKey := string(pubKey)

	privKey, err := ioutil.ReadFile(conf.PrivateKey)
	if err != nil {
		log.Fatalln(err)
	}
	privateKey := string(privKey)

	return privateKey, publicKey
}

func getKontrolURL(ownURL string) string {
	// read kontrolURL from kite.key if it doesn't exist.
	kontrolURL := kiteconfig.MustGet().KontrolURL

	if ownURL != "" {
		u, err := url.Parse(ownURL)
		if err != nil {
			log.Fatalln(err)
		}

		kontrolURL = u.String()
	}

	return kontrolURL
}
<commit_msg>kloud: fix nil ptr deref<commit_after>package kloud

import (
	"errors"
	_ "expvar"
	"fmt"
	"io/ioutil"
	"log"
	_ "net/http/pprof"
	"net/url"
	"strings"
	"time"

	"golang.org/x/net/context"

	"koding/artifact"
	"koding/db/mongodb/modelhelper"
	"koding/httputil"
	"koding/kites/common"
	"koding/kites/keygen"
	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/contexthelper/publickeys"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/dnsstorage"
	"koding/kites/kloud/keycreator"
	"koding/kites/kloud/pkg/dnsclient"
	"koding/kites/kloud/queue"
	"koding/kites/kloud/stack"
	"koding/kites/kloud/stackplan"
	"koding/kites/kloud/stackplan/stackcred"
	"koding/kites/kloud/terraformer"
	"koding/kites/kloud/userdata"

	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/koding/kite"
	kiteconfig "github.com/koding/kite/config"
	"github.com/koding/logging"
)

//go:generate go run genimport.go -o import.go
//go:generate go fmt import.go

// Name holds kite name
var Name = "kloud"

// Kloud represents a configured kloud kite.
type Kloud struct {
	Kite   *kite.Kite
	Stack  *stack.Kloud
	Keygen *keygen.Server

	// Queue is responsible for executing checks and actions on user
	// machines. Given the interval they are queued and processed,
	// thus the naming. For example queue is responsible for
	// shutting down a non-always-on vm when it idles for more
	// than 1h.
	Queue *queue.Queue
}

// Config defines the configuration that Kloud needs to operate.
type Config struct {
	// ---  KLOUD SPECIFIC ---
	IP          string
	Port        int
	Region      string
	Environment string

	// Connect to Koding mongodb
	MongoURL string `required:"true"`

	// CredentialEndpoint is an API for managing stack credentials.
	CredentialEndpoint string

	// --- DEVELOPMENT CONFIG ---
	// Show version and exit if enabled
	Version bool

	// Enable debug log mode
	DebugMode bool

	// Enable production mode, operates on production channel
	ProdMode bool

	// Enable test mode, disabled some authentication checks
	TestMode bool

	// Defines the base domain for domain creation
	HostedZone string `required:"true"`

	// MaxResults limits the max items fetched per page for each
	// AWS Describe* API calls.
	MaxResults int `default:"500"`

	// --- KLIENT DEVELOPMENT ---
	// KontrolURL to connect and to de deployed with klient
	KontrolURL string `required:"true"`

	// KlientURL overwrites the Klient deb url returned by userdata.GetLatestDeb
	// method.
	KlientURL string

	// TunnelURL overwrites default tunnelserver url. Used by vagrant provider.
	TunnelURL string

	// Private key to create kite.key
	PrivateKey string `required:"true"`

	// Public key to create kite.key
	PublicKey string `required:"true"`

	// Private and public key to put a ssh key into the users VM's so we can
	// have access to it. Note that these are different then from the Kontrol
	// keys.
	UserPublicKey  string `required:"true"`
	UserPrivateKey string `required:"true"`

	// Keygen configuration.
	KeygenAccessKey string
	KeygenSecretKey string
	KeygenBucket    string
	KeygenRegion    string        `default:"us-east-1"`
	KeygenTokenTTL  time.Duration `default:"3h"`

	// --- KONTROL CONFIGURATION ---
	Public      bool   // Try to register with a public ip
	RegisterURL string // Explicitly register with this given url

	AWSAccessKeyId     string
	AWSSecretAccessKey string

	SLUsername string
	SLAPIKey   string

	JanitorSecretKey     string
	VmwatcherSecretKey   string
	KloudSecretKey       string
	TerraformerSecretKey string
}

// New gives new, registered kloud kite.
//
// If conf contains invalid or missing configuration, it return non-nil error.
func New(conf *Config) (*Kloud, error) {
	k := kite.New(stack.NAME, stack.VERSION)
	k.Config = kiteconfig.MustGet()
	k.Config.Port = conf.Port

	k.ClientFunc = httputil.ClientFunc(conf.DebugMode)

	if conf.DebugMode {
		k.SetLogLevel(kite.DEBUG)
	}

	if conf.Region != "" {
		k.Config.Region = conf.Region
	}

	if conf.Environment != "" {
		k.Config.Environment = conf.Environment
	}

	// TODO(rjeczalik): refactor modelhelper methods to not use global DB
	modelhelper.Initialize(conf.MongoURL)

	sess, err := newSession(conf, k)
	if err != nil {
		return nil, err
	}

	authUsers := map[string]string{
		"kloudctl":  conf.KloudSecretKey,
		"janitor":   conf.JanitorSecretKey,
		"vmwatcher": conf.VmwatcherSecretKey,
	}

	var credURL *url.URL

	if conf.CredentialEndpoint != "" {
		if u, err := url.Parse(conf.CredentialEndpoint); err == nil {
			credURL = u
		}
	}

	if credURL == nil {
		sess.Log.Warning(`disabling "Sneaker" for storing stack credential data`)
	}

	storeOpts := &stackcred.StoreOptions{
		MongoDB: sess.DB,
		Log:     sess.Log.New("stackcred"),
		CredURL: credURL,
		Client:  httputil.DefaultRestClient(conf.DebugMode),
	}

	stacker := &stackplan.Stacker{
		DB:             sess.DB,
		Log:            sess.Log,
		Kite:           sess.Kite,
		Userdata:       sess.Userdata,
		Debug:          conf.DebugMode,
		KloudSecretKey: conf.KloudSecretKey,
		CredStore:      stackcred.NewStore(storeOpts),
		TunnelURL:      conf.TunnelURL,
	}

	stats := common.MustInitMetrics(Name)

	kloud := &Kloud{
		Kite:  k,
		Stack: stack.New(),
		Queue: &queue.Queue{
			Interval: 5 * time.Second,
			Log:      sess.Log.New("queue"),
			Kite:     k,
			MongoDB:  sess.DB,
		},
	}

	kloud.Stack.ContextCreator = func(ctx context.Context) context.Context {
		return session.NewContext(ctx, sess)
	}

	kloud.Stack.Metrics = stats
	userPrivateKey, userPublicKey := userMachinesKeys(conf.UserPublicKey, conf.UserPrivateKey)

	// RSA key pair that we add to the newly created machine for
	// provisioning.
	kloud.Stack.PublicKeys = &publickeys.Keys{
		KeyName:    publickeys.DeployKeyName,
		PrivateKey: userPrivateKey,
		PublicKey:  userPublicKey,
	}
	kloud.Stack.DomainStorage = sess.DNSStorage
	kloud.Stack.Domainer = sess.DNSClient
	kloud.Stack.Locker = stacker
	kloud.Stack.Log = sess.Log
	kloud.Stack.SecretKey = conf.KloudSecretKey

	for _, p := range stackplan.All() {
		s := stacker.New(p)

		if err = kloud.Stack.AddProvider(p.Name, s); err != nil {
			return nil, err
		}

		kloud.Queue.Register(s)
	}

	go kloud.Queue.Run()

	if conf.KeygenAccessKey != "" && conf.KeygenSecretKey != "" {
		cfg := &keygen.Config{
			AccessKey:  conf.KeygenAccessKey,
			SecretKey:  conf.KeygenSecretKey,
			Region:     conf.KeygenRegion,
			Bucket:     conf.KeygenBucket,
			AuthExpire: conf.KeygenTokenTTL,
			AuthFunc:   kloud.Stack.ValidateUser,
			Kite:       k,
		}

		kloud.Keygen = keygen.NewServer(cfg)
	} else {
		k.Log.Warning(`disabling "keygen" methods due to missing S3/STS credentials`)
	}

	// Teams/stack handling methods
	k.HandleFunc("plan", kloud.Stack.Plan)
	k.HandleFunc("apply", kloud.Stack.Apply)
	k.HandleFunc("describeStack", kloud.Stack.Status)
	k.HandleFunc("authenticate", kloud.Stack.Authenticate)
	k.HandleFunc("bootstrap", kloud.Stack.Bootstrap)

	// Single machine handling
	k.HandleFunc("stop", kloud.Stack.Stop)
	k.HandleFunc("start", kloud.Stack.Start)
	k.HandleFunc("info", kloud.Stack.Info)
	k.HandleFunc("event", kloud.Stack.Event)

	// Klient proxy methods
	k.HandleFunc("admin.add", kloud.Stack.AdminAdd)
	k.HandleFunc("admin.remove", kloud.Stack.AdminRemove)

	k.HandleHTTPFunc("/healthCheck", artifact.HealthCheckHandler(Name))
	k.HandleHTTPFunc("/version", artifact.VersionHandler())

	for worker, key := range authUsers {
		worker, key := worker, key
		k.Authenticators[worker] = func(r *kite.Request) error {
			if r.Auth.Key != key {
				return errors.New("wrong secret key passed, you are not authenticated")
			}
			return nil
		}
	}

	if conf.DebugMode {
		// This should be actually debug level 2. It outputs every single Kite
		// message and enables the kite debugging system. So enable it only if
		// you need it.
		// k.SetLogLevel(kite.DEBUG)
		k.Log.Info("Debug mode enabled")
	}

	if conf.TestMode {
		k.Log.Info("Test mode enabled")
	}

	registerURL := k.RegisterURL(!conf.Public)
	if conf.RegisterURL != "" {
		u, err := url.Parse(conf.RegisterURL)
		if err != nil {
			return nil, fmt.Errorf("Couldn't parse register url: %s", err)
		}

		registerURL = u
	}

	if err := k.RegisterForever(registerURL); err != nil {
		return nil, err
	}

	return kloud, nil
}

func newSession(conf *Config, k *kite.Kite) (*session.Session, error) {
	c := credentials.NewStaticCredentials(conf.AWSAccessKeyId, conf.AWSSecretAccessKey, "")

	kontrolPrivateKey, kontrolPublicKey := kontrolKeys(conf)

	klientFolder := "development/latest"
	if conf.ProdMode {
		k.Log.Info("Prod mode enabled")
		klientFolder = "production/latest"
	}

	k.Log.Info("Klient distribution channel is: %s", klientFolder)

	// Credential belongs to the `koding-kloud` user in AWS IAM's
	sess := &session.Session{
		DB:   modelhelper.Mongo,
		Kite: k,
		Userdata: &userdata.Userdata{
			Keycreator: &keycreator.Key{
				KontrolURL:        getKontrolURL(conf.KontrolURL),
				KontrolPrivateKey: kontrolPrivateKey,
				KontrolPublicKey:  kontrolPublicKey,
			},
			KlientURL: conf.KlientURL,
			Bucket:    userdata.NewBucket("koding-klient", klientFolder, c),
		},
		Terraformer: &terraformer.Options{
			Endpoint:  "http://127.0.0.1:2300/kite",
			SecretKey: conf.TerraformerSecretKey,
			Kite:      k,
		},
		Log: logging.NewCustom("kloud", conf.DebugMode),
	}

	sess.DNSStorage = dnsstorage.NewMongodbStorage(sess.DB)

	if conf.AWSAccessKeyId != "" && conf.AWSSecretAccessKey != "" {

		dnsOpts := &dnsclient.Options{
			Creds:      c,
			HostedZone: conf.HostedZone,
			Log:        logging.NewCustom("kloud-dns", conf.DebugMode),
			Debug:      conf.DebugMode,
		}

		dns, err := dnsclient.NewRoute53Client(dnsOpts)
		if err != nil {
			return nil, err
		}

		sess.DNSClient = dns

		opts := &amazon.ClientOptions{
			Credentials: c,
			Regions:     amazon.ProductionRegions,
			Log:         logging.NewCustom("kloud-koding", conf.DebugMode),
			MaxResults:  int64(conf.MaxResults),
			Debug:       conf.DebugMode,
		}

		ec2clients, err := amazon.NewClients(opts)
		if err != nil {
			return nil, err
		}

		sess.AWSClients = ec2clients
	}

	return sess, nil
}

func userMachinesKeys(publicPath, privatePath string) (string, string) {
	pubKey, err := ioutil.ReadFile(publicPath)
	if err != nil {
		log.Fatalln(err)
	}
	publicKey := string(pubKey)

	privKey, err := ioutil.ReadFile(privatePath)
	if err != nil {
		log.Fatalln(err)
	}
	privateKey := string(privKey)

	return strings.TrimSpace(privateKey), strings.TrimSpace(publicKey)
}

func kontrolKeys(conf *Config) (string, string) {
	pubKey, err := ioutil.ReadFile(conf.PublicKey)
	if err != nil {
		log.Fatalln(err)
	}
	publicKey := string(pubKey)

	privKey, err := ioutil.ReadFile(conf.PrivateKey)
	if err != nil {
		log.Fatalln(err)
	}
	privateKey := string(privKey)

	return privateKey, publicKey
}

func getKontrolURL(ownURL string) string {
	// read kontrolURL from kite.key if it doesn't exist.
	kontrolURL := kiteconfig.MustGet().KontrolURL

	if ownURL != "" {
		u, err := url.Parse(ownURL)
		if err != nil {
			log.Fatalln(err)
		}

		kontrolURL = u.String()
	}

	return kontrolURL
}
<|endoftext|>


<commit_before>package koding

import (
	"fmt"
	"koding/kites/kloud/klient"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"github.com/koding/kloud"
	"github.com/koding/kloud/machinestate"
	"github.com/koding/kloud/protocol"
)

func (p *Provider) Info(opts *protocol.Machine) (result *protocol.InfoArtifact, err error) {
	a, err := p.NewClient(opts)
	if err != nil {
		return nil, err
	}

	// otherwise ask AWS to get an machine state
	infoResp, err := a.Info()
	if err != nil {
		return nil, err
	}

	p.Log.Info("[%s] info initials: current db state is '%s'. amazon ec2 state is '%s'",
		opts.MachineId, dbState, awsState)

	dbState := opts.State
	awsState := infoResp.State

	// result state is the final state that is send back to the request
	resultState := dbState

	// we don't check if the state is something else. Klient is only available
	// when the machine is running
	klientChecked := false
	if dbState.In(machinestate.Running, machinestate.Stopped) && awsState == machinestate.Running {
		klientChecked = true
		// for the rest ask again to klient so we know if it's running or not
		machineData, ok := opts.CurrentData.(*Machine)
		if !ok {
			return nil, fmt.Errorf("current data is malformed: %v", opts.CurrentData)
		}

		klientRef, err := klient.NewWithTimeout(p.Kite, machineData.QueryString, time.Second*5)
		if err != nil {
			p.Log.Warning("[%s] state is '%s' but I can't connect to klient.",
				opts.MachineId, resultState)
			resultState = machinestate.Stopped
		} else {
			defer klientRef.Close()

			// now assume it's running
			resultState = machinestate.Running

			// ping the klient again just to see if it can respond to us
			if err := klientRef.Ping(); err != nil {
				p.Log.Warning("[%s] state is '%s' but I can't send a ping. Err: %s",
					opts.MachineId, resultState, err.Error())

				// seems we can't send even a simple ping! It's not
				// functional so we assume it's stopped
				resultState = machinestate.Stopped
			}
		}

		if resultState != dbState {
			// return an error anything here if the DB is locked.
			if err := p.CheckAndUpdateState(opts.MachineId, resultState); err == mgo.ErrNotFound {
				return nil, kloud.ErrLockAcquired
			}
		}

		p.Log.Info("[%s] info decision: based on klient interaction: '%s'",
			opts.MachineId, resultState)
	}

	// fix db state if the aws state is different than dbState. This will not
	// break existing actions like building,starting,stopping etc.. because
	// CheckAndUpdateState only update the state if there is no lock available
	if dbState != awsState && !klientChecked {
		// this is only set if the lock is unlocked. Thefore it will not
		// change the db state if there is an ongoing process. If there is no
		// error than it means there is no lock so we could update it with the
		// state from amazon. Therefore send it back!
		err := p.CheckAndUpdateState(opts.MachineId, awsState)
		if err == nil {
			p.Log.Info("[%s] info decision : inconsistent state. using amazon state '%s'",
				opts.MachineId, awsState)
			resultState = awsState
		}
	}

	p.Log.Info("[%s] info result   : '%s'", opts.MachineId, resultState)

	return &protocol.InfoArtifact{
		State: resultState,
		Name:  infoResp.Name,
	}, nil

}

// CheckAndUpdate state updates only if the given machine id is not used by
// anyone else
func (p *Provider) CheckAndUpdateState(id string, state machinestate.State) error {
	p.Log.Info("[%s] storage state update request to state %v", id, state)
	err := p.Session.Run("jMachines", func(c *mgo.Collection) error {
		return c.Update(
			bson.M{
				"_id": bson.ObjectIdHex(id),
				"assignee.inProgress": false, // only update if it's not locked by someone else
			},
			bson.M{
				"$set": bson.M{
					"status.state":      state.String(),
					"status.modifiedAt": time.Now().UTC(),
				},
			},
		)
	})

	if err == mgo.ErrNotFound {
		p.Log.Warning("[%s] info can't update db state because lock is acquired by someone else", id)
	}

	return err
}
<commit_msg>kloud/info: prevent invalid changes<commit_after>package koding

import (
	"fmt"
	"koding/kites/kloud/klient"
	"sync"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"github.com/koding/kloud"
	"github.com/koding/kloud/machinestate"
	"github.com/koding/kloud/protocol"
)

// invalidChanges is a list of exceptions that applies when we fix the DB state
// with the state coming from Amazon. For example if the db state is "stopped"
// there is no need to change it with the Amazon state "stopping"
var invalidChanges = map[machinestate.State]machinestate.State{
	machinestate.Stopped:    machinestate.Stopping,
	machinestate.Running:    machinestate.Starting,
	machinestate.Terminated: machinestate.Terminating,
}

// protects invalidChanges
var rwLock sync.RWMutex

// validChange returns true if the given db state is a valid to change with the
// aws state
func validChange(db, aws machinestate.State) bool {
	rwLock.Lock()
	defer rwLock.Unlock()

	return invalidChanges[db] != aws
}

func (p *Provider) Info(opts *protocol.Machine) (result *protocol.InfoArtifact, err error) {
	a, err := p.NewClient(opts)
	if err != nil {
		return nil, err
	}

	// otherwise ask AWS to get an machine state
	infoResp, err := a.Info()
	if err != nil {
		return nil, err
	}

	dbState := opts.State
	awsState := infoResp.State

	// result state is the final state that is send back to the request
	resultState := dbState

	p.Log.Info("[%s] info initials: current db state is '%s'. amazon ec2 state is '%s'",
		opts.MachineId, dbState, awsState)

	// we don't check if the state is something else. Klient is only available
	// when the machine is running
	klientChecked := false
	if dbState.In(machinestate.Running, machinestate.Stopped) && awsState == machinestate.Running {
		klientChecked = true
		// for the rest ask again to klient so we know if it's running or not
		machineData, ok := opts.CurrentData.(*Machine)
		if !ok {
			return nil, fmt.Errorf("current data is malformed: %v", opts.CurrentData)
		}

		klientRef, err := klient.NewWithTimeout(p.Kite, machineData.QueryString, time.Second*5)
		if err != nil {
			p.Log.Warning("[%s] state is '%s' but I can't connect to klient.",
				opts.MachineId, resultState)
			resultState = machinestate.Stopped
		} else {
			defer klientRef.Close()

			// now assume it's running
			resultState = machinestate.Running

			// ping the klient again just to see if it can respond to us
			if err := klientRef.Ping(); err != nil {
				p.Log.Warning("[%s] state is '%s' but I can't send a ping. Err: %s",
					opts.MachineId, resultState, err.Error())

				// seems we can't send even a simple ping! It's not
				// functional so we assume it's stopped
				resultState = machinestate.Stopped
			}
		}

		if resultState != dbState {
			// return an error anything here if the DB is locked.
			if err := p.CheckAndUpdateState(opts.MachineId, resultState); err == mgo.ErrNotFound {
				return nil, kloud.ErrLockAcquired
			}
		}

		p.Log.Info("[%s] info decision: based on klient interaction: '%s'",
			opts.MachineId, resultState)
	}

	// fix db state if the aws state is different than dbState. This will not
	// break existing actions like building,starting,stopping etc.. because
	// CheckAndUpdateState only update the state if there is no lock available.
	// however only fix when it's there was no klient checking and the state
	// chanign is a valid transformation (for example prevent if it's "Stopped"
	// -> "Stopping"
	if dbState != awsState && !klientChecked && validChange(dbState, awsState) {
		// this is only set if the lock is unlocked. Thefore it will not
		// change the db state if there is an ongoing process. If there is no
		// error than it means there is no lock so we could update it with the
		// state from amazon. Therefore send it back!
		err := p.CheckAndUpdateState(opts.MachineId, awsState)
		if err == nil {
			p.Log.Info("[%s] info decision : inconsistent state. using amazon state '%s'",
				opts.MachineId, awsState)
			resultState = awsState
		}
	}

	p.Log.Info("[%s] info result   : '%s'", opts.MachineId, resultState)

	return &protocol.InfoArtifact{
		State: resultState,
		Name:  infoResp.Name,
	}, nil

}

// CheckAndUpdate state updates only if the given machine id is not used by
// anyone else
func (p *Provider) CheckAndUpdateState(id string, state machinestate.State) error {
	p.Log.Info("[%s] storage state update request to state %v", id, state)
	err := p.Session.Run("jMachines", func(c *mgo.Collection) error {
		return c.Update(
			bson.M{
				"_id": bson.ObjectIdHex(id),
				"assignee.inProgress": false, // only update if it's not locked by someone else
			},
			bson.M{
				"$set": bson.M{
					"status.state":      state.String(),
					"status.modifiedAt": time.Now().UTC(),
				},
			},
		)
	})

	if err == mgo.ErrNotFound {
		p.Log.Warning("[%s] info can't update db state because lock is acquired by someone else", id)
	}

	return err
}
<|endoftext|>


<commit_before>package models

import (
	"errors"

	"github.com/koding/bongo"
	uuid "github.com/satori/go.uuid"
)

func NewAccount() *Account {
	return &Account{}
}

func (a Account) GetId() int64 {
	return a.Id
}

func (a Account) BongoName() string {
	return "api.account"
}

func (a *Account) BeforeCreate() error {
	return a.createToken()
}

func (a *Account) BeforeUpdate() error {
	return a.createToken()
}

func (a *Account) createToken() error {
	if a.Token == "" {
		token := uuid.NewV4()
		a.Token = token.String()
	}

	return nil
}

func (a *Account) AfterUpdate() {
	SetAccountToCache(a)
	bongo.B.AfterUpdate(a)
}

func (a *Account) AfterCreate() {
	SetAccountToCache(a)
	bongo.B.AfterCreate(a)
}

func (a *Account) One(q *bongo.Query) error {
	return bongo.B.One(a, a, q)
}

func (a *Account) ById(id int64) error {
	return bongo.B.ById(a, id)
}

func (a *Account) Update() error {
	return bongo.B.Update(a)
}

func (a *Account) Create() error {
	if a.OldId == "" {
		return errors.New("old id is not set")
	}

	if a.Nick == "guestuser" {
		return ErrGuestsAreNotAllowed
	}

	return bongo.B.Create(a)
}

func (a *Account) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(a, data, q)
}

func (a *Account) FetchByIds(ids []int64) ([]Account, error) {
	var accounts []Account

	if len(ids) == 0 {
		return accounts, nil
	}

	if err := bongo.B.FetchByIds(a, &accounts, ids); err != nil {
		return nil, err
	}

	return accounts, nil
}
<commit_msg>socialapibongo: add byToken func<commit_after>package models

import (
	"errors"

	"github.com/koding/bongo"
	uuid "github.com/satori/go.uuid"
)

func NewAccount() *Account {
	return &Account{}
}

func (a Account) GetId() int64 {
	return a.Id
}

func (a Account) BongoName() string {
	return "api.account"
}

func (a *Account) BeforeCreate() error {
	return a.createToken()
}

func (a *Account) BeforeUpdate() error {
	return a.createToken()
}

func (a *Account) createToken() error {
	if a.Token == "" {
		token := uuid.NewV4()
		a.Token = token.String()
	}

	return nil
}

func (a *Account) AfterUpdate() {
	SetAccountToCache(a)
	bongo.B.AfterUpdate(a)
}

func (a *Account) AfterCreate() {
	SetAccountToCache(a)
	bongo.B.AfterCreate(a)
}

func (a *Account) One(q *bongo.Query) error {
	return bongo.B.One(a, a, q)
}

func (a *Account) ById(id int64) error {
	return bongo.B.ById(a, id)
}

func (a *Account) ByToken(token string) error {
	if token == "" {
		return ErrIdIsNotSet
	}
	selector := map[string]interface{}{
		"token": token,
	}

	// return bongo.B.ById(a, token)
	return a.One(bongo.NewQS(selector))
}

func (a *Account) Update() error {
	return bongo.B.Update(a)
}

func (a *Account) Create() error {
	if a.OldId == "" {
		return errors.New("old id is not set")
	}

	if a.Nick == "guestuser" {
		return ErrGuestsAreNotAllowed
	}

	return bongo.B.Create(a)
}

func (a *Account) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(a, data, q)
}

func (a *Account) FetchByIds(ids []int64) ([]Account, error) {
	var accounts []Account

	if len(ids) == 0 {
		return accounts, nil
	}

	if err := bongo.B.FetchByIds(a, &accounts, ids); err != nil {
		return nil, err
	}

	return accounts, nil
}
<|endoftext|>


<commit_before>package db

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/url"
	"time"

	"github.com/bosssauce/ponzu/system/admin/config"

	"github.com/boltdb/bolt"
	"github.com/gorilla/schema"
)

var configCache url.Values

func init() {
	configCache = make(url.Values)
}

// SetConfig sets key:value pairs in the db for configuration settings
func SetConfig(data url.Values) error {
	err := store.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("_config"))

		cfg := &config.Config{}
		dec := schema.NewDecoder()
		dec.SetAliasTag("json")     // allows simpler struct tagging when creating a content type
		dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct
		err := dec.Decode(cfg, data)
		if err != nil {
			return err
		}

		// check for "invalidate" value to reset the Etag
		if len(cfg.CacheInvalidate) > 0 && cfg.CacheInvalidate[0] == "invalidate" {
			cfg.Etag = NewEtag()
		}

		j, err := json.Marshal(cfg)
		if err != nil {
			return err
		}

		err = b.Put([]byte("settings"), j)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	configCache = data

	return nil
}

// Config gets the value of a key in the configuration from the db
func Config(key string) ([]byte, error) {
	kv := make(map[string]interface{})

	cfg, err := ConfigAll()
	if err != nil {
		return nil, err
	}

	if len(cfg) < 1 {
		return nil, nil
	}

	err = json.Unmarshal(cfg, &kv)
	if err != nil {
		return nil, err
	}

	return []byte(kv[key].(string)), nil
}

// ConfigAll gets the configuration from the db
func ConfigAll() ([]byte, error) {
	val := &bytes.Buffer{}
	err := store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("_config"))
		val.Write(b.Get([]byte("settings")))

		return nil
	})
	if err != nil {
		return nil, err
	}

	return val.Bytes(), nil
}

// ConfigCache is a in-memory cache of the Configs for quicker lookups
func ConfigCache(key string) string {
	return configCache.Get(key)
}

// NewEtag generates a new Etag for response caching
func NewEtag() string {
	now := fmt.Sprintf("%d", time.Now().Unix())
	etag := base64.StdEncoding.EncodeToString([]byte(now))

	return etag
}
<commit_msg>testing check for cache invalidation to after values are decoded<commit_after>package db

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/url"
	"time"

	"github.com/bosssauce/ponzu/system/admin/config"

	"github.com/boltdb/bolt"
	"github.com/gorilla/schema"
)

var configCache url.Values

func init() {
	configCache = make(url.Values)
}

// SetConfig sets key:value pairs in the db for configuration settings
func SetConfig(data url.Values) error {
	err := store.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("_config"))

		cfg := &config.Config{}
		dec := schema.NewDecoder()
		dec.SetAliasTag("json")     // allows simpler struct tagging when creating a content type
		dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct
		err := dec.Decode(cfg, data)
		if err != nil {
			return err
		}

		// check for "invalidate" value to reset the Etag
		fmt.Println(cfg.CacheInvalidate)
		if len(cfg.CacheInvalidate) > 0 && cfg.CacheInvalidate[0] == "invalidate" {
			cfg.Etag = NewEtag()
		}

		j, err := json.Marshal(cfg)
		if err != nil {
			return err
		}

		err = b.Put([]byte("settings"), j)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	configCache = data

	return nil
}

// Config gets the value of a key in the configuration from the db
func Config(key string) ([]byte, error) {
	kv := make(map[string]interface{})

	cfg, err := ConfigAll()
	if err != nil {
		return nil, err
	}

	if len(cfg) < 1 {
		return nil, nil
	}

	err = json.Unmarshal(cfg, &kv)
	if err != nil {
		return nil, err
	}

	return []byte(kv[key].(string)), nil
}

// ConfigAll gets the configuration from the db
func ConfigAll() ([]byte, error) {
	val := &bytes.Buffer{}
	err := store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("_config"))
		val.Write(b.Get([]byte("settings")))

		return nil
	})
	if err != nil {
		return nil, err
	}

	return val.Bytes(), nil
}

// ConfigCache is a in-memory cache of the Configs for quicker lookups
func ConfigCache(key string) string {
	return configCache.Get(key)
}

// NewEtag generates a new Etag for response caching
func NewEtag() string {
	now := fmt.Sprintf("%d", time.Now().Unix())
	etag := base64.StdEncoding.EncodeToString([]byte(now))

	return etag
}
<|endoftext|>