<commit_before>// Copyright 2017 The OPA Authors.  All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package configmap

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/open-policy-agent/kube-mgmt/pkg/opa"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
)

const (
	policyLabelKey            = "openpolicyagent.org/policy"
	policyLabelValue          = "rego"
	policyStatusAnnotationKey = "openpolicyagent.org/policy-status"

	dataLabelKey            = "openpolicyagent.org/data"
	dataLabelValue          = "opa"
	dataStatusAnnotationKey = "openpolicyagent.org/data-status"

	// Special namespace in Kubernetes federation that holds scheduling policies.
	kubeFederationSchedulingPolicy = "kube-federation-scheduling-policy"

	resyncPeriod        = time.Second * 60
	syncResetBackoffMin = time.Second
	syncResetBackoffMax = time.Second * 30
)

// DefaultConfigMapMatcher returns a function that will match configmaps in
// specified namespaces and/or with a policy or data label. The first bool return
// value specifies a policy/data match and the second bool indicates if the configmap
// contains a policy.
func DefaultConfigMapMatcher(namespaces []string, requirePolicyLabel, enablePolicies, enableData bool) func(*v1.ConfigMap) (bool, bool) {
	return func(cm *v1.ConfigMap) (bool, bool) {
		var match, isPolicy bool

		// Check for data label. This label needs to be set on any
		// configmap that contains JSON data to be loaded into OPA.
		if enableData {
			match = matchesNamespace(cm, namespaces) && matchesLabel(cm, dataLabelKey, dataLabelValue)
		}

		// Check for explicit policy label or match on any policy namespace.
		if !match && enablePolicies {
			if requirePolicyLabel {
				match = matchesNamespace(cm, namespaces) && matchesLabel(cm, policyLabelKey, policyLabelValue)
			} else {
				match = matchesNamespace(cm, namespaces) || matchesLabel(cm, policyLabelKey, policyLabelValue)
			}

			if match {
				isPolicy = true
			}
		}
		return match, isPolicy
	}
}

func matchesLabel(cm *v1.ConfigMap, labelKey, labelValue string) bool {
	return cm.Labels[labelKey] == labelValue
}

func matchesNamespace(cm *v1.ConfigMap, namespaces []string) bool {
	for _, ns := range namespaces {
		if ns == cm.Namespace || ns == "*" {
			return true
		}
	}
	return false
}

// Sync replicates policies or data stored in the API server as ConfigMaps into OPA.
type Sync struct {
	kubeconfig *rest.Config
	opa        opa.Client
	clientset  *kubernetes.Clientset
	matcher    func(*v1.ConfigMap) (bool, bool)
}

// New returns a new Sync that can be started.
func New(kubeconfig *rest.Config, opa opa.Client, matcher func(*v1.ConfigMap) (bool, bool)) *Sync {
	cpy := *kubeconfig
	cpy.GroupVersion = &schema.GroupVersion{
		Version: "v1",
	}
	cpy.APIPath = "/api"
	cpy.ContentType = runtime.ContentTypeJSON
	scheme := runtime.NewScheme()
	cpy.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)}
	builder := runtime.NewSchemeBuilder(func(scheme *runtime.Scheme) error {
		scheme.AddKnownTypes(
			*cpy.GroupVersion,
			&metav1.ListOptions{},
			&v1.ConfigMapList{},
			&v1.ConfigMap{})
		return nil
	})
	builder.AddToScheme(scheme)
	return &Sync{
		kubeconfig: &cpy,
		opa:        opa,
		matcher:    matcher,
	}
}

// Run starts the synchronizer. To stop the synchronizer send a message to the
// channel.
func (s *Sync) Run(namespaces []string) (chan struct{}, error) {
	client, err := rest.RESTClientFor(s.kubeconfig)
	if err != nil {
		return nil, err
	}
	s.clientset, err = kubernetes.NewForConfig(s.kubeconfig)
	if err != nil {
		return nil, err
	}
	quit := make(chan struct{})

	if namespaces[0] == "*" {
		namespaces[0] = v1.NamespaceAll
		namespaces = namespaces[0:1]
	}

	for _, namespace := range namespaces {
		source := cache.NewListWatchFromClient(
			client,
			"configmaps",
			namespace,
			fields.Everything())
		store, controller := cache.NewInformer(
			source,
			&v1.ConfigMap{},
			time.Second*60,
			cache.ResourceEventHandlerFuncs{
				AddFunc:    s.add,
				UpdateFunc: s.update,
				DeleteFunc: s.delete,
			})
		for _, obj := range store.List() {
			cm := obj.(*v1.ConfigMap)
			if match, isPolicy := s.matcher(cm); match {
				s.syncAdd(cm, isPolicy)
			}
		}
		go controller.Run(quit)
	}
	return quit, nil
}

func (s *Sync) add(obj interface{}) {
	cm := obj.(*v1.ConfigMap)
	if match, isPolicy := s.matcher(cm); match {
		s.syncAdd(cm, isPolicy)
	}
}

func (s *Sync) update(_, obj interface{}) {
	cm := obj.(*v1.ConfigMap)
	if match, isPolicy := s.matcher(cm); match {
		s.syncAdd(cm, isPolicy)
	}
}

func (s *Sync) delete(obj interface{}) {
	cm := obj.(*v1.ConfigMap)
	if match, isPolicy := s.matcher(cm); match {
		s.syncRemove(cm, isPolicy)
	}
}

func (s *Sync) syncAdd(cm *v1.ConfigMap, isPolicy bool) {
	path := fmt.Sprintf("%v/%v", cm.Namespace, cm.Name)
	for key, value := range cm.Data {
		id := fmt.Sprintf("%v/%v", path, key)

		var err error
		if isPolicy {
			err = s.opa.InsertPolicy(id, []byte(value))
		} else {
			// We don't need to know the JSON structure, just pass it
			// directly to the OPA data store.
			var data map[string]interface{}
			if err = json.Unmarshal([]byte(value), &data); err != nil {
				logrus.Errorf("Failed to parse JSON data in configmap with id: %s", id)
			} else {
				err = s.opa.PutData(id, data)
			}
		}

		if err != nil {
			s.setStatusAnnotation(cm, status{
				Status: "error",
				Error:  err,
			}, isPolicy)
		} else {
			s.setStatusAnnotation(cm, status{
				Status: "ok",
			}, isPolicy)
		}
	}
}

func (s *Sync) syncRemove(cm *v1.ConfigMap, isPolicy bool) {
	path := fmt.Sprintf("%v/%v", cm.Namespace, cm.Name)
	for key := range cm.Data {
		id := fmt.Sprintf("%v/%v", path, key)

		if isPolicy {
			if err := s.opa.DeletePolicy(id); err != nil {
				logrus.Errorf("Failed to delete policy %v: %v", id, err)
			}
		} else {
			if err := s.opa.PatchData(path, "remove", nil); err != nil {
				logrus.Errorf("Failed to remove %v (will reset OPA data and resync in %v): %v", id, resyncPeriod, err)
				s.syncReset(id)
			}
		}
	}
}

func (s *Sync) setStatusAnnotation(cm *v1.ConfigMap, st status, isPolicy bool) {
	bs, err := json.Marshal(st)

	statusAnnotationKey := policyStatusAnnotationKey
	if !isPolicy {
		statusAnnotationKey = dataStatusAnnotationKey
	}
	if err != nil {
		logrus.Errorf("Failed to serialize %v for %v/%v: %v", statusAnnotationKey, cm.Namespace, cm.Name, err)
	}
	patch := map[string]interface{}{
		"metadata": map[string]interface{}{
			"annotations": map[string]interface{}{
				policyStatusAnnotationKey: string(bs),
			},
		},
	}
	bs, err = json.Marshal(patch)
	if err != nil {
		logrus.Errorf("Failed to serialize patch for %v/%v: %v", cm.Namespace, cm.Name, err)
	}
	_, err = s.clientset.CoreV1().ConfigMaps(cm.Namespace).Patch(cm.Name, types.StrategicMergePatchType, bs)
	if err != nil {
		logrus.Errorf("Failed to %v for %v/%v: %v", statusAnnotationKey, cm.Namespace, cm.Name, err)
	}
}

func (s *Sync) syncReset(id string) {
	d := syncResetBackoffMin
	for {
		if err := s.opa.PutData("/", map[string]interface{}{}); err != nil {
			logrus.Errorf("Failed to reset OPA data for %v (will retry after %v): %v", id, d, err)
		} else {
			return
		}
		time.Sleep(d)
		d = d * 2
		if d > syncResetBackoffMax {
			d = syncResetBackoffMax
		}
	}
}

type status struct {
	Status string `json:"status"`
	Error  error  `json:"error,omitempty"`
}
<commit_msg>Fix decode error message on configmap synchronizer<commit_after>// Copyright 2017 The OPA Authors.  All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package configmap

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/open-policy-agent/kube-mgmt/pkg/opa"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
)

const (
	policyLabelKey            = "openpolicyagent.org/policy"
	policyLabelValue          = "rego"
	policyStatusAnnotationKey = "openpolicyagent.org/policy-status"

	dataLabelKey            = "openpolicyagent.org/data"
	dataLabelValue          = "opa"
	dataStatusAnnotationKey = "openpolicyagent.org/data-status"

	// Special namespace in Kubernetes federation that holds scheduling policies.
	kubeFederationSchedulingPolicy = "kube-federation-scheduling-policy"

	resyncPeriod        = time.Second * 60
	syncResetBackoffMin = time.Second
	syncResetBackoffMax = time.Second * 30
)

// DefaultConfigMapMatcher returns a function that will match configmaps in
// specified namespaces and/or with a policy or data label. The first bool return
// value specifies a policy/data match and the second bool indicates if the configmap
// contains a policy.
func DefaultConfigMapMatcher(namespaces []string, requirePolicyLabel, enablePolicies, enableData bool) func(*v1.ConfigMap) (bool, bool) {
	return func(cm *v1.ConfigMap) (bool, bool) {
		var match, isPolicy bool

		// Check for data label. This label needs to be set on any
		// configmap that contains JSON data to be loaded into OPA.
		if enableData {
			match = matchesNamespace(cm, namespaces) && matchesLabel(cm, dataLabelKey, dataLabelValue)
		}

		// Check for explicit policy label or match on any policy namespace.
		if !match && enablePolicies {
			if requirePolicyLabel {
				match = matchesNamespace(cm, namespaces) && matchesLabel(cm, policyLabelKey, policyLabelValue)
			} else {
				match = matchesNamespace(cm, namespaces) || matchesLabel(cm, policyLabelKey, policyLabelValue)
			}

			if match {
				isPolicy = true
			}
		}
		return match, isPolicy
	}
}

func matchesLabel(cm *v1.ConfigMap, labelKey, labelValue string) bool {
	return cm.Labels[labelKey] == labelValue
}

func matchesNamespace(cm *v1.ConfigMap, namespaces []string) bool {
	for _, ns := range namespaces {
		if ns == cm.Namespace || ns == "*" {
			return true
		}
	}
	return false
}

// Sync replicates policies or data stored in the API server as ConfigMaps into OPA.
type Sync struct {
	kubeconfig *rest.Config
	opa        opa.Client
	clientset  *kubernetes.Clientset
	matcher    func(*v1.ConfigMap) (bool, bool)
}

// New returns a new Sync that can be started.
func New(kubeconfig *rest.Config, opa opa.Client, matcher func(*v1.ConfigMap) (bool, bool)) *Sync {
	cpy := *kubeconfig
	cpy.GroupVersion = &schema.GroupVersion{
		Version: "v1",
	}
	cpy.APIPath = "/api"
	cpy.ContentType = runtime.ContentTypeJSON
	scheme := runtime.NewScheme()
	cpy.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)}
	builder := runtime.NewSchemeBuilder(func(scheme *runtime.Scheme) error {
		scheme.AddKnownTypes(
			*cpy.GroupVersion,
			&metav1.ListOptions{},
			&metav1.Status{},
			&v1.ConfigMapList{},
			&v1.ConfigMap{})
		return nil
	})
	builder.AddToScheme(scheme)
	return &Sync{
		kubeconfig: &cpy,
		opa:        opa,
		matcher:    matcher,
	}
}

// Run starts the synchronizer. To stop the synchronizer send a message to the
// channel.
func (s *Sync) Run(namespaces []string) (chan struct{}, error) {
	client, err := rest.RESTClientFor(s.kubeconfig)
	if err != nil {
		return nil, err
	}
	s.clientset, err = kubernetes.NewForConfig(s.kubeconfig)
	if err != nil {
		return nil, err
	}
	quit := make(chan struct{})

	if namespaces[0] == "*" {
		namespaces[0] = v1.NamespaceAll
		namespaces = namespaces[0:1]
	}

	for _, namespace := range namespaces {
		source := cache.NewListWatchFromClient(
			client,
			"configmaps",
			namespace,
			fields.Everything())
		store, controller := cache.NewInformer(
			source,
			&v1.ConfigMap{},
			time.Second*60,
			cache.ResourceEventHandlerFuncs{
				AddFunc:    s.add,
				UpdateFunc: s.update,
				DeleteFunc: s.delete,
			})
		for _, obj := range store.List() {
			cm := obj.(*v1.ConfigMap)
			if match, isPolicy := s.matcher(cm); match {
				s.syncAdd(cm, isPolicy)
			}
		}
		go controller.Run(quit)
	}
	return quit, nil
}

func (s *Sync) add(obj interface{}) {
	cm := obj.(*v1.ConfigMap)
	if match, isPolicy := s.matcher(cm); match {
		s.syncAdd(cm, isPolicy)
	}
}

func (s *Sync) update(_, obj interface{}) {
	cm := obj.(*v1.ConfigMap)
	if match, isPolicy := s.matcher(cm); match {
		s.syncAdd(cm, isPolicy)
	}
}

func (s *Sync) delete(obj interface{}) {
	cm := obj.(*v1.ConfigMap)
	if match, isPolicy := s.matcher(cm); match {
		s.syncRemove(cm, isPolicy)
	}
}

func (s *Sync) syncAdd(cm *v1.ConfigMap, isPolicy bool) {
	path := fmt.Sprintf("%v/%v", cm.Namespace, cm.Name)
	for key, value := range cm.Data {
		id := fmt.Sprintf("%v/%v", path, key)

		var err error
		if isPolicy {
			err = s.opa.InsertPolicy(id, []byte(value))
		} else {
			// We don't need to know the JSON structure, just pass it
			// directly to the OPA data store.
			var data map[string]interface{}
			if err = json.Unmarshal([]byte(value), &data); err != nil {
				logrus.Errorf("Failed to parse JSON data in configmap with id: %s", id)
			} else {
				err = s.opa.PutData(id, data)
			}
		}

		if err != nil {
			s.setStatusAnnotation(cm, status{
				Status: "error",
				Error:  err,
			}, isPolicy)
		} else {
			s.setStatusAnnotation(cm, status{
				Status: "ok",
			}, isPolicy)
		}
	}
}

func (s *Sync) syncRemove(cm *v1.ConfigMap, isPolicy bool) {
	path := fmt.Sprintf("%v/%v", cm.Namespace, cm.Name)
	for key := range cm.Data {
		id := fmt.Sprintf("%v/%v", path, key)

		if isPolicy {
			if err := s.opa.DeletePolicy(id); err != nil {
				logrus.Errorf("Failed to delete policy %v: %v", id, err)
			}
		} else {
			if err := s.opa.PatchData(path, "remove", nil); err != nil {
				logrus.Errorf("Failed to remove %v (will reset OPA data and resync in %v): %v", id, resyncPeriod, err)
				s.syncReset(id)
			}
		}
	}
}

func (s *Sync) setStatusAnnotation(cm *v1.ConfigMap, st status, isPolicy bool) {
	bs, err := json.Marshal(st)

	statusAnnotationKey := policyStatusAnnotationKey
	if !isPolicy {
		statusAnnotationKey = dataStatusAnnotationKey
	}
	if err != nil {
		logrus.Errorf("Failed to serialize %v for %v/%v: %v", statusAnnotationKey, cm.Namespace, cm.Name, err)
	}
	patch := map[string]interface{}{
		"metadata": map[string]interface{}{
			"annotations": map[string]interface{}{
				policyStatusAnnotationKey: string(bs),
			},
		},
	}
	bs, err = json.Marshal(patch)
	if err != nil {
		logrus.Errorf("Failed to serialize patch for %v/%v: %v", cm.Namespace, cm.Name, err)
	}
	_, err = s.clientset.CoreV1().ConfigMaps(cm.Namespace).Patch(cm.Name, types.StrategicMergePatchType, bs)
	if err != nil {
		logrus.Errorf("Failed to %v for %v/%v: %v", statusAnnotationKey, cm.Namespace, cm.Name, err)
	}
}

func (s *Sync) syncReset(id string) {
	d := syncResetBackoffMin
	for {
		if err := s.opa.PutData("/", map[string]interface{}{}); err != nil {
			logrus.Errorf("Failed to reset OPA data for %v (will retry after %v): %v", id, d, err)
		} else {
			return
		}
		time.Sleep(d)
		d = d * 2
		if d > syncResetBackoffMax {
			d = syncResetBackoffMax
		}
	}
}

type status struct {
	Status string `json:"status"`
	Error  error  `json:"error,omitempty"`
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	mrand "math/rand"
	"os"
	"time"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/types"
)

// storageRenter unlocks the wallet, mines some currency, sets an allowance
// using that currency, and uploads some files.  It will periodically try to
// download those files, printing any errors that occur.
func (j *JobRunner) storageRenter() {
	j.tg.Add()
	defer j.tg.Done()

	err := j.client.Post("/wallet/unlock", fmt.Sprintf("encryptionpassword=%s&dictionary=%s", j.walletPassword, "english"), nil)
	if err != nil {
		log.Printf("[%v jobStorageRenter ERROR]: %v\n", j.siaDirectory, err)
		return
	}

	err = j.client.Get("/miner/start", nil)
	if err != nil {
		log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		return
	}

	// Mine at least 100,000 SC
	desiredbalance := types.NewCurrency64(100000).Mul(types.SiacoinPrecision)
	success := false
	for start := time.Now(); time.Since(start) < 5*time.Minute; {
		select {
		case <-j.tg.StopChan():
			return
		case <-time.After(time.Second):
		}

		var walletInfo api.WalletGET
		err = j.client.Get("/wallet", &walletInfo)
		if err != nil {
			log.Printf("[%v jobStorageRenter ERROR]: %v\n", j.siaDirectory, err)
			return
		}
		if walletInfo.ConfirmedSiacoinBalance.Cmp(desiredbalance) > 0 {
			success = true
			break
		}
	}
	if !success {
		log.Printf("[%v jobStorageRenter ERROR]: timeout: could not mine enough currency after 5 minutes\n", j.siaDirectory)
		return
	}

	// Set an initial 50ksc allowance
	allowance := types.NewCurrency64(50000).Mul(types.SiacoinPrecision)
	if err := j.client.Post("/renter", fmt.Sprintf("funds=%v&period=100", allowance), nil); err != nil {
		log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
	}

	// Every 1000 seconds, set a new allowance.
	go func() {
		for {
			j.tg.Add()

			select {
			case <-j.tg.StopChan():
				j.tg.Done()
				return
			case <-time.After(time.Second * 1000):
			}

			// set an allowance of 50kSC + a random offset from 0-10ksc
			allowance := types.NewCurrency64(50000).Mul(types.SiacoinPrecision)
			if err := j.client.Post("/renter", fmt.Sprintf("funds=%v&period=100", allowance), nil); err != nil {
				log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
			}

			j.tg.Done()
		}
	}()

	// Every 120 seconds, upload a 500MB file.  Delete one file at random once every 10 files.
	go func() {
		var files []string
		for i := 0; ; i++ {
			j.tg.Add()

			select {
			case <-j.tg.StopChan():
				j.tg.Done()
				return
			case <-time.After(time.Second * 120):
			}

			// After 10 files, delete one file at random every iteration.
			if i >= 10 {
				randindex := mrand.Intn(len(files))
				if err := j.client.Post(fmt.Sprintf("/renter/delete/%v", files[randindex]), "", nil); err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}
				files = append(files[:randindex], files[randindex+1:]...)
			}

			// Generate some random data to upload
			f, err := ioutil.TempFile("", "antfarm-renter")
			if err != nil {
				log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
			}
			defer os.Remove(f.Name())

			_, err = io.CopyN(f, rand.Reader, 500000000)
			if err != nil {
				log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
			}

			// Upload the random data
			if err = j.client.Post(fmt.Sprintf("/renter/upload/%v", f.Name()), fmt.Sprintf("source=%v", f.Name()), nil); err != nil {
				log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
			}

			files = append(files, f.Name())

			j.tg.Done()
		}
	}()

	// Every 200 seconds, verify that not more than the allowance has been spent.
	go func() {
		var renterInfo api.RenterGET
		if err := j.client.Get("/renter", &renterInfo); err != nil {
			log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		}

		var walletInfo api.WalletGET
		if err := j.client.Get("/wallet", &walletInfo); err != nil {
			log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		}

		initialBalance := walletInfo.ConfirmedSiacoinBalance

		for {
			j.tg.Add()

			if err = j.client.Get("/wallet", &walletInfo); err != nil {
				log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
			}

			spent := initialBalance.Sub(walletInfo.ConfirmedSiacoinBalance)
			if spent.Cmp(renterInfo.Settings.Allowance.Funds) > 0 {
				log.Printf("[%v jobStorageRenter ERROR: spent more than allowance: spent %v, allowance %v\n", j.siaDirectory, spent, renterInfo.Settings.Allowance.Funds)
			}

			select {
			case <-j.tg.StopChan():
				j.tg.Done()
				return
			case <-time.After(time.Second * 200):
			}

			var walletInfo api.WalletGET
			if err := j.client.Get("/wallet", &walletInfo); err != nil {
			}

			j.tg.Done()
		}
	}()
}
<commit_msg>use j.tg.Add; defer done pattern using func<commit_after>package main

import (
	"crypto/rand"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	mrand "math/rand"
	"os"
	"time"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/types"
)

// storageRenter unlocks the wallet, mines some currency, sets an allowance
// using that currency, and uploads some files.  It will periodically try to
// download those files, printing any errors that occur.
func (j *JobRunner) storageRenter() {
	j.tg.Add()
	defer j.tg.Done()

	err := j.client.Post("/wallet/unlock", fmt.Sprintf("encryptionpassword=%s&dictionary=%s", j.walletPassword, "english"), nil)
	if err != nil {
		log.Printf("[%v jobStorageRenter ERROR]: %v\n", j.siaDirectory, err)
		return
	}

	err = j.client.Get("/miner/start", nil)
	if err != nil {
		log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		return
	}

	// Mine at least 100,000 SC
	desiredbalance := types.NewCurrency64(100000).Mul(types.SiacoinPrecision)
	success := false
	for start := time.Now(); time.Since(start) < 5*time.Minute; {
		select {
		case <-j.tg.StopChan():
			return
		case <-time.After(time.Second):
		}

		var walletInfo api.WalletGET
		err = j.client.Get("/wallet", &walletInfo)
		if err != nil {
			log.Printf("[%v jobStorageRenter ERROR]: %v\n", j.siaDirectory, err)
			return
		}
		if walletInfo.ConfirmedSiacoinBalance.Cmp(desiredbalance) > 0 {
			success = true
			break
		}
	}
	if !success {
		log.Printf("[%v jobStorageRenter ERROR]: timeout: could not mine enough currency after 5 minutes\n", j.siaDirectory)
		return
	}

	// Set an initial 50ksc allowance
	allowance := types.NewCurrency64(50000).Mul(types.SiacoinPrecision)
	if err := j.client.Post("/renter", fmt.Sprintf("funds=%v&period=100", allowance), nil); err != nil {
		log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
	}

	// Every 1000 seconds, set a new allowance.
	go func() {
		for {
			func() {
				j.tg.Add()
				defer j.tg.Done()

				select {
				case <-j.tg.StopChan():
					return
				case <-time.After(time.Second * 1000):
				}

				// set an allowance of 50k SC
				allowance := types.NewCurrency64(50000).Mul(types.SiacoinPrecision)
				if err := j.client.Post("/renter", fmt.Sprintf("funds=%v&period=100", allowance), nil); err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}
			}()
		}
	}()

	// Every 120 seconds, upload a 500MB file.  After ten files, delete one file
	// at random each iteration.
	go func() {
		var files []string
		for i := 0; ; i++ {
			func() {
				j.tg.Add()
				defer j.tg.Done()

				select {
				case <-j.tg.StopChan():
					return
				case <-time.After(time.Second * 120):
				}

				if i >= 10 {
					randindex := mrand.Intn(len(files))
					if err := j.client.Post(fmt.Sprintf("/renter/delete/%v", files[randindex]), "", nil); err != nil {
						log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
					}
					files = append(files[:randindex], files[randindex+1:]...)
				}

				// Generate some random data to upload
				f, err := ioutil.TempFile("", "antfarm-renter")
				if err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}
				defer os.Remove(f.Name())

				_, err = io.CopyN(f, rand.Reader, 500000000)
				if err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}

				// Upload the random data
				if err = j.client.Post(fmt.Sprintf("/renter/upload/%v", f.Name()), fmt.Sprintf("source=%v", f.Name()), nil); err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}

				files = append(files, f.Name())
			}()
		}
	}()

	// Every 200 seconds, verify that not more than the allowance has been spent.
	go func() {
		var renterInfo api.RenterGET
		if err := j.client.Get("/renter", &renterInfo); err != nil {
			log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		}

		var walletInfo api.WalletGET
		if err := j.client.Get("/wallet", &walletInfo); err != nil {
			log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		}

		initialBalance := walletInfo.ConfirmedSiacoinBalance

		for {
			func() {
				j.tg.Add()
				defer j.tg.Done()

				select {
				case <-j.tg.StopChan():
					return
				case <-time.After(time.Second * 200):
				}

				if err = j.client.Get("/wallet", &walletInfo); err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}

				spent := initialBalance.Sub(walletInfo.ConfirmedSiacoinBalance)
				if spent.Cmp(renterInfo.Settings.Allowance.Funds) > 0 {
					log.Printf("[%v jobStorageRenter ERROR: spent more than allowance: spent %v, allowance %v\n", j.siaDirectory, spent, renterInfo.Settings.Allowance.Funds)
				}
			}()
		}
	}()
}
<|endoftext|>


<commit_before>package executor

import (
	log "github.com/Sirupsen/logrus"
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

// TestLocal tests the execution of process on local machine.
func TestLocal(t *testing.T) {
	log.SetLevel(log.ErrorLevel)

	Convey("While using Local Shell", t, func() {
		l := NewLocal()

		Convey("When blocking infinitively sleep command "+
			"is executed", func() {
			task, err := l.Execute("sleep inf")

			Convey("There should be no error", func() {
				So(err, ShouldBeNil)

				task.Stop()
			})

			Convey("Task should be still running and status should be nil", func() {
				taskState, taskStatus := task.Status()
				So(taskState, ShouldEqual, RUNNING)
				So(taskStatus, ShouldBeNil)

				task.Stop()
			})

			Convey("When we wait for task termination with the 1ms timeout", func() {
				isTaskTerminated := task.Wait(1)

				Convey("The timeout should exceed and the task not terminated ", func() {
					So(isTaskTerminated, ShouldBeFalse)
				})

				Convey("The task should be still running and status should be nil", func() {
					taskState, taskStatus := task.Status()
					So(taskState, ShouldEqual, RUNNING)
					So(taskStatus, ShouldBeNil)
				})

				task.Stop()
			})

			Convey("When we stop the task", func() {
				err := task.Stop()

				Convey("There should be no error", func() {
					So(err, ShouldBeNil)
				})

				Convey("The task should be terminated and the task status should be -1", func() {
					taskState, taskStatus := task.Status()
					So(taskState, ShouldEqual, TERMINATED)
					So(taskStatus.ExitCode, ShouldEqual, -1)
				})
			})
		})

		Convey("When command `echo output` is executed", func() {
			task, err := l.Execute("echo output")

			Convey("There should be no error", func() {
				So(err, ShouldBeNil)

				task.Stop()
			})

			Convey("When we wait for the task to terminate", func() {
				isTaskTerminated := task.Wait(500)

				Convey("Wait should states that task terminated", func() {
					So(isTaskTerminated, ShouldBeTrue)
				})

				taskState, taskStatus := task.Status()

				Convey("The task should be terminated", func() {
					So(taskState, ShouldEqual, TERMINATED)
				})

				Convey("And the exit status should be 0", func() {
					So(taskStatus.ExitCode, ShouldEqual, 0)
				})

				Convey("And command stdout needs to match 'output", func() {
					So(taskStatus.Stdout, ShouldEqual, "output\n")
				})
			})
		})

		Convey("When command which does not exists is executed", func() {
			task, err := l.Execute("commandThatDoesNotExists")

			Convey("There should be no error", func() {
				So(err, ShouldBeNil)

				task.Stop()
			})

			Convey("When we wait for the task to terminate", func() {
				isTaskTerminated := task.Wait(500)

				Convey("Wait should state that task terminated", func() {
					So(isTaskTerminated, ShouldBeTrue)
				})

				taskState, taskStatus := task.Status()

				Convey("The task should be terminated", func() {
					So(taskState, ShouldEqual, TERMINATED)
				})

				Convey("And the exit status should be 127", func() {
					So(taskStatus.ExitCode, ShouldEqual, 127)
				})
			})
		})

		Convey("When we execute two tasks in the same time", func() {
			task, err := l.Execute("echo output1")
			task2, err2 := l.Execute("echo output2")

			Convey("There should be no errors", func() {
				So(err, ShouldBeNil)
				So(err2, ShouldBeNil)
			})

			Convey("When we wait for the tasks to terminate", func() {
				isTaskTerminated := task.Wait(0)
				isTaskTerminated2 := task2.Wait(0)

				Convey("Wait should state that tasks are terminated", func() {
					So(isTaskTerminated, ShouldBeTrue)
					So(isTaskTerminated2, ShouldBeTrue)
				})

				taskState1, taskStatus1 := task.Status()
				taskState2, taskStatus2 := task2.Status()

				Convey("The tasks should be terminated", func() {
					So(taskState1, ShouldEqual, TERMINATED)
					So(taskState2, ShouldEqual, TERMINATED)
				})

				Convey("The commands stdouts needs to match 'output1' & 'output2'", func() {
					So(taskStatus1.Stdout, ShouldEqual, "output1\n")
					So(taskStatus2.Stdout, ShouldEqual, "output2\n")
				})

				Convey("Both exit statuses should be 0", func() {
					So(taskStatus1.ExitCode, ShouldEqual, 0)
					So(taskStatus2.ExitCode, ShouldEqual, 0)
				})
			})
		})
	})
}
<commit_msg>Small refactor.<commit_after>package executor

import (
	log "github.com/Sirupsen/logrus"
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

// TestLocal tests the execution of process on local machine.
func TestLocal(t *testing.T) {
	log.SetLevel(log.ErrorLevel)

	Convey("While using Local Shell", t, func() {
		l := NewLocal()

		Convey("When blocking infinitively sleep command is executed", func() {
			task, err := l.Execute("sleep inf")

			Convey("There should be no error", func() {
				So(err, ShouldBeNil)

				task.Stop()
			})

			Convey("Task should be still running and status should be nil", func() {
				taskState, taskStatus := task.Status()
				So(taskState, ShouldEqual, RUNNING)
				So(taskStatus, ShouldBeNil)

				task.Stop()
			})

			Convey("When we wait for task termination with the 1ms timeout", func() {
				isTaskTerminated := task.Wait(1)

				Convey("The timeout should exceed and the task not terminated ", func() {
					So(isTaskTerminated, ShouldBeFalse)
				})

				Convey("The task should be still running and status should be nil", func() {
					taskState, taskStatus := task.Status()
					So(taskState, ShouldEqual, RUNNING)
					So(taskStatus, ShouldBeNil)
				})

				task.Stop()
			})

			Convey("When we stop the task", func() {
				err := task.Stop()

				Convey("There should be no error", func() {
					So(err, ShouldBeNil)
				})

				Convey("The task should be terminated and the task status should be -1", func() {
					taskState, taskStatus := task.Status()
					So(taskState, ShouldEqual, TERMINATED)
					So(taskStatus.ExitCode, ShouldEqual, -1)
				})
			})
		})

		Convey("When command `echo output` is executed", func() {
			task, err := l.Execute("echo output")

			Convey("There should be no error", func() {
				So(err, ShouldBeNil)

				task.Stop()
			})

			Convey("When we wait for the task to terminate", func() {
				isTaskTerminated := task.Wait(500)

				Convey("Wait should states that task terminated", func() {
					So(isTaskTerminated, ShouldBeTrue)
				})

				taskState, taskStatus := task.Status()

				Convey("The task should be terminated", func() {
					So(taskState, ShouldEqual, TERMINATED)
				})

				Convey("And the exit status should be 0", func() {
					So(taskStatus.ExitCode, ShouldEqual, 0)
				})

				Convey("And command stdout needs to match 'output", func() {
					So(taskStatus.Stdout, ShouldEqual, "output\n")
				})
			})
		})

		Convey("When command which does not exists is executed", func() {
			task, err := l.Execute("commandThatDoesNotExists")

			Convey("There should be no error", func() {
				So(err, ShouldBeNil)

				task.Stop()
			})

			Convey("When we wait for the task to terminate", func() {
				isTaskTerminated := task.Wait(500)

				Convey("Wait should state that task terminated", func() {
					So(isTaskTerminated, ShouldBeTrue)
				})

				taskState, taskStatus := task.Status()

				Convey("The task should be terminated", func() {
					So(taskState, ShouldEqual, TERMINATED)
				})

				Convey("And the exit status should be 127", func() {
					So(taskStatus.ExitCode, ShouldEqual, 127)
				})
			})
		})

		Convey("When we execute two tasks in the same time", func() {
			task, err := l.Execute("echo output1")
			task2, err2 := l.Execute("echo output2")

			Convey("There should be no errors", func() {
				So(err, ShouldBeNil)
				So(err2, ShouldBeNil)
			})

			Convey("When we wait for the tasks to terminate", func() {
				isTaskTerminated := task.Wait(0)
				isTaskTerminated2 := task2.Wait(0)

				Convey("Wait should state that tasks are terminated", func() {
					So(isTaskTerminated, ShouldBeTrue)
					So(isTaskTerminated2, ShouldBeTrue)
				})

				taskState1, taskStatus1 := task.Status()
				taskState2, taskStatus2 := task2.Status()

				Convey("The tasks should be terminated", func() {
					So(taskState1, ShouldEqual, TERMINATED)
					So(taskState2, ShouldEqual, TERMINATED)
				})

				Convey("The commands stdouts needs to match 'output1' & 'output2'", func() {
					So(taskStatus1.Stdout, ShouldEqual, "output1\n")
					So(taskStatus2.Stdout, ShouldEqual, "output2\n")
				})

				Convey("Both exit statuses should be 0", func() {
					So(taskStatus1.ExitCode, ShouldEqual, 0)
					So(taskStatus2.ExitCode, ShouldEqual, 0)
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>// Package gateway connects a Sia node to the Sia flood network. The flood
// network is used to propagate blocks and transactions. The gateway is the
// primary avenue that a node uses to hear about transactions and blocks, and
// is the primary avenue used to tell the network about blocks that you have
// mined or about transactions that you have created.
package gateway

import (
	"time"
)

// For the user to be securely connected to the network, the user must be
// connected to at least one node which will send them all of the blocks. An
// attacker can trick the user into thinking that a different blockchain is the
// full blockchain if the user is not connected to any nodes who are seeing +
// broadcasting the real chain (and instead is connected only to attacker nodes
// or to nodes that are not broadcasting). This situation is called an eclipse
// attack.
//
// Connecting to a large number of nodes increases the resiliancy of the
// network, but also puts a networking burden on the nodes and can slow down
// block propagation or increase orphan rates. The gateway's job is to keep the
// network efficient while also protecting the user against attacks.
//
// The gateway keeps a list of nodes that it knows about. It uses this list to
// form connections with other nodes, and then uses those connections to
// participate in the flood network. The primary vector for an attacker to
// achieve an eclipse attack is node list domination. If a gateway's nodelist
// is heavily dominated by attacking nodes, then when the gateway chooses to
// make random connections the gateway is at risk of selecting only attacker
// nodes.
//
// The gateway defends itself from these attacks by minimizing the amount of
// control that an attacker has over the node list and peer list. The first
// major defense is that the gateway maintains 8 'outbound' relationships,
// which means that the gateway created those relationships instead of an
// attacker. If a node forms a connection to you, that node is called
// 'inbound', and because it may be an attacker node, it is not trusted.
// Outbound nodes can also be attacker nodes, but they are less likely to be
// attacker nodes because you chose them, instead of them choosing you.
//
// If the gateway forms too many connections, the gateway will allow incoming
// connections by kicking an existing peer. But, to limit the amount of control
// that an attacker may have, only inbound peers are selected to be kicked.
// Furthermore, to increase the difficulty of attack, if a new inbound
// connection shares the same IP address as an existing connection, the shared
// connection is the connection that gets dropped (unless that connection is a
// local or outbound connection).
//
// Nodes are added to a peerlist in two methods. The first method is that a
// gateway will ask its outbound peers for a list of nodes. If the node list is
// below a certain size (see consts.go), the gateway will repeatedly ask
// outbound peers to expand the list. Nodes are also added to the nodelist
// after they successfully form a connection with the gateway. To limit the
// attacker's ability to add nodes to the nodelist, connections are
// ratelimited. An attacker with lots of IP addresses still has the ability to
// fill up the nodelist, however getting 90% dominance of the nodelist requires
// forming thousands of connections, which will take hours or days. By that
// time, the attacked node should already have its set of outbound peers,
// limiting the amount of damage that the attacker can do.
//
// To limit DNS-based tomfoolry, nodes are only added to the nodelist if their
// connection information takes the form of an IP address.
//
// Some research has been done on Bitcoin's flood networks. The more relevant
// research has been listed below. The papers listed first are more relevant.
//     Eclipse Attacks on Bitcoin's Peer-to-Peer Network (Heilman, Kendler, Zohar, Goldberg)
//     Stubborn Mining: Generalizing Selfish Mining and Combining with an Eclipse Attack (Nayak, Kumar, Miller, Shi)
//     An Overview of BGP Hijacking (https://www.bishopfox.com/blog/2015/08/an-overview-of-bgp-hijacking/)

// TODO: Currently the gateway does not do much in terms of bucketing. The
// gateway should make sure that it has outbound peers from a wide range of IP
// addresses, and when kicking inbound peers it shouldn't just favor kicking
// peers of the same IP address, it should favor kicking peers of the same ip
// address range.
//
// TODO: Currently the gateway does not save a list of its outbound
// connections. When it restarts, it will have a full nodelist (which may be
// primarily attacker nodes) and it will be connecting primarily to nodes in
// the nodelist. Instead, it should start by trying to connect to peers that
// have previously been outbound peers, as it is less likely that those have
// been manipulated.
//
// TODO: When peers connect to eachother, and when they add nodes to the node
// list, there is no verification that the peers are running on the same Sia
// network, something that will be problematic if we set up a large testnet.
// It's already problematic, and currently only solved by using a different set
// of bootstrap nodes. If there is any cross-polination (which an attacker
// could do pretty easily), the gateways will not clean up over time, which
// will degrade the quality of the flood network as the two networks will
// continuously flood eachother with irrelevant information. Additionally, there
// is no public key exhcange, so communications cannot be effectively encrypted
// or authenticated. The nodes must have some way to share keys.
//
// TODO: Gateway hostname discovery currently has significant centralization,
// namely the fallback is a single third-party website that can easily form any
// response it wants. Instead, multiple TLS-protected third party websites
// should be used, and the plurality answer should be accepted as the true
// hostname.
//
// TODO: The gateway currently does hostname discovery in a non-blocking way,
// which means that the first few peers that it connects to may not get the
// correct hostname. This means that you may give the remote peer the wrong
// hostname, which means they will not be able to dial you back, which means
// they will not add you to their node list.
//
// TODO: The gateway should encrypt and authenticate all communications. Though
// the gateway participates in a flood network, practical attacks have been
// demonstrated which have been able to confuse nodes by manipulating messages
// from their peers. Encryption + authentication would have made the attack
// more difficult.
//
// TODO: The gateway does an unofficial ping in two places within nodes.go.
// These unofficial pings can be found by searching for the string "0.0.0". To
// perform the unofficial ping, the gateway connects to a peer and writes its
// version as 0.0.0. When the connection completes and the handshake passes,
// the gateway writes its version as '0.0.0', causing the other node to reject
// the connection and disconnect. The gateway then can tell that the node on
// the other end of the line is a Sia gateway, without having to add that node
// as a peer. A better solution to this hack is to just add an RPC for the ping
// function. Rollout will of course have to be gradual, as new nodes and old
// nodes need to be able to successfully ping eachother until a sufficient
// portion of the network has upgraded to the new code.
//
// TODO: The gateway, when connecting to a peer, currently has no way of
// recognizing whether that peer is itself or not. Because of this, the gateway
// does occasionally try to connect to itself, especially if it has a really
// small node list. This can cause problems, especially during testing, if the
// gateway is intentionally limiting the number of local outbount peers that it
// has. For that reason, the 'permanentPeerManager' will allow the gateway to
// add local peers as outbound peers all the way up to 3 total connections. The
// gateway needs a proper way to recognize if it is trying to connect itself
// (sending a nonce when performing the version handshake, for example). Once
// that has been implemented, maxLocalOutobundPeers can be reduced to 2 or 1
// (probably 2).

import (
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	siasync "github.com/NebulousLabs/Sia/sync"
)

var (
	errNoPeers     = errors.New("no peers")
	errUnreachable = errors.New("peer did not respond to ping")
)

// Gateway implements the modules.Gateway interface.
type Gateway struct {
	listener net.Listener
	myAddr   modules.NetAddress
	port     string

	// handlers are the RPCs that the Gateway can handle.
	//
	// initRPCs are the RPCs that the Gateway calls upon connecting to a peer.
	handlers map[rpcID]modules.RPCFunc
	initRPCs map[string]modules.RPCFunc

	// nodes is the set of all known nodes (i.e. potential peers).
	//
	// peers are the nodes that the gateway is currently connected to.
	//
	// peerTG is a special thread group for tracking peer connections, and will
	// block shutdown until all peer connections have been closed out. The peer
	// connections are put in a separate TG because of their unique
	// requirements - they have the potential to live for the lifetime of the
	// program, but also the potential to close early. Calling threads.OnStop
	// for each peer could create a huge backlog of functions that do nothing
	// (because most of the peers disconnected prior to shutdown). And they
	// can't call threads.Add because they are potentially very long running
	// and would block any threads.Flush() calls. So a second threadgroup is
	// added which handles clean-shutdown for the peers, without blocking
	// threads.Flush() calls.
	nodes  map[modules.NetAddress]struct{}
	peers  map[modules.NetAddress]*peer
	peerTG siasync.ThreadGroup

	// Utilities.
	log        *persist.Logger
	mu         sync.RWMutex
	persistDir string
	threads    siasync.ThreadGroup
}

// managedSleep will sleep for the given period of time. If the full time
// elapses, 'true' is returned. If the sleep is interrupted for shutdown,
// 'false' is returned.
func (g *Gateway) managedSleep(t time.Duration) (completed bool) {
	select {
	case <-time.After(t):
		return true
	case <-g.threads.StopChan():
		return false
	}
}

// Address returns the NetAddress of the Gateway.
func (g *Gateway) Address() modules.NetAddress {
	g.mu.RLock()
	defer g.mu.RUnlock()
	return g.myAddr
}

// Close saves the state of the Gateway and stops its listener process.
func (g *Gateway) Close() error {
	if err := g.threads.Stop(); err != nil {
		return err
	}
	g.mu.RLock()
	defer g.mu.RUnlock()
	return g.saveSync()
}

// New returns an initialized Gateway.
func New(addr string, bootstrap bool, persistDir string) (*Gateway, error) {
	// Create the directory if it doesn't exist.
	err := os.MkdirAll(persistDir, 0700)
	if err != nil {
		return nil, err
	}

	g := &Gateway{
		handlers: make(map[rpcID]modules.RPCFunc),
		initRPCs: make(map[string]modules.RPCFunc),

		peers: make(map[modules.NetAddress]*peer),
		nodes: make(map[modules.NetAddress]struct{}),

		persistDir: persistDir,
	}

	// Create the logger.
	g.log, err = persist.NewFileLogger(filepath.Join(g.persistDir, logFile))
	if err != nil {
		return nil, err
	}
	// Establish the closing of the logger.
	g.threads.AfterStop(func() {
		if err := g.log.Close(); err != nil {
			// The logger may or may not be working here, so use a println
			// instead.
			fmt.Println("Failed to close the gateway logger:", err)
		}
	})
	g.log.Println("INFO: gateway created, started logging")

	// Establish that the peerTG must complete shutdown before the primary
	// thread group completes shutdown.
	g.threads.OnStop(func() {
		err = g.peerTG.Stop()
		if err != nil {
			g.log.Println("ERROR: peerTG experienced errors while shutting down:", err)
		}
	})

	// Register RPCs.
	g.RegisterRPC("ShareNodes", g.shareNodes)
	g.RegisterConnectCall("ShareNodes", g.requestNodes)
	// Establish the de-registration of the RPCs.
	g.threads.OnStop(func() {
		g.UnregisterRPC("ShareNodes")
		g.UnregisterConnectCall("ShareNodes")
	})

	// Load the old node list. If it doesn't exist, no problem, but if it does,
	// we want to know about any errors preventing us from loading it.
	if loadErr := g.load(); loadErr != nil && !os.IsNotExist(loadErr) {
		return nil, loadErr
	}
	// Spawn the thread to periodically save the gateway.
	go g.threadedSaveLoop()
	// Make sure that the gateway saves after shutdown.
	g.threads.AfterStop(func() {
		g.mu.Lock()
		err = g.saveSync()
		g.mu.Unlock()
		if err != nil {
			g.log.Println("ERROR: Unable to save gateway:", err)
		}
	})

	// Add the bootstrap peers to the node list.
	if bootstrap {
		for _, addr := range modules.BootstrapPeers {
			err := g.addNode(addr)
			if err != nil && err != errNodeExists {
				g.log.Printf("WARN: failed to add the bootstrap node '%v': %v", addr, err)
			}
		}
	}

	// Create the listener which will listen for new connections from peers.
	permanentListenClosedChan := make(chan struct{})
	g.listener, err = net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}
	// Automatically close the listener when g.threads.Stop() is called.
	g.threads.OnStop(func() {
		err := g.listener.Close()
		if err != nil {
			g.log.Println("WARN: closing the listener failed:", err)
		}
		<-permanentListenClosedChan
	})
	// Set the address and port of the gateway.
	_, g.port, err = net.SplitHostPort(g.listener.Addr().String())
	if err != nil {
		return nil, err
	}
	// Set myAddr equal to the address returned by the listener. It will be
	// overwritten by threadedLearnHostname later on.
	g.myAddr = modules.NetAddress(g.listener.Addr().String())

	// Spawn the peer connection listener.
	go g.permanentListen(permanentListenClosedChan)

	// Spawn the peer manager and provide tools for ensuring clean shutdown.
	peerManagerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-peerManagerClosedChan
	})
	go g.permanentPeerManager(peerManagerClosedChan)

	// Spawn the node manager and provide tools for ensuring clean shudown.
	nodeManagerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-nodeManagerClosedChan
	})
	go g.permanentNodeManager(nodeManagerClosedChan)

	// Spawn the node purger and provide tools for ensuring clean shutdown.
	nodePurgerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-nodePurgerClosedChan
	})
	go g.permanentNodePurger(nodePurgerClosedChan)

	// Spawn threads to take care of port forwarding and hostname discovery.
	go g.threadedForwardPort(g.port)
	go g.threadedLearnHostname()

	return g, nil
}

// enforce that Gateway satisfies the modules.Gateway interface
var _ modules.Gateway = (*Gateway)(nil)
<commit_msg>exclusive lock on gateway while saving<commit_after>// Package gateway connects a Sia node to the Sia flood network. The flood
// network is used to propagate blocks and transactions. The gateway is the
// primary avenue that a node uses to hear about transactions and blocks, and
// is the primary avenue used to tell the network about blocks that you have
// mined or about transactions that you have created.
package gateway

import (
	"time"
)

// For the user to be securely connected to the network, the user must be
// connected to at least one node which will send them all of the blocks. An
// attacker can trick the user into thinking that a different blockchain is the
// full blockchain if the user is not connected to any nodes who are seeing +
// broadcasting the real chain (and instead is connected only to attacker nodes
// or to nodes that are not broadcasting). This situation is called an eclipse
// attack.
//
// Connecting to a large number of nodes increases the resiliancy of the
// network, but also puts a networking burden on the nodes and can slow down
// block propagation or increase orphan rates. The gateway's job is to keep the
// network efficient while also protecting the user against attacks.
//
// The gateway keeps a list of nodes that it knows about. It uses this list to
// form connections with other nodes, and then uses those connections to
// participate in the flood network. The primary vector for an attacker to
// achieve an eclipse attack is node list domination. If a gateway's nodelist
// is heavily dominated by attacking nodes, then when the gateway chooses to
// make random connections the gateway is at risk of selecting only attacker
// nodes.
//
// The gateway defends itself from these attacks by minimizing the amount of
// control that an attacker has over the node list and peer list. The first
// major defense is that the gateway maintains 8 'outbound' relationships,
// which means that the gateway created those relationships instead of an
// attacker. If a node forms a connection to you, that node is called
// 'inbound', and because it may be an attacker node, it is not trusted.
// Outbound nodes can also be attacker nodes, but they are less likely to be
// attacker nodes because you chose them, instead of them choosing you.
//
// If the gateway forms too many connections, the gateway will allow incoming
// connections by kicking an existing peer. But, to limit the amount of control
// that an attacker may have, only inbound peers are selected to be kicked.
// Furthermore, to increase the difficulty of attack, if a new inbound
// connection shares the same IP address as an existing connection, the shared
// connection is the connection that gets dropped (unless that connection is a
// local or outbound connection).
//
// Nodes are added to a peerlist in two methods. The first method is that a
// gateway will ask its outbound peers for a list of nodes. If the node list is
// below a certain size (see consts.go), the gateway will repeatedly ask
// outbound peers to expand the list. Nodes are also added to the nodelist
// after they successfully form a connection with the gateway. To limit the
// attacker's ability to add nodes to the nodelist, connections are
// ratelimited. An attacker with lots of IP addresses still has the ability to
// fill up the nodelist, however getting 90% dominance of the nodelist requires
// forming thousands of connections, which will take hours or days. By that
// time, the attacked node should already have its set of outbound peers,
// limiting the amount of damage that the attacker can do.
//
// To limit DNS-based tomfoolry, nodes are only added to the nodelist if their
// connection information takes the form of an IP address.
//
// Some research has been done on Bitcoin's flood networks. The more relevant
// research has been listed below. The papers listed first are more relevant.
//     Eclipse Attacks on Bitcoin's Peer-to-Peer Network (Heilman, Kendler, Zohar, Goldberg)
//     Stubborn Mining: Generalizing Selfish Mining and Combining with an Eclipse Attack (Nayak, Kumar, Miller, Shi)
//     An Overview of BGP Hijacking (https://www.bishopfox.com/blog/2015/08/an-overview-of-bgp-hijacking/)

// TODO: Currently the gateway does not do much in terms of bucketing. The
// gateway should make sure that it has outbound peers from a wide range of IP
// addresses, and when kicking inbound peers it shouldn't just favor kicking
// peers of the same IP address, it should favor kicking peers of the same ip
// address range.
//
// TODO: Currently the gateway does not save a list of its outbound
// connections. When it restarts, it will have a full nodelist (which may be
// primarily attacker nodes) and it will be connecting primarily to nodes in
// the nodelist. Instead, it should start by trying to connect to peers that
// have previously been outbound peers, as it is less likely that those have
// been manipulated.
//
// TODO: When peers connect to eachother, and when they add nodes to the node
// list, there is no verification that the peers are running on the same Sia
// network, something that will be problematic if we set up a large testnet.
// It's already problematic, and currently only solved by using a different set
// of bootstrap nodes. If there is any cross-polination (which an attacker
// could do pretty easily), the gateways will not clean up over time, which
// will degrade the quality of the flood network as the two networks will
// continuously flood eachother with irrelevant information. Additionally, there
// is no public key exhcange, so communications cannot be effectively encrypted
// or authenticated. The nodes must have some way to share keys.
//
// TODO: Gateway hostname discovery currently has significant centralization,
// namely the fallback is a single third-party website that can easily form any
// response it wants. Instead, multiple TLS-protected third party websites
// should be used, and the plurality answer should be accepted as the true
// hostname.
//
// TODO: The gateway currently does hostname discovery in a non-blocking way,
// which means that the first few peers that it connects to may not get the
// correct hostname. This means that you may give the remote peer the wrong
// hostname, which means they will not be able to dial you back, which means
// they will not add you to their node list.
//
// TODO: The gateway should encrypt and authenticate all communications. Though
// the gateway participates in a flood network, practical attacks have been
// demonstrated which have been able to confuse nodes by manipulating messages
// from their peers. Encryption + authentication would have made the attack
// more difficult.
//
// TODO: The gateway does an unofficial ping in two places within nodes.go.
// These unofficial pings can be found by searching for the string "0.0.0". To
// perform the unofficial ping, the gateway connects to a peer and writes its
// version as 0.0.0. When the connection completes and the handshake passes,
// the gateway writes its version as '0.0.0', causing the other node to reject
// the connection and disconnect. The gateway then can tell that the node on
// the other end of the line is a Sia gateway, without having to add that node
// as a peer. A better solution to this hack is to just add an RPC for the ping
// function. Rollout will of course have to be gradual, as new nodes and old
// nodes need to be able to successfully ping eachother until a sufficient
// portion of the network has upgraded to the new code.
//
// TODO: The gateway, when connecting to a peer, currently has no way of
// recognizing whether that peer is itself or not. Because of this, the gateway
// does occasionally try to connect to itself, especially if it has a really
// small node list. This can cause problems, especially during testing, if the
// gateway is intentionally limiting the number of local outbount peers that it
// has. For that reason, the 'permanentPeerManager' will allow the gateway to
// add local peers as outbound peers all the way up to 3 total connections. The
// gateway needs a proper way to recognize if it is trying to connect itself
// (sending a nonce when performing the version handshake, for example). Once
// that has been implemented, maxLocalOutobundPeers can be reduced to 2 or 1
// (probably 2).

import (
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	siasync "github.com/NebulousLabs/Sia/sync"
)

var (
	errNoPeers     = errors.New("no peers")
	errUnreachable = errors.New("peer did not respond to ping")
)

// Gateway implements the modules.Gateway interface.
type Gateway struct {
	listener net.Listener
	myAddr   modules.NetAddress
	port     string

	// handlers are the RPCs that the Gateway can handle.
	//
	// initRPCs are the RPCs that the Gateway calls upon connecting to a peer.
	handlers map[rpcID]modules.RPCFunc
	initRPCs map[string]modules.RPCFunc

	// nodes is the set of all known nodes (i.e. potential peers).
	//
	// peers are the nodes that the gateway is currently connected to.
	//
	// peerTG is a special thread group for tracking peer connections, and will
	// block shutdown until all peer connections have been closed out. The peer
	// connections are put in a separate TG because of their unique
	// requirements - they have the potential to live for the lifetime of the
	// program, but also the potential to close early. Calling threads.OnStop
	// for each peer could create a huge backlog of functions that do nothing
	// (because most of the peers disconnected prior to shutdown). And they
	// can't call threads.Add because they are potentially very long running
	// and would block any threads.Flush() calls. So a second threadgroup is
	// added which handles clean-shutdown for the peers, without blocking
	// threads.Flush() calls.
	nodes  map[modules.NetAddress]struct{}
	peers  map[modules.NetAddress]*peer
	peerTG siasync.ThreadGroup

	// Utilities.
	log        *persist.Logger
	mu         sync.RWMutex
	persistDir string
	threads    siasync.ThreadGroup
}

// managedSleep will sleep for the given period of time. If the full time
// elapses, 'true' is returned. If the sleep is interrupted for shutdown,
// 'false' is returned.
func (g *Gateway) managedSleep(t time.Duration) (completed bool) {
	select {
	case <-time.After(t):
		return true
	case <-g.threads.StopChan():
		return false
	}
}

// Address returns the NetAddress of the Gateway.
func (g *Gateway) Address() modules.NetAddress {
	g.mu.RLock()
	defer g.mu.RUnlock()
	return g.myAddr
}

// Close saves the state of the Gateway and stops its listener process.
func (g *Gateway) Close() error {
	if err := g.threads.Stop(); err != nil {
		return err
	}
	g.mu.Lock()
	defer g.mu.Unlock()
	return g.saveSync()
}

// New returns an initialized Gateway.
func New(addr string, bootstrap bool, persistDir string) (*Gateway, error) {
	// Create the directory if it doesn't exist.
	err := os.MkdirAll(persistDir, 0700)
	if err != nil {
		return nil, err
	}

	g := &Gateway{
		handlers: make(map[rpcID]modules.RPCFunc),
		initRPCs: make(map[string]modules.RPCFunc),

		peers: make(map[modules.NetAddress]*peer),
		nodes: make(map[modules.NetAddress]struct{}),

		persistDir: persistDir,
	}

	// Create the logger.
	g.log, err = persist.NewFileLogger(filepath.Join(g.persistDir, logFile))
	if err != nil {
		return nil, err
	}
	// Establish the closing of the logger.
	g.threads.AfterStop(func() {
		if err := g.log.Close(); err != nil {
			// The logger may or may not be working here, so use a println
			// instead.
			fmt.Println("Failed to close the gateway logger:", err)
		}
	})
	g.log.Println("INFO: gateway created, started logging")

	// Establish that the peerTG must complete shutdown before the primary
	// thread group completes shutdown.
	g.threads.OnStop(func() {
		err = g.peerTG.Stop()
		if err != nil {
			g.log.Println("ERROR: peerTG experienced errors while shutting down:", err)
		}
	})

	// Register RPCs.
	g.RegisterRPC("ShareNodes", g.shareNodes)
	g.RegisterConnectCall("ShareNodes", g.requestNodes)
	// Establish the de-registration of the RPCs.
	g.threads.OnStop(func() {
		g.UnregisterRPC("ShareNodes")
		g.UnregisterConnectCall("ShareNodes")
	})

	// Load the old node list. If it doesn't exist, no problem, but if it does,
	// we want to know about any errors preventing us from loading it.
	if loadErr := g.load(); loadErr != nil && !os.IsNotExist(loadErr) {
		return nil, loadErr
	}
	// Spawn the thread to periodically save the gateway.
	go g.threadedSaveLoop()
	// Make sure that the gateway saves after shutdown.
	g.threads.AfterStop(func() {
		g.mu.Lock()
		err = g.saveSync()
		g.mu.Unlock()
		if err != nil {
			g.log.Println("ERROR: Unable to save gateway:", err)
		}
	})

	// Add the bootstrap peers to the node list.
	if bootstrap {
		for _, addr := range modules.BootstrapPeers {
			err := g.addNode(addr)
			if err != nil && err != errNodeExists {
				g.log.Printf("WARN: failed to add the bootstrap node '%v': %v", addr, err)
			}
		}
	}

	// Create the listener which will listen for new connections from peers.
	permanentListenClosedChan := make(chan struct{})
	g.listener, err = net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}
	// Automatically close the listener when g.threads.Stop() is called.
	g.threads.OnStop(func() {
		err := g.listener.Close()
		if err != nil {
			g.log.Println("WARN: closing the listener failed:", err)
		}
		<-permanentListenClosedChan
	})
	// Set the address and port of the gateway.
	_, g.port, err = net.SplitHostPort(g.listener.Addr().String())
	if err != nil {
		return nil, err
	}
	// Set myAddr equal to the address returned by the listener. It will be
	// overwritten by threadedLearnHostname later on.
	g.myAddr = modules.NetAddress(g.listener.Addr().String())

	// Spawn the peer connection listener.
	go g.permanentListen(permanentListenClosedChan)

	// Spawn the peer manager and provide tools for ensuring clean shutdown.
	peerManagerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-peerManagerClosedChan
	})
	go g.permanentPeerManager(peerManagerClosedChan)

	// Spawn the node manager and provide tools for ensuring clean shudown.
	nodeManagerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-nodeManagerClosedChan
	})
	go g.permanentNodeManager(nodeManagerClosedChan)

	// Spawn the node purger and provide tools for ensuring clean shutdown.
	nodePurgerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-nodePurgerClosedChan
	})
	go g.permanentNodePurger(nodePurgerClosedChan)

	// Spawn threads to take care of port forwarding and hostname discovery.
	go g.threadedForwardPort(g.port)
	go g.threadedLearnHostname()

	return g, nil
}

// enforce that Gateway satisfies the modules.Gateway interface
var _ modules.Gateway = (*Gateway)(nil)
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rules

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"

	"github.com/ghodss/yaml"
	"k8s.io/helm/pkg/chartutil"
	"k8s.io/helm/pkg/engine"
	"k8s.io/helm/pkg/lint/support"
	cpb "k8s.io/helm/pkg/proto/hapi/chart"
	"k8s.io/helm/pkg/timeconv"
	tversion "k8s.io/helm/pkg/version"
)

// Templates lints the templates in the Linter.
func Templates(linter *support.Linter, values []byte, namespace string, strict bool) {
	path := "templates/"
	templatesPath := filepath.Join(linter.ChartDir, path)

	templatesDirExist := linter.RunLinterRule(support.WarningSev, path, validateTemplatesDir(templatesPath))

	// Templates directory is optional for now
	if !templatesDirExist {
		return
	}

	// Load chart and parse templates, based on tiller/release_server
	chart, err := chartutil.Load(linter.ChartDir)

	chartLoaded := linter.RunLinterRule(support.ErrorSev, path, err)

	if !chartLoaded {
		return
	}

	options := chartutil.ReleaseOptions{Name: "testRelease", Time: timeconv.Now(), Namespace: namespace}
	caps := &chartutil.Capabilities{
		APIVersions:   chartutil.DefaultVersionSet,
		KubeVersion:   chartutil.DefaultKubeVersion,
		TillerVersion: tversion.GetVersionProto(),
	}
	cvals, err := chartutil.CoalesceValues(chart, &cpb.Config{Raw: string(values)})
	if err != nil {
		return
	}
	// convert our values back into config
	yvals, err := cvals.YAML()
	if err != nil {
		return
	}
	cc := &cpb.Config{Raw: yvals}
	valuesToRender, err := chartutil.ToRenderValuesCaps(chart, cc, options, caps)
	if err != nil {
		// FIXME: This seems to generate a duplicate, but I can't find where the first
		// error is coming from.
		//linter.RunLinterRule(support.ErrorSev, err)
		return
	}
	e := engine.New()
	e.LintMode = true
	if strict {
		e.Strict = true
	}
	renderedContentMap, err := e.Render(chart, valuesToRender)

	renderOk := linter.RunLinterRule(support.ErrorSev, path, err)

	if !renderOk {
		return
	}

	/* Iterate over all the templates to check:
	- It is a .yaml file
	- All the values in the template file is defined
	- {{}} include | quote
	- Generated content is a valid Yaml file
	- Metadata.Namespace is not set
	*/
	for _, template := range chart.Templates {
		fileName, _ := template.Name, template.Data
		path = fileName

		linter.RunLinterRule(support.WarningSev, path, validateAllowedExtension(fileName))

		// We only apply the following lint rules to yaml files
		if filepath.Ext(fileName) != ".yaml" || filepath.Ext(fileName) == ".yml" {
			continue
		}

		// NOTE: disabled for now, Refs https://github.com/kubernetes/helm/issues/1463
		// Check that all the templates have a matching value
		//linter.RunLinterRule(support.WarningSev, path, validateNoMissingValues(templatesPath, valuesToRender, preExecutedTemplate))

		// NOTE: disabled for now, Refs https://github.com/kubernetes/helm/issues/1037
		// linter.RunLinterRule(support.WarningSev, path, validateQuotes(string(preExecutedTemplate)))

		renderedContent := renderedContentMap[filepath.Join(chart.GetMetadata().Name, fileName)]
		var yamlStruct K8sYamlStruct
		// Even though K8sYamlStruct only defines Metadata namespace, an error in any other
		// key will be raised as well
		err := yaml.Unmarshal([]byte(renderedContent), &yamlStruct)

		validYaml := linter.RunLinterRule(support.ErrorSev, path, validateYamlContent(err))

		if !validYaml {
			continue
		}
	}
}

// Validation functions
func validateTemplatesDir(templatesPath string) error {
	if fi, err := os.Stat(templatesPath); err != nil {
		return errors.New("directory not found")
	} else if err == nil && !fi.IsDir() {
		return errors.New("not a directory")
	}
	return nil
}

func validateAllowedExtension(fileName string) error {
	ext := filepath.Ext(fileName)
	validExtensions := []string{".yaml", ".yml", ".tpl", ".txt"}

	for _, b := range validExtensions {
		if b == ext {
			return nil
		}
	}

	return fmt.Errorf("file extension '%s' not valid. Valid extensions are .yaml, .yml, .tpl, or .txt", ext)
}

func validateYamlContent(err error) error {
	if err != nil {
		return fmt.Errorf("unable to parse YAML\n\t%s", err)
	}
	return nil
}

// K8sYamlStruct stubs a Kubernetes YAML file.
// Need to access for now to Namespace only
type K8sYamlStruct struct {
	Metadata struct {
		Namespace string
	}
}
<commit_msg>Clean code: delete commented code<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rules

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"

	"github.com/ghodss/yaml"
	"k8s.io/helm/pkg/chartutil"
	"k8s.io/helm/pkg/engine"
	"k8s.io/helm/pkg/lint/support"
	cpb "k8s.io/helm/pkg/proto/hapi/chart"
	"k8s.io/helm/pkg/timeconv"
	tversion "k8s.io/helm/pkg/version"
)

// Templates lints the templates in the Linter.
func Templates(linter *support.Linter, values []byte, namespace string, strict bool) {
	path := "templates/"
	templatesPath := filepath.Join(linter.ChartDir, path)

	templatesDirExist := linter.RunLinterRule(support.WarningSev, path, validateTemplatesDir(templatesPath))

	// Templates directory is optional for now
	if !templatesDirExist {
		return
	}

	// Load chart and parse templates, based on tiller/release_server
	chart, err := chartutil.Load(linter.ChartDir)

	chartLoaded := linter.RunLinterRule(support.ErrorSev, path, err)

	if !chartLoaded {
		return
	}

	options := chartutil.ReleaseOptions{Name: "testRelease", Time: timeconv.Now(), Namespace: namespace}
	caps := &chartutil.Capabilities{
		APIVersions:   chartutil.DefaultVersionSet,
		KubeVersion:   chartutil.DefaultKubeVersion,
		TillerVersion: tversion.GetVersionProto(),
	}
	cvals, err := chartutil.CoalesceValues(chart, &cpb.Config{Raw: string(values)})
	if err != nil {
		return
	}
	// convert our values back into config
	yvals, err := cvals.YAML()
	if err != nil {
		return
	}
	cc := &cpb.Config{Raw: yvals}
	valuesToRender, err := chartutil.ToRenderValuesCaps(chart, cc, options, caps)
	if err != nil {
		// FIXME: This seems to generate a duplicate, but I can't find where the first
		// error is coming from.
		//linter.RunLinterRule(support.ErrorSev, err)
		return
	}
	e := engine.New()
	e.LintMode = true
	if strict {
		e.Strict = true
	}
	renderedContentMap, err := e.Render(chart, valuesToRender)

	renderOk := linter.RunLinterRule(support.ErrorSev, path, err)

	if !renderOk {
		return
	}

	/* Iterate over all the templates to check:
	- It is a .yaml file
	- All the values in the template file is defined
	- {{}} include | quote
	- Generated content is a valid Yaml file
	- Metadata.Namespace is not set
	*/
	for _, template := range chart.Templates {
		fileName, _ := template.Name, template.Data
		path = fileName

		linter.RunLinterRule(support.WarningSev, path, validateAllowedExtension(fileName))

		// We only apply the following lint rules to yaml files
		if filepath.Ext(fileName) != ".yaml" || filepath.Ext(fileName) == ".yml" {
			continue
		}

		renderedContent := renderedContentMap[filepath.Join(chart.GetMetadata().Name, fileName)]
		var yamlStruct K8sYamlStruct
		// Even though K8sYamlStruct only defines Metadata namespace, an error in any other
		// key will be raised as well
		err := yaml.Unmarshal([]byte(renderedContent), &yamlStruct)

		validYaml := linter.RunLinterRule(support.ErrorSev, path, validateYamlContent(err))

		if !validYaml {
			continue
		}
	}
}

// Validation functions
func validateTemplatesDir(templatesPath string) error {
	if fi, err := os.Stat(templatesPath); err != nil {
		return errors.New("directory not found")
	} else if err == nil && !fi.IsDir() {
		return errors.New("not a directory")
	}
	return nil
}

func validateAllowedExtension(fileName string) error {
	ext := filepath.Ext(fileName)
	validExtensions := []string{".yaml", ".yml", ".tpl", ".txt"}

	for _, b := range validExtensions {
		if b == ext {
			return nil
		}
	}

	return fmt.Errorf("file extension '%s' not valid. Valid extensions are .yaml, .yml, .tpl, or .txt", ext)
}

func validateYamlContent(err error) error {
	if err != nil {
		return fmt.Errorf("unable to parse YAML\n\t%s", err)
	}
	return nil
}

// K8sYamlStruct stubs a Kubernetes YAML file.
// Need to access for now to Namespace only
type K8sYamlStruct struct {
	Metadata struct {
		Namespace string
	}
}
<|endoftext|>


<commit_before>package localkubectl
<commit_msg>added basic docker wrapper<commit_after>package localkubectl

import (
	"fmt"

	docker "github.com/fsouza/go-dockerclient"
)

const (
	// LocalkubeLabel is the label that identifies localkube containers
	LocalkubeLabel = "rsprd.com/name=localkube"
)

// Docker provides a wrapper around the Docker client for easy control of localkube.
type Docker struct {
	*docker.Client
}

// NewDocker returns a localkube Docker client from a created *docker.Client
func NewDocker(client *docker.Client) (*Docker, error) {
	_, err := client.Version()
	if err != nil {
		return nil, fmt.Errorf("Unable to establish connection with Docker daemon: %v", err)
	}

	return &Docker{
		Client: client,
	}, nil
}

// NewDockerFromEnv creates a new Docker client using environment clues.
func NewDockerFromEnv() (*Docker, error) {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return nil, fmt.Errorf("Could not create Docker client: %v", err)
	}

	return NewDocker(client)
}

// localkubeCtrs lists the containers associated with localkube. If running is true, only running containers will be listed.
func (d *Docker) localkubeCtrs(runningOnly bool) ([]docker.APIContainers, error) {
	ctrs, err := d.ListContainers(docker.ListContainersOptions{
		All: !runningOnly,
		Filters: map[string][]string{
			"label": {LocalkubeLabel},
		},
	})

	if err != nil {
		return nil, fmt.Errorf("Could not list containers: %v", err)
	}
	return ctrs, nil
}
<|endoftext|>


<commit_before>package notifications

import (
	"encoding/base64"
	"fmt"
	"github.com/spf13/cobra"
	"net/smtp"
	"os"
	"strings"
	"time"

	t "github.com/containrrr/watchtower/pkg/types"
	log "github.com/sirupsen/logrus"
	"strconv"
)

const (
	emailType = "email"
)

// Implements Notifier, logrus.Hook
// The default logrus email integration would have several issues:
// - It would send one email per log output
// - It would only send errors
// We work around that by holding on to log entries until the update cycle is done.
type emailTypeNotifier struct {
	From, To                           string
	Server, User, Password, SubjectTag string
	Port                               int
	tlsSkipVerify                      bool
	entries                            []*log.Entry
	logLevels                          []log.Level
	delay                              time.Duration
}

func newEmailNotifier(c *cobra.Command, acceptedLogLevels []log.Level) t.Notifier {
	flags := c.PersistentFlags()

	from, _ := flags.GetString("notification-email-from")
	to, _ := flags.GetString("notification-email-to")
	server, _ := flags.GetString("notification-email-server")
	user, _ := flags.GetString("notification-email-server-user")
	password, _ := flags.GetString("notification-email-server-password")
	port, _ := flags.GetInt("notification-email-server-port")
	tlsSkipVerify, _ := flags.GetBool("notification-email-server-tls-skip-verify")
	delay, _ := flags.GetInt("notification-email-delay")
	subjecttag, _ := flags.GetString("notification-email-subjecttag")

	n := &emailTypeNotifier{
		From:          from,
		To:            to,
		Server:        server,
		User:          user,
		Password:      password,
		Port:          port,
		tlsSkipVerify: tlsSkipVerify,
		logLevels:     acceptedLogLevels,
		delay:         time.Duration(delay) * time.Second,
		SubjectTag:    subjecttag,
	}

	log.AddHook(n)

	return n
}

func (e *emailTypeNotifier) buildMessage(entries []*log.Entry) []byte {
	var emailSubject string

	if e.SubjectTag == "" {
		emailSubject = "Watchtower updates"
	} else {
		emailSubject = e.SubjectTag + " Watchtower updates"
	}
	if hostname, err := os.Hostname(); err == nil {
		emailSubject += " on " + hostname
	}
	body := ""
	for _, entry := range entries {
		body += entry.Time.Format("2006-01-02 15:04:05") + " (" + entry.Level.String() + "): " + entry.Message + "\r\n"
		// We don't use fields in watchtower, so don't bother sending them.
	}

	t := time.Now()

	header := make(map[string]string)
	header["From"] = e.From
	header["To"] = e.To
	header["Subject"] = emailSubject
	header["Date"] = t.Format(time.RFC1123Z)
	header["MIME-Version"] = "1.0"
	header["Content-Type"] = "text/plain; charset=\"utf-8\""
	header["Content-Transfer-Encoding"] = "base64"

	message := ""
	for k, v := range header {
		message += fmt.Sprintf("%s: %s\r\n", k, v)
	}

	encodedBody := base64.StdEncoding.EncodeToString([]byte(body))
	//RFC 2045 base64 encoding demands line no longer than 76 characters.
	for _, line := range SplitSubN(encodedBody, 76) {
		message += "\r\n" + line
	}

	return []byte(message)
}

func (e *emailTypeNotifier) sendEntries(entries []*log.Entry) {
	// Do the sending in a separate goroutine so we don't block the main process.
	msg := e.buildMessage(entries)
	go func() {
		var auth smtp.Auth
		if e.User != "" {
			auth = smtp.PlainAuth("", e.User, e.Password, e.Server)
		}
		err := SendMail(e.Server+":"+strconv.Itoa(e.Port), e.tlsSkipVerify, auth, e.From, strings.Split(e.To, ","), msg)
		if err != nil {
			// Use fmt so it doesn't trigger another email.
			fmt.Println("Failed to send notification email: ", err)
		}
	}()
}

func (e *emailTypeNotifier) StartNotification() {
	if e.entries == nil {
		e.entries = make([]*log.Entry, 0, 10)
	}
}

func (e *emailTypeNotifier) SendNotification() {
	if e.entries == nil || len(e.entries) <= 0 {
		return
	}

	if e.delay > 0 {
		time.Sleep(e.delay)
	}

	e.sendEntries(e.entries)
	e.entries = nil
}

func (e *emailTypeNotifier) Levels() []log.Level {
	return e.logLevels
}

func (e *emailTypeNotifier) Fire(entry *log.Entry) error {
	if e.entries != nil {
		e.entries = append(e.entries, entry)
	} else {
		// Log output generated outside a cycle is sent immediately.
		e.sendEntries([]*log.Entry{entry})
	}
	return nil
}
<commit_msg>fix(email): always use configured delay for notifications (#536)<commit_after>package notifications

import (
	"encoding/base64"
	"fmt"
	"github.com/spf13/cobra"
	"net/smtp"
	"os"
	"strings"
	"time"

	t "github.com/containrrr/watchtower/pkg/types"
	log "github.com/sirupsen/logrus"
	"strconv"
)

const (
	emailType = "email"
)

// Implements Notifier, logrus.Hook
// The default logrus email integration would have several issues:
// - It would send one email per log output
// - It would only send errors
// We work around that by holding on to log entries until the update cycle is done.
type emailTypeNotifier struct {
	From, To                           string
	Server, User, Password, SubjectTag string
	Port                               int
	tlsSkipVerify                      bool
	entries                            []*log.Entry
	logLevels                          []log.Level
	delay                              time.Duration
}

func newEmailNotifier(c *cobra.Command, acceptedLogLevels []log.Level) t.Notifier {
	flags := c.PersistentFlags()

	from, _ := flags.GetString("notification-email-from")
	to, _ := flags.GetString("notification-email-to")
	server, _ := flags.GetString("notification-email-server")
	user, _ := flags.GetString("notification-email-server-user")
	password, _ := flags.GetString("notification-email-server-password")
	port, _ := flags.GetInt("notification-email-server-port")
	tlsSkipVerify, _ := flags.GetBool("notification-email-server-tls-skip-verify")
	delay, _ := flags.GetInt("notification-email-delay")
	subjecttag, _ := flags.GetString("notification-email-subjecttag")

	n := &emailTypeNotifier{
		From:          from,
		To:            to,
		Server:        server,
		User:          user,
		Password:      password,
		Port:          port,
		tlsSkipVerify: tlsSkipVerify,
		logLevels:     acceptedLogLevels,
		delay:         time.Duration(delay) * time.Second,
		SubjectTag:    subjecttag,
	}

	log.AddHook(n)

	return n
}

func (e *emailTypeNotifier) buildMessage(entries []*log.Entry) []byte {
	var emailSubject string

	if e.SubjectTag == "" {
		emailSubject = "Watchtower updates"
	} else {
		emailSubject = e.SubjectTag + " Watchtower updates"
	}
	if hostname, err := os.Hostname(); err == nil {
		emailSubject += " on " + hostname
	}
	body := ""
	for _, entry := range entries {
		body += entry.Time.Format("2006-01-02 15:04:05") + " (" + entry.Level.String() + "): " + entry.Message + "\r\n"
		// We don't use fields in watchtower, so don't bother sending them.
	}

	t := time.Now()

	header := make(map[string]string)
	header["From"] = e.From
	header["To"] = e.To
	header["Subject"] = emailSubject
	header["Date"] = t.Format(time.RFC1123Z)
	header["MIME-Version"] = "1.0"
	header["Content-Type"] = "text/plain; charset=\"utf-8\""
	header["Content-Transfer-Encoding"] = "base64"

	message := ""
	for k, v := range header {
		message += fmt.Sprintf("%s: %s\r\n", k, v)
	}

	encodedBody := base64.StdEncoding.EncodeToString([]byte(body))
	//RFC 2045 base64 encoding demands line no longer than 76 characters.
	for _, line := range SplitSubN(encodedBody, 76) {
		message += "\r\n" + line
	}

	return []byte(message)
}

func (e *emailTypeNotifier) sendEntries(entries []*log.Entry) {
	// Do the sending in a separate goroutine so we don't block the main process.
	msg := e.buildMessage(entries)
	go func() {
		if e.delay > 0 {
			time.Sleep(e.delay)
		}

		var auth smtp.Auth
		if e.User != "" {
			auth = smtp.PlainAuth("", e.User, e.Password, e.Server)
		}
		err := SendMail(e.Server+":"+strconv.Itoa(e.Port), e.tlsSkipVerify, auth, e.From, strings.Split(e.To, ","), msg)
		if err != nil {
			// Use fmt so it doesn't trigger another email.
			fmt.Println("Failed to send notification email: ", err)
		}
	}()
}

func (e *emailTypeNotifier) StartNotification() {
	if e.entries == nil {
		e.entries = make([]*log.Entry, 0, 10)
	}
}

func (e *emailTypeNotifier) SendNotification() {
	if e.entries == nil || len(e.entries) <= 0 {
		return
	}

	e.sendEntries(e.entries)
	e.entries = nil
}

func (e *emailTypeNotifier) Levels() []log.Level {
	return e.logLevels
}

func (e *emailTypeNotifier) Fire(entry *log.Entry) error {
	if e.entries != nil {
		e.entries = append(e.entries, entry)
	} else {
		e.sendEntries([]*log.Entry{entry})
	}
	return nil
}
<|endoftext|>


<commit_before>package social

import (
	"encoding/json"
	"fmt"
	"net/http"
	"regexp"

	"github.com/grafana/grafana/pkg/models"

	"golang.org/x/oauth2"
)

type SocialGithub struct {
	*SocialBase
	allowedDomains       []string
	allowedOrganizations []string
	apiUrl               string
	allowSignup          bool
	teamIds              []int
}

var (
	ErrMissingTeamMembership         = &Error{"User not a member of one of the required teams"}
	ErrMissingOrganizationMembership = &Error{"User not a member of one of the required organizations"}
)

func (s *SocialGithub) Type() int {
	return int(models.GITHUB)
}

func (s *SocialGithub) IsEmailAllowed(email string) bool {
	return isEmailAllowed(email, s.allowedDomains)
}

func (s *SocialGithub) IsSignupAllowed() bool {
	return s.allowSignup
}

func (s *SocialGithub) IsTeamMember(client *http.Client) bool {
	if len(s.teamIds) == 0 {
		return true
	}

	teamMemberships, err := s.FetchTeamMemberships(client)
	if err != nil {
		return false
	}

	for _, teamId := range s.teamIds {
		for _, membershipId := range teamMemberships {
			if teamId == membershipId {
				return true
			}
		}
	}

	return false
}

func (s *SocialGithub) IsOrganizationMember(client *http.Client, organizationsUrl string) bool {
	if len(s.allowedOrganizations) == 0 {
		return true
	}

	organizations, err := s.FetchOrganizations(client, organizationsUrl)
	if err != nil {
		return false
	}

	for _, allowedOrganization := range s.allowedOrganizations {
		for _, organization := range organizations {
			if organization == allowedOrganization {
				return true
			}
		}
	}

	return false
}

func (s *SocialGithub) FetchPrivateEmail(client *http.Client) (string, error) {
	type Record struct {
		Email    string `json:"email"`
		Primary  bool   `json:"primary"`
		Verified bool   `json:"verified"`
	}

	response, err := HttpGet(client, fmt.Sprintf(s.apiUrl+"/emails"))
	if err != nil {
		return "", fmt.Errorf("Error getting email address: %s", err)
	}

	var records []Record

	err = json.Unmarshal(response.Body, &records)
	if err != nil {
		return "", fmt.Errorf("Error getting email address: %s", err)
	}

	var email = ""
	for _, record := range records {
		if record.Primary {
			email = record.Email
		}
	}

	return email, nil
}

func (s *SocialGithub) FetchTeamMemberships(client *http.Client) ([]int, error) {
	type Record struct {
		Id int `json:"id"`
	}

	url := fmt.Sprintf(s.apiUrl + "/teams?per_page=100")
	hasMore := true
	ids := make([]int, 0)

	for hasMore {

		response, err := HttpGet(client, url)
		if err != nil {
			return nil, fmt.Errorf("Error getting team memberships: %s", err)
		}

		var records []Record

		err = json.Unmarshal(response.Body, &records)
		if err != nil {
			return nil, fmt.Errorf("Error getting team memberships: %s", err)
		}

		newRecords := len(records)
		existingRecords := len(ids)
		tempIds := make([]int, (newRecords + existingRecords))
		copy(tempIds, ids)
		ids = tempIds

		for i, record := range records {
			ids[i] = record.Id
		}

		url, hasMore = s.HasMoreRecords(response.Headers)
	}

	return ids, nil
}

func (s *SocialGithub) HasMoreRecords(headers http.Header) (string, bool) {

	value, exists := headers["Link"]
	if !exists {
		return "", false
	}

	pattern := regexp.MustCompile(`<([^>]+)>; rel="next"`)
	matches := pattern.FindStringSubmatch(value[0])

	if matches == nil {
		return "", false
	}

	url := matches[1]

	return url, true

}

func (s *SocialGithub) FetchOrganizations(client *http.Client, organizationsUrl string) ([]string, error) {
	type Record struct {
		Login string `json:"login"`
	}

	response, err := HttpGet(client, organizationsUrl)
	if err != nil {
		return nil, fmt.Errorf("Error getting organizations: %s", err)
	}

	var records []Record

	err = json.Unmarshal(response.Body, &records)
	if err != nil {
		return nil, fmt.Errorf("Error getting organizations: %s", err)
	}

	var logins = make([]string, len(records))
	for i, record := range records {
		logins[i] = record.Login
	}

	return logins, nil
}

func (s *SocialGithub) UserInfo(client *http.Client, token *oauth2.Token) (*BasicUserInfo, error) {

	var data struct {
		Id               int    `json:"id"`
		Login            string `json:"login"`
		Email            string `json:"email"`
		OrganizationsUrl string `json:"organizations_url"`
	}

	response, err := HttpGet(client, s.apiUrl)
	if err != nil {
		return nil, fmt.Errorf("Error getting user info: %s", err)
	}

	err = json.Unmarshal(response.Body, &data)
	if err != nil {
		return nil, fmt.Errorf("Error getting user info: %s", err)
	}

	userInfo := &BasicUserInfo{
		Name:  data.Login,
		Login: data.Login,
		Email: data.Email,
	}

	if !s.IsTeamMember(client) {
		return nil, ErrMissingTeamMembership
	}

	if !s.IsOrganizationMember(client, data.OrganizationsUrl) {
		return nil, ErrMissingOrganizationMembership
	}

	if userInfo.Email == "" {
		userInfo.Email, err = s.FetchPrivateEmail(client)
		if err != nil {
			return nil, err
		}
	}

	return userInfo, nil
}
<commit_msg>Fix #10823 (#10851)<commit_after>package social

import (
	"encoding/json"
	"fmt"
	"net/http"
	"regexp"

	"github.com/grafana/grafana/pkg/models"

	"golang.org/x/oauth2"
)

type SocialGithub struct {
	*SocialBase
	allowedDomains       []string
	allowedOrganizations []string
	apiUrl               string
	allowSignup          bool
	teamIds              []int
}

var (
	ErrMissingTeamMembership         = &Error{"User not a member of one of the required teams"}
	ErrMissingOrganizationMembership = &Error{"User not a member of one of the required organizations"}
)

func (s *SocialGithub) Type() int {
	return int(models.GITHUB)
}

func (s *SocialGithub) IsEmailAllowed(email string) bool {
	return isEmailAllowed(email, s.allowedDomains)
}

func (s *SocialGithub) IsSignupAllowed() bool {
	return s.allowSignup
}

func (s *SocialGithub) IsTeamMember(client *http.Client) bool {
	if len(s.teamIds) == 0 {
		return true
	}

	teamMemberships, err := s.FetchTeamMemberships(client)
	if err != nil {
		return false
	}

	for _, teamId := range s.teamIds {
		for _, membershipId := range teamMemberships {
			if teamId == membershipId {
				return true
			}
		}
	}

	return false
}

func (s *SocialGithub) IsOrganizationMember(client *http.Client, organizationsUrl string) bool {
	if len(s.allowedOrganizations) == 0 {
		return true
	}

	organizations, err := s.FetchOrganizations(client, organizationsUrl)
	if err != nil {
		return false
	}

	for _, allowedOrganization := range s.allowedOrganizations {
		for _, organization := range organizations {
			if organization == allowedOrganization {
				return true
			}
		}
	}

	return false
}

func (s *SocialGithub) FetchPrivateEmail(client *http.Client) (string, error) {
	type Record struct {
		Email    string `json:"email"`
		Primary  bool   `json:"primary"`
		Verified bool   `json:"verified"`
	}

	response, err := HttpGet(client, fmt.Sprintf(s.apiUrl+"/emails"))
	if err != nil {
		return "", fmt.Errorf("Error getting email address: %s", err)
	}

	var records []Record

	err = json.Unmarshal(response.Body, &records)
	if err != nil {
		return "", fmt.Errorf("Error getting email address: %s", err)
	}

	var email = ""
	for _, record := range records {
		if record.Primary {
			email = record.Email
		}
	}

	return email, nil
}

func (s *SocialGithub) FetchTeamMemberships(client *http.Client) ([]int, error) {
	type Record struct {
		Id int `json:"id"`
	}

	url := fmt.Sprintf(s.apiUrl + "/teams?per_page=100")
	hasMore := true
	ids := make([]int, 0)

	for hasMore {

		response, err := HttpGet(client, url)
		if err != nil {
			return nil, fmt.Errorf("Error getting team memberships: %s", err)
		}

		var records []Record

		err = json.Unmarshal(response.Body, &records)
		if err != nil {
			return nil, fmt.Errorf("Error getting team memberships: %s", err)
		}

		newRecords := len(records)
		existingRecords := len(ids)
		tempIds := make([]int, (newRecords + existingRecords))
		copy(tempIds, ids)
		ids = tempIds

		for i, record := range records {
			ids[i] = record.Id
		}

		url, hasMore = s.HasMoreRecords(response.Headers)
	}

	return ids, nil
}

func (s *SocialGithub) HasMoreRecords(headers http.Header) (string, bool) {

	value, exists := headers["Link"]
	if !exists {
		return "", false
	}

	pattern := regexp.MustCompile(`<([^>]+)>; rel="next"`)
	matches := pattern.FindStringSubmatch(value[0])

	if matches == nil {
		return "", false
	}

	url := matches[1]

	return url, true

}

func (s *SocialGithub) FetchOrganizations(client *http.Client, organizationsUrl string) ([]string, error) {
	type Record struct {
		Login string `json:"login"`
	}

	response, err := HttpGet(client, organizationsUrl)
	if err != nil {
		return nil, fmt.Errorf("Error getting organizations: %s", err)
	}

	var records []Record

	err = json.Unmarshal(response.Body, &records)
	if err != nil {
		return nil, fmt.Errorf("Error getting organizations: %s", err)
	}

	var logins = make([]string, len(records))
	for i, record := range records {
		logins[i] = record.Login
	}

	return logins, nil
}

func (s *SocialGithub) UserInfo(client *http.Client, token *oauth2.Token) (*BasicUserInfo, error) {

	var data struct {
		Id               int    `json:"id"`
		Login            string `json:"login"`
		Email            string `json:"email"`
		OrganizationsUrl string `json:"organizations_url"`
	}

	response, err := HttpGet(client, s.apiUrl)
	if err != nil {
		return nil, fmt.Errorf("Error getting user info: %s", err)
	}

	err = json.Unmarshal(response.Body, &data)
	if err != nil {
		return nil, fmt.Errorf("Error getting user info: %s", err)
	}
	data.OrganizationsUrl = s.apiUrl + "/user/orgs"
	userInfo := &BasicUserInfo{
		Name:  data.Login,
		Login: data.Login,
		Email: data.Email,
	}

	if !s.IsTeamMember(client) {
		return nil, ErrMissingTeamMembership
	}

	if !s.IsOrganizationMember(client, data.OrganizationsUrl) {
		return nil, ErrMissingOrganizationMembership
	}

	if userInfo.Email == "" {
		userInfo.Email, err = s.FetchPrivateEmail(client)
		if err != nil {
			return nil, err
		}
	}

	return userInfo, nil
}
<|endoftext|>