<commit_before>package server

import (
	"context"
	"fmt"
	"path"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/pkg/tracing"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
		"Unschedulable":    true,
	}

	zero     int32 // used to turn down RCs in scaleDownWorkersForPipeline
	falseVal bool  // used to delete RCs in deletePipelineResources and restartPipeline()
)

type eventType int

const (
	writeEv eventType = iota
	deleteEv
)

type pipelineEvent struct {
	eventType
	pipeline string

	// etcdVer and etcdRev record the etcd version and key revision at which a
	// write/delete was observed. These are recorded in traces and are useful for
	// debugging (e.g. if two consecutive spans have non-consecutive versions,
	// that would indicate a concurrent write). These will not be set for events
	// created by pollPipelines.
	etcdVer int64
	etcdRev int64
}

type ppsMaster struct {
	// The PPS APIServer that owns this struct
	a *apiServer

	// masterClient is a pachyderm client containing a context that is cancelled if
	// the current pps master loses its master status
	// Note: 'masterClient' is unauthenticated.  This uses the PPS token (via
	// a.sudo()) to authenticate requests as needed.
	masterClient *client.APIClient

	// fields for monitorPipeline goros, monitorCrashingPipeline goros, etc.
	monitorCancelsMu       sync.Mutex
	monitorCancels         map[string]func() // protected by monitorCancelsMu
	crashingMonitorCancels map[string]func() // also protected by monitorCancelsMu

	// fields for the pollPipelines and pollPipelinePods goros
	pollPipelinesMu sync.Mutex
	pollCancel      func() // protected by pollPipelinesMu
	pollPodsCancel  func() // protected by pollPipelinesMu
	pollEtcdCancel  func() // protected by pollPipelinesMu

	// channel through which pipeline events are passed
	eventCh chan *pipelineEvent
}

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	m := &ppsMaster{
		a:                      a,
		monitorCancels:         make(map[string]func()),
		crashingMonitorCancels: make(map[string]func()),
		eventCh:                make(chan *pipelineEvent, 1), // avoid thrashing
	}

	masterLock := dlock.NewDLock(a.env.GetEtcdClient(), path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)

		log.Infof("PPS master: launching master process")
		m.masterClient = a.env.GetPachClient(ctx)
		m.run()
		return errors.Wrapf(ctx.Err(), "ppsMaster.Run() exited unexpectedly")
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		log.Errorf("PPS master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
	panic("internal error: PPS master has somehow exited. Restarting pod...")
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_FAILURE, reason)
}

func (a *apiServer) setPipelineCrashing(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_CRASHING, reason)
}

func (m *ppsMaster) run() {
	// close m.eventCh after all cancels have returned and therefore all pollers
	// (which are what write to m.eventCh) have exited
	defer close(m.eventCh)
	defer m.cancelAllMonitorsAndCrashingMonitors()
	// start pollers in the background--cancel functions ensure poll/monitor
	// goroutines all definitely stop (either because cancelXYZ returns or because
	// the binary panics)
	m.startPipelinePoller()
	defer m.cancelPipelinePoller()
	m.startPipelinePodsPoller()
	defer m.cancelPipelinePodsPoller()
	m.startPipelineEtcdPoller()
	defer m.cancelPipelineEtcdPoller()

	masterCtx := m.masterClient.Ctx()
eventLoop:
	for {
		select {
		case e := <-m.eventCh:
			switch e.eventType {
			case writeEv:
				// Create/Modify/Delete pipeline resources as needed per new state
				if err := m.step(e.pipeline, e.etcdVer, e.etcdRev); err != nil {
					log.Errorf("PPS master: could not update resources for pipeline %q: %v",
						e.pipeline, err)
				}
			case deleteEv:
				// TODO(msteffen) trace this call
				if err := m.deletePipelineResources(e.pipeline); err != nil {
					log.Errorf("PPS master: could not delete resources for pipeline %q: %v",
						e.pipeline, err)
				}
			}
		case <-masterCtx.Done():
			break eventLoop
		}
	}
}

func (m *ppsMaster) deletePipelineResources(pipelineName string) (retErr error) {
	log.Infof("PPS master: deleting resources for pipeline %q", pipelineName)
	span, ctx := tracing.AddSpanToAnyExisting(m.masterClient.Ctx(),
		"/pps.Master/DeletePipelineResources", "pipeline", pipelineName)
	defer func() {
		tracing.TagAnySpan(ctx, "err", retErr)
		tracing.FinishAnySpan(span)
	}()

	// Cancel any running monitorPipeline call
	m.cancelMonitor(pipelineName)
	// Same for cancelCrashingMonitor
	m.cancelCrashingMonitor(pipelineName)

	kubeClient := m.a.env.GetKubeClient()
	namespace := m.a.namespace
	// Delete any services associated with op.pipeline
	selector := fmt.Sprintf("%s=%s", pipelineNameLabel, pipelineName)
	opts := &metav1.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	services, err := kubeClient.CoreV1().Services(namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list services")
	}
	for _, service := range services.Items {
		if err := kubeClient.CoreV1().Services(namespace).Delete(service.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete service %q", service.Name)
			}
		}
	}
	rcs, err := kubeClient.CoreV1().ReplicationControllers(namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list RCs")
	}
	for _, rc := range rcs.Items {
		if err := kubeClient.CoreV1().ReplicationControllers(namespace).Delete(rc.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete RC %q: %v", rc.Name)
			}
		}
	}
	// delete any secrets associated with the pipeline
	secrets, err := kubeClient.CoreV1().Secrets(a.namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list secrets")
	}
	for _, secret := range secrets.Items {
		if err := kubeClient.CoreV1().Secrets(a.namespace).Delete(secret.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete secret %q", secret.Name)
			}
		}
	}
	return nil
}

// setPipelineState is a PPS-master-specific helper that wraps
// ppsutil.SetPipelineState in a trace
func (a *apiServer) setPipelineState(ctx context.Context, pipeline string, state pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/SetPipelineState", "pipeline", pipeline, "new-state", state)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, nil, state, reason)
}

// transitionPipelineState is similar to setPipelineState, except that it sets
// 'from' and logs a different trace
func (a *apiServer) transitionPipelineState(ctx context.Context, pipeline string, from []pps.PipelineState, to pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/TransitionPipelineState", "pipeline", pipeline,
		"from-state", from, "to-state", to)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, from, to, reason)
}
<commit_msg>Incorporate spout auth changes<commit_after>package server

import (
	"context"
	"fmt"
	"path"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/pkg/tracing"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
		"Unschedulable":    true,
	}

	zero     int32 // used to turn down RCs in scaleDownWorkersForPipeline
	falseVal bool  // used to delete RCs in deletePipelineResources and restartPipeline()
)

type eventType int

const (
	writeEv eventType = iota
	deleteEv
)

type pipelineEvent struct {
	eventType
	pipeline string

	// etcdVer and etcdRev record the etcd version and key revision at which a
	// write/delete was observed. These are recorded in traces and are useful for
	// debugging (e.g. if two consecutive spans have non-consecutive versions,
	// that would indicate a concurrent write). These will not be set for events
	// created by pollPipelines.
	etcdVer int64
	etcdRev int64
}

type ppsMaster struct {
	// The PPS APIServer that owns this struct
	a *apiServer

	// masterClient is a pachyderm client containing a context that is cancelled if
	// the current pps master loses its master status
	// Note: 'masterClient' is unauthenticated.  This uses the PPS token (via
	// a.sudo()) to authenticate requests as needed.
	masterClient *client.APIClient

	// fields for monitorPipeline goros, monitorCrashingPipeline goros, etc.
	monitorCancelsMu       sync.Mutex
	monitorCancels         map[string]func() // protected by monitorCancelsMu
	crashingMonitorCancels map[string]func() // also protected by monitorCancelsMu

	// fields for the pollPipelines and pollPipelinePods goros
	pollPipelinesMu sync.Mutex
	pollCancel      func() // protected by pollPipelinesMu
	pollPodsCancel  func() // protected by pollPipelinesMu
	pollEtcdCancel  func() // protected by pollPipelinesMu

	// channel through which pipeline events are passed
	eventCh chan *pipelineEvent
}

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	m := &ppsMaster{
		a:                      a,
		monitorCancels:         make(map[string]func()),
		crashingMonitorCancels: make(map[string]func()),
		eventCh:                make(chan *pipelineEvent, 1), // avoid thrashing
	}

	masterLock := dlock.NewDLock(a.env.GetEtcdClient(), path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)

		log.Infof("PPS master: launching master process")
		m.masterClient = a.env.GetPachClient(ctx)
		m.run()
		return errors.Wrapf(ctx.Err(), "ppsMaster.Run() exited unexpectedly")
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		log.Errorf("PPS master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
	panic("internal error: PPS master has somehow exited. Restarting pod...")
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_FAILURE, reason)
}

func (a *apiServer) setPipelineCrashing(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_CRASHING, reason)
}

func (m *ppsMaster) run() {
	// close m.eventCh after all cancels have returned and therefore all pollers
	// (which are what write to m.eventCh) have exited
	defer close(m.eventCh)
	defer m.cancelAllMonitorsAndCrashingMonitors()
	// start pollers in the background--cancel functions ensure poll/monitor
	// goroutines all definitely stop (either because cancelXYZ returns or because
	// the binary panics)
	m.startPipelinePoller()
	defer m.cancelPipelinePoller()
	m.startPipelinePodsPoller()
	defer m.cancelPipelinePodsPoller()
	m.startPipelineEtcdPoller()
	defer m.cancelPipelineEtcdPoller()

	masterCtx := m.masterClient.Ctx()
eventLoop:
	for {
		select {
		case e := <-m.eventCh:
			switch e.eventType {
			case writeEv:
				// Create/Modify/Delete pipeline resources as needed per new state
				if err := m.step(e.pipeline, e.etcdVer, e.etcdRev); err != nil {
					log.Errorf("PPS master: could not update resources for pipeline %q: %v",
						e.pipeline, err)
				}
			case deleteEv:
				// TODO(msteffen) trace this call
				if err := m.deletePipelineResources(e.pipeline); err != nil {
					log.Errorf("PPS master: could not delete resources for pipeline %q: %v",
						e.pipeline, err)
				}
			}
		case <-masterCtx.Done():
			break eventLoop
		}
	}
}

func (m *ppsMaster) deletePipelineResources(pipelineName string) (retErr error) {
	log.Infof("PPS master: deleting resources for pipeline %q", pipelineName)
	span, ctx := tracing.AddSpanToAnyExisting(m.masterClient.Ctx(),
		"/pps.Master/DeletePipelineResources", "pipeline", pipelineName)
	defer func() {
		tracing.TagAnySpan(ctx, "err", retErr)
		tracing.FinishAnySpan(span)
	}()

	// Cancel any running monitorPipeline call
	m.cancelMonitor(pipelineName)
	// Same for cancelCrashingMonitor
	m.cancelCrashingMonitor(pipelineName)

	kubeClient := m.a.env.GetKubeClient()
	namespace := m.a.namespace

	// Delete any services associated with op.pipeline
	selector := fmt.Sprintf("%s=%s", pipelineNameLabel, pipelineName)
	opts := &metav1.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	services, err := kubeClient.CoreV1().Services(namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list services")
	}
	for _, service := range services.Items {
		if err := kubeClient.CoreV1().Services(namespace).Delete(service.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete service %q", service.Name)
			}
		}
	}

	// Delete any secrets associated with op.pipeline
	secrets, err := kubeClient.CoreV1().Secrets(namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list secrets")
	}
	for _, secret := range secrets.Items {
		if err := kubeClient.CoreV1().Secrets(namespace).Delete(secret.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete secret %q", secret.Name)
			}
		}
	}

	// Finally, delete op.pipeline's RC, which will cause pollPipelines to stop
	// polling it.
	rcs, err := kubeClient.CoreV1().ReplicationControllers(namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list RCs")
	}
	for _, rc := range rcs.Items {
		if err := kubeClient.CoreV1().ReplicationControllers(namespace).Delete(rc.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete RC %q: %v", rc.Name)
			}
		}
	}

	return nil
}

// setPipelineState is a PPS-master-specific helper that wraps
// ppsutil.SetPipelineState in a trace
func (a *apiServer) setPipelineState(ctx context.Context, pipeline string, state pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/SetPipelineState", "pipeline", pipeline, "new-state", state)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, nil, state, reason)
}

// transitionPipelineState is similar to setPipelineState, except that it sets
// 'from' and logs a different trace
func (a *apiServer) transitionPipelineState(ctx context.Context, pipeline string, from []pps.PipelineState, to pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/TransitionPipelineState", "pipeline", pipeline,
		"from-state", from, "to-state", to)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, from, to, reason)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/barnslig/torture/lib/elastic"
	"github.com/dustin/go-humanize"
	"regexp"
	"strings"
)

type ElasticSearch struct {
	url string
}

type hash map[string]interface{}

func CreateElasticSearch(host string) (es *ElasticSearch, err error) {
	es = &ElasticSearch{url: host}
	return
}

func (es *ElasticSearch) Search(stmt Statement, perPage int, page int) (result elastic.Result, err error) {
	query := strings.Join(stmt.Phrases, " ")

	matchQ := hash{
		"function_score": hash{
			"query": hash{
				"match": hash{
					"Servers.Path": hash{
						"query":     query,
						"fuzziness": 1,
					},
				},
			},
			"field_value_factor": hash{
				"field":    "Size",
				"missing":  1,
				"modifier": "log1p",
				"factor":   0.000000001, // Increase Bytes to Gigabytes
			},
		},
	}

	filterQ := []hash{}
	for _, treat := range stmt.Treats {

		// Filter for file extensions, e.g. extension:pdf or extension!mkv
		if treat.Key == keyExtension {
			regex := ""
			switch treat.Operator {
			case EQUALS:
				regex = fmt.Sprintf(`.+.%s`, ExtractRegexSave(treat.Value))
				break
			case NOT:
				regex = fmt.Sprintf(`@&~(.+.%s)`, ExtractRegexSave(treat.Value))
				break
			default:
				continue
			}

			filterQ = append(filterQ, hash{
				"regexp": hash{
					"Filename": regex,
				},
			})
		}

		// Filter for size, e.g. size>1gb or size<20mb
		if treat.Key == keySize {
			// Try to parse the given size
			size, err := humanize.ParseBytes(treat.Value)
			if err != nil {
				continue
			}

			rangeQ := hash{}
			switch treat.Operator {
			case LTE:
				rangeQ = hash{
					"lte": size,
				}
				break
			case GTE:
				rangeQ = hash{
					"gte": size,
				}
			default:
				continue
			}

			filterQ = append(filterQ, hash{
				"range": hash{
					"Size": rangeQ,
				},
			})
		}

	}

	data, err := elastic.Request("GET", elastic.URL(es.url, "/torture/file/_search"), hash{
		"size": perPage,
		"from": perPage * page,
		"query": hash{
			"filtered": hash{
				"query": matchQ,
				"filter": hash{
					"and": hash{
						"filters": filterQ,
					},
				},
			},
		},
	})
	if err != nil {
		return
	}

	result = elastic.Result{}
	err = json.Unmarshal(data, &result)

	return
}

// Extract only regex-save characters from a string so we can Sprintf
func ExtractRegexSave(src string) string {
	r := regexp.MustCompile(`\w+`)
	return strings.Join(r.FindAllString(src, -1), "")
}
<commit_msg>Frontend: Fix fuzziness to the magic value. Related to #6<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/barnslig/torture/lib/elastic"
	"github.com/dustin/go-humanize"
	"regexp"
	"strings"
)

type ElasticSearch struct {
	url string
}

type hash map[string]interface{}

func CreateElasticSearch(host string) (es *ElasticSearch, err error) {
	es = &ElasticSearch{url: host}
	return
}

func (es *ElasticSearch) Search(stmt Statement, perPage int, page int) (result elastic.Result, err error) {
	query := strings.Join(stmt.Phrases, " ")

	matchQ := hash{
		"function_score": hash{
			"query": hash{
				"match": hash{
					"Servers.Path": hash{
						"query":     query,
						"fuzziness": 0.4,
					},
				},
			},
			"field_value_factor": hash{
				"field":    "Size",
				"missing":  1,
				"modifier": "log1p",
				"factor":   0.000000001, // Increase Bytes to Gigabytes
			},
		},
	}

	filterQ := []hash{}
	for _, treat := range stmt.Treats {

		// Filter for file extensions, e.g. extension:pdf or extension!mkv
		if treat.Key == keyExtension {
			regex := ""
			switch treat.Operator {
			case EQUALS:
				regex = fmt.Sprintf(`.+.%s`, ExtractRegexSave(treat.Value))
				break
			case NOT:
				regex = fmt.Sprintf(`@&~(.+.%s)`, ExtractRegexSave(treat.Value))
				break
			default:
				continue
			}

			filterQ = append(filterQ, hash{
				"regexp": hash{
					"Filename": regex,
				},
			})
		}

		// Filter for size, e.g. size>1gb or size<20mb
		if treat.Key == keySize {
			// Try to parse the given size
			size, err := humanize.ParseBytes(treat.Value)
			if err != nil {
				continue
			}

			rangeQ := hash{}
			switch treat.Operator {
			case LTE:
				rangeQ = hash{
					"lte": size,
				}
				break
			case GTE:
				rangeQ = hash{
					"gte": size,
				}
			default:
				continue
			}

			filterQ = append(filterQ, hash{
				"range": hash{
					"Size": rangeQ,
				},
			})
		}

	}

	data, err := elastic.Request("GET", elastic.URL(es.url, "/torture/file/_search"), hash{
		"size": perPage,
		"from": perPage * page,
		"query": hash{
			"filtered": hash{
				"query": matchQ,
				"filter": hash{
					"and": hash{
						"filters": filterQ,
					},
				},
			},
		},
	})
	if err != nil {
		return
	}

	result = elastic.Result{}
	err = json.Unmarshal(data, &result)

	return
}

// Extract only regex-save characters from a string so we can Sprintf
func ExtractRegexSave(src string) string {
	r := regexp.MustCompile(`\w+`)
	return strings.Join(r.FindAllString(src, -1), "")
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pzsvc

// TODO: Will probably want to rename/rearrange/refactor the pzsvc-exec package so as to better conform
// to go coding standards/naming conventions at some point.

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"strconv"
	//"strings"
	"time"
)

type JobResp struct {
	Type  string
	JobId string
}

type StatusJsonResult struct {
	Id               string
	Name             string
	Type             string
	DataId           string
	Message          string
	Details          string
	ResourceMetadata map[string]string
}

type StatusJsonResp struct {
	Type    string
	JobId   string
	Result  StatusJsonResult
	Results []StatusJsonResult
	Status  string
}

func submitMultipart(bodyStr, runId, address, upload string) (*http.Response, error) {

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	err := writer.WriteField("body", bodyStr)
	if err != nil {
		return nil, err
	}

	if upload != "" {
		file, err := os.Open(fmt.Sprintf(`./%s/%s`, runId, upload))
		if err != nil {
			return nil, err
		}

		defer file.Close()

		part, err := writer.CreateFormFile("file", upload)
		if err != nil {
			return nil, err
		}

		_, err = io.Copy(part, file)
		if err != nil {
			return nil, err
		}
	}

	err = writer.Close()
	if err != nil {
		return nil, err
	}

	fileReq, err := http.NewRequest("POST", address, body)
	if err != nil {
		return nil, err
	}

	fileReq.Header.Add("Content-Type", writer.FormDataContentType())

	client := &http.Client{}
	resp, err := client.Do(fileReq)
	if err != nil {
		return nil, err
	}

	return resp, err
}

func submitSinglePart(method, bodyStr, address string) (*http.Response, error) {

	fileReq, err := http.NewRequest(method, address, bytes.NewBuffer([]byte(bodyStr)))
	if err != nil {
		return nil, err
	}

    fileReq.Header.Add("Content-Type", "application/json")
    fileReq.Header.Add("size", "30")
    fileReq.Header.Add("from", "0")
    fileReq.Header.Add("key", "stamp")
    fileReq.Header.Add("order", "true")

	client := &http.Client{}
	resp, err := client.Do(fileReq)
	if err != nil {
		return nil, err
	}

	return resp, err
}

func Download(dataId, runId, pzAddr string) (string, error) {

	resp, err := http.Get(pzAddr + "/file/" + dataId)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return "", err
	}

	contDisp := resp.Header.Get("Content-Disposition")
	_, params, err := mime.ParseMediaType(contDisp)
	filename := params["filename"]
	if filename == "" {
		filename = "dummy.txt"
	}
	filepath := fmt.Sprintf(`./%s/%s`, runId, filename)

	out, err := os.Create(filepath)
	if err != nil {
		return "", err
	}

	defer out.Close()
	io.Copy(out, resp.Body)

	return filename, nil
}

func getDataId(jobId, pzAddr string) (string, error) {

	type JobResult struct {
		Type			string
		DataId			string
	}

	type JobProg struct {
		PercentComplete	int
	}

	type JobResp struct {
		Type			string
		JobId			string
		Result			JobResult
		Status			string
		JobType			string
		Progress		JobProg
		Message			string
		Origin			string
	}

	time.Sleep(1000 * time.Millisecond)

	for i := 0; i < 100; i++ {

		resp, err := http.Get(pzAddr + "/job/" + jobId)
		if resp != nil {
			defer resp.Body.Close()
		}
		if err != nil {
			return "", err
		}

		respBuf := &bytes.Buffer{}

		_, err = respBuf.ReadFrom(resp.Body)
		if err != nil {
			return "", err
		}

		fmt.Println(respBuf.String())

		var respObj JobResp
		err = json.Unmarshal(respBuf.Bytes(), &respObj)
		if err != nil {
			return "", err
		}

		if respObj.Status == "Submitted" || respObj.Status == "Running" || respObj.Status == "Pending" || respObj.Message == "Job Not Found" {
			time.Sleep(200 * time.Millisecond)
		} else if respObj.Status == "Success" {
			return respObj.Result.DataId, nil
		} else if respObj.Status == "Error" || respObj.Status == "Fail" {
			return "", errors.New(respObj.Status + ": " + respObj.Message)
		} else {
			return "", errors.New("Unknown status: " + respObj.Status)
		}
	}

	return "", errors.New("Never completed.")
}

func ingestMultipart(bodyStr, runId, pzAddr, filename string) (string, error) {

	resp, err := submitMultipart(bodyStr, runId, (pzAddr + "/job"), filename)
	if err != nil {
		return "", err
	}

	respBuf := &bytes.Buffer{}

	_, err = respBuf.ReadFrom(resp.Body)
	if err != nil {
		return "", err
	}

	fmt.Println(respBuf.String())

	var respObj JobResp
	err = json.Unmarshal(respBuf.Bytes(), &respObj)
	if err != nil {
		fmt.Println("error:", err)
	}

	return getDataId(respObj.JobId, pzAddr)
}

func IngestTiff(filename, runId, pzAddr, cmdName string) (string, error) {

	jsonStr := fmt.Sprintf(`{ "userName": "my-api-key-38n987", "jobType": { "type": "ingest", "host": "true", "data" : { "dataType": { "type": "raster" }, "metadata": { "name": "%s", "description": "raster uploaded by pzsvc-exec for %s.", "classType": { "classification": "unclassified" } } } } }`, filename, cmdName)

	return ingestMultipart(jsonStr, runId, pzAddr, filename)
}

func IngestGeoJson(filename, runId, pzAddr, cmdName string) (string, error) {

	jsonStr := fmt.Sprintf(`{ "userName": "my-api-key-38n987", "jobType": { "type": "ingest", "host": "true", "data" : { "dataType": { "type": "geojson" }, "metadata": { "name": "%s", "description": "GeoJson uploaded by pzsvc-exec for %s.", "classType": { "classification": "unclassified" } } } } }`, filename, cmdName)

	return ingestMultipart(jsonStr, runId, pzAddr, filename)
}

func IngestTxt(filename, runId, pzAddr, cmdName string) (string, error) {
	textblock, err := ioutil.ReadFile(fmt.Sprintf(`./%s/%s`, runId, filename))
	if err != nil {
		return "", err
	}

	jsonStr := fmt.Sprintf(`{ "userName": "my-api-key-38n987", "jobType": { "type": "ingest", "host": "true", "data" :{ "dataType": { "type": "text", "mimeType": "application/text", "content": "%s" }, "metadata": { "name": "%s", "description": "text output from pzsvc-exec for %s.", "classType": { "classification": "unclassified" } } } } }`, strconv.QuoteToASCII(string(textblock)), filename, cmdName)

	return ingestMultipart(jsonStr, runId, pzAddr, "")
}

func RegisterSvc(svcName, svcDesc, svcUrl, pzAddr string) error {

	// TODO: add customizable metadata once the inputs are stabilized

	jsonStr := fmt.Sprintf(`{ "inputs": [], "outputs": [], "url": "%s", "resourceMetadata": { "name": "%s", "description": "%s", "method": "POST" } }`, svcUrl, svcName, svcDesc)
fmt.Println(jsonStr)
	_, err := submitSinglePart("POST", jsonStr, pzAddr + "/service")
	if err != nil {
		return err
	}

	return nil
}

func UpdateSvc(svcName, svcId, svcDesc, svcUrl, pzAddr string) error {
	jsonStr := fmt.Sprintf(`{ "serviceId": "%s", "url": "%s", "resourceMetadata": { "name": "%s", "description": "%s", "method": "POST" } }`, svcId, svcUrl, svcName, svcDesc)

	_, err := submitSinglePart("PUT", jsonStr, pzAddr + "/service/" + svcId)
	if err != nil {
		return err
	}

	return nil
}

// Searches Pz for a service matching the input information.  if it finds one,
// returns the service ID.  If it does not, returns an empty string.
// currently only semi-functional.  Read through and vet carefully before declaring functional.

func FindMySvc(svcName, pzAddr string) (string, error) {

	type SvcData struct {
		ServiceId			string
		Url					string
		ResourceMetadata	map[string]string
	}

	type SvcWrapper struct {
		Type		string
		Data		[]SvcData
		Pagination	map[string]int

	}

fmt.Println(pzAddr + "/service?per_page=1000&keyword=" + url.QueryEscape(svcName))

	resp, err := http.Get(pzAddr + "/service?per_page=1000&keyword=" + url.QueryEscape(svcName))
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return "", err
	}

	respBuf, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var respObj SvcWrapper
	err = json.Unmarshal(respBuf, &respObj)
	if err != nil {
		return "", err
	}

	for _, checkServ := range respObj.Data {
		if checkServ.ResourceMetadata["name"] == svcName {
			return checkServ.ServiceId, nil
		}
	}

	return "", nil
}

func ManageRegistration(svcName, svcDesc, svcUrl, pzAddr string) error {
fmt.Println("Finding")
	svcId, err := FindMySvc(svcName, pzAddr)
	if err != nil {
		return err
	}

	if (svcId == "") {
fmt.Println("Registering")
		err = RegisterSvc(svcName, svcDesc, svcUrl, pzAddr)
	} else {
fmt.Println("Updating")
		err = UpdateSvc(svcName, svcId, svcDesc, svcUrl, pzAddr)
	}
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>fixed a small but important URL bug<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pzsvc

// TODO: Will probably want to rename/rearrange/refactor the pzsvc-exec package so as to better conform
// to go coding standards/naming conventions at some point.

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"strconv"
	//"strings"
	"time"
)

type JobResp struct {
	Type  string
	JobId string
}

type StatusJsonResult struct {
	Id               string
	Name             string
	Type             string
	DataId           string
	Message          string
	Details          string
	ResourceMetadata map[string]string
}

type StatusJsonResp struct {
	Type    string
	JobId   string
	Result  StatusJsonResult
	Results []StatusJsonResult
	Status  string
}

func submitMultipart(bodyStr, runId, address, upload string) (*http.Response, error) {

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	err := writer.WriteField("body", bodyStr)
	if err != nil {
		return nil, err
	}

	if upload != "" {
		file, err := os.Open(fmt.Sprintf(`./%s/%s`, runId, upload))
		if err != nil {
			return nil, err
		}

		defer file.Close()

		part, err := writer.CreateFormFile("file", upload)
		if err != nil {
			return nil, err
		}

		_, err = io.Copy(part, file)
		if err != nil {
			return nil, err
		}
	}

	err = writer.Close()
	if err != nil {
		return nil, err
	}

	fileReq, err := http.NewRequest("POST", address, body)
	if err != nil {
		return nil, err
	}

	fileReq.Header.Add("Content-Type", writer.FormDataContentType())

	client := &http.Client{}
	resp, err := client.Do(fileReq)
	if err != nil {
		return nil, err
	}

	return resp, err
}

func submitSinglePart(method, bodyStr, address string) (*http.Response, error) {

	fileReq, err := http.NewRequest(method, address, bytes.NewBuffer([]byte(bodyStr)))
	if err != nil {
		return nil, err
	}

    fileReq.Header.Add("Content-Type", "application/json")
    fileReq.Header.Add("size", "30")
    fileReq.Header.Add("from", "0")
    fileReq.Header.Add("key", "stamp")
    fileReq.Header.Add("order", "true")

	client := &http.Client{}
	resp, err := client.Do(fileReq)
	if err != nil {
		return nil, err
	}

	return resp, err
}

func Download(dataId, runId, pzAddr string) (string, error) {

	resp, err := http.Get(pzAddr + "/file/" + dataId)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return "", err
	}

	contDisp := resp.Header.Get("Content-Disposition")
	_, params, err := mime.ParseMediaType(contDisp)
	filename := params["filename"]
	if filename == "" {
		filename = "dummy.txt"
	}
	filepath := fmt.Sprintf(`./%s/%s`, runId, filename)

	out, err := os.Create(filepath)
	if err != nil {
		return "", err
	}

	defer out.Close()
	io.Copy(out, resp.Body)

	return filename, nil
}

func getDataId(jobId, pzAddr string) (string, error) {

	type JobResult struct {
		Type			string
		DataId			string
	}

	type JobProg struct {
		PercentComplete	int
	}

	type JobResp struct {
		Type			string
		JobId			string
		Result			JobResult
		Status			string
		JobType			string
		Progress		JobProg
		Message			string
		Origin			string
	}

	time.Sleep(1000 * time.Millisecond)

	for i := 0; i < 100; i++ {

		resp, err := http.Get(pzAddr + "/job/" + jobId)
		if resp != nil {
			defer resp.Body.Close()
		}
		if err != nil {
			return "", err
		}

		respBuf := &bytes.Buffer{}

		_, err = respBuf.ReadFrom(resp.Body)
		if err != nil {
			return "", err
		}

		fmt.Println(respBuf.String())

		var respObj JobResp
		err = json.Unmarshal(respBuf.Bytes(), &respObj)
		if err != nil {
			return "", err
		}

		if respObj.Status == "Submitted" || respObj.Status == "Running" || respObj.Status == "Pending" || respObj.Message == "Job Not Found" {
			time.Sleep(200 * time.Millisecond)
		} else if respObj.Status == "Success" {
			return respObj.Result.DataId, nil
		} else if respObj.Status == "Error" || respObj.Status == "Fail" {
			return "", errors.New(respObj.Status + ": " + respObj.Message)
		} else {
			return "", errors.New("Unknown status: " + respObj.Status)
		}
	}

	return "", errors.New("Never completed.")
}

func ingestMultipart(bodyStr, runId, pzAddr, filename string) (string, error) {

	resp, err := submitMultipart(bodyStr, runId, (pzAddr + "/job"), filename)
	if err != nil {
		return "", err
	}

	respBuf := &bytes.Buffer{}

	_, err = respBuf.ReadFrom(resp.Body)
	if err != nil {
		return "", err
	}

	fmt.Println(respBuf.String())

	var respObj JobResp
	err = json.Unmarshal(respBuf.Bytes(), &respObj)
	if err != nil {
		fmt.Println("error:", err)
	}

	return getDataId(respObj.JobId, pzAddr)
}

func IngestTiff(filename, runId, pzAddr, cmdName string) (string, error) {

	jsonStr := fmt.Sprintf(`{ "userName": "my-api-key-38n987", "jobType": { "type": "ingest", "host": "true", "data" : { "dataType": { "type": "raster" }, "metadata": { "name": "%s", "description": "raster uploaded by pzsvc-exec for %s.", "classType": { "classification": "unclassified" } } } } }`, filename, cmdName)

	return ingestMultipart(jsonStr, runId, pzAddr, filename)
}

func IngestGeoJson(filename, runId, pzAddr, cmdName string) (string, error) {

	jsonStr := fmt.Sprintf(`{ "userName": "my-api-key-38n987", "jobType": { "type": "ingest", "host": "true", "data" : { "dataType": { "type": "geojson" }, "metadata": { "name": "%s", "description": "GeoJson uploaded by pzsvc-exec for %s.", "classType": { "classification": "unclassified" } } } } }`, filename, cmdName)

	return ingestMultipart(jsonStr, runId, pzAddr, filename)
}

func IngestTxt(filename, runId, pzAddr, cmdName string) (string, error) {
	textblock, err := ioutil.ReadFile(fmt.Sprintf(`./%s/%s`, runId, filename))
	if err != nil {
		return "", err
	}

	jsonStr := fmt.Sprintf(`{ "userName": "my-api-key-38n987", "jobType": { "type": "ingest", "host": "true", "data" :{ "dataType": { "type": "text", "mimeType": "application/text", "content": "%s" }, "metadata": { "name": "%s", "description": "text output from pzsvc-exec for %s.", "classType": { "classification": "unclassified" } } } } }`, strconv.QuoteToASCII(string(textblock)), filename, cmdName)

	return ingestMultipart(jsonStr, runId, pzAddr, "")
}

func RegisterSvc(svcName, svcDesc, svcUrl, pzAddr string) error {

	// TODO: add customizable metadata once the inputs are stabilized

	jsonStr := fmt.Sprintf(`{ "inputs": [], "outputs": [], "url": "%s/execute", "resourceMetadata": { "name": "%s", "description": "%s", "method": "POST" } }`, svcUrl, svcName, svcDesc)
fmt.Println(jsonStr)
	_, err := submitSinglePart("POST", jsonStr, pzAddr + "/service")
	if err != nil {
		return err
	}

	return nil
}

func UpdateSvc(svcName, svcId, svcDesc, svcUrl, pzAddr string) error {
	jsonStr := fmt.Sprintf(`{ "serviceId": "%s", "url": "%s/execute", "resourceMetadata": { "name": "%s", "description": "%s", "method": "POST" } }`, svcId, svcUrl, svcName, svcDesc)

	_, err := submitSinglePart("PUT", jsonStr, pzAddr + "/service/" + svcId)
	if err != nil {
		return err
	}

	return nil
}

// Searches Pz for a service matching the input information.  if it finds one,
// returns the service ID.  If it does not, returns an empty string.
// currently only semi-functional.  Read through and vet carefully before declaring functional.

func FindMySvc(svcName, pzAddr string) (string, error) {

	type SvcData struct {
		ServiceId			string
		Url					string
		ResourceMetadata	map[string]string
	}

	type SvcWrapper struct {
		Type		string
		Data		[]SvcData
		Pagination	map[string]int

	}

fmt.Println(pzAddr + "/service?per_page=1000&keyword=" + url.QueryEscape(svcName))

	resp, err := http.Get(pzAddr + "/service?per_page=1000&keyword=" + url.QueryEscape(svcName))
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return "", err
	}

	respBuf, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var respObj SvcWrapper
	err = json.Unmarshal(respBuf, &respObj)
	if err != nil {
		return "", err
	}

	for _, checkServ := range respObj.Data {
		if checkServ.ResourceMetadata["name"] == svcName {
			return checkServ.ServiceId, nil
		}
	}

	return "", nil
}

func ManageRegistration(svcName, svcDesc, svcUrl, pzAddr string) error {
fmt.Println("Finding")
	svcId, err := FindMySvc(svcName, pzAddr)
	if err != nil {
		return err
	}

	if (svcId == "") {
fmt.Println("Registering")
		err = RegisterSvc(svcName, svcDesc, svcUrl, pzAddr)
	} else {
fmt.Println("Updating")
		err = UpdateSvc(svcName, svcId, svcDesc, svcUrl, pzAddr)
	}
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tests

import (
	"testing"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"

	"sigs.k8s.io/gateway-api/apis/v1alpha2"
	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
	"sigs.k8s.io/gateway-api/conformance/utils/suite"
)

func init() {
	ConformanceTests = append(ConformanceTests, GatewaySecretReferenceGrantAllInNamespace)
}

var GatewaySecretReferenceGrantAllInNamespace = suite.ConformanceTest{
	ShortName:   "GatewaySecretReferenceGrantAllInNamespace",
	Description: "A Gateway in the gateway-conformance-infra namespace should become ready if the Gateway has a certificateRef for a Secret in the gateway-conformance-web-backend namespace and a ReferenceGrant granting permission to all Secrets in the namespace exists",
	Features:    []suite.SupportedFeature{suite.SupportReferenceGrant},
	Manifests:   []string{"tests/gateway-secret-reference-grant-specific.yaml"},
	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
		gwNN := types.NamespacedName{Name: "gateway-secret-reference-grant", Namespace: "gateway-conformance-infra"}

		t.Run("Gateway listener should have a true ResolvedRefs condition and a true Ready condition", func(t *testing.T) {
			listeners := []v1alpha2.ListenerStatus{{
				Name: v1alpha2.SectionName("https"),
				SupportedKinds: []v1alpha2.RouteGroupKind{{
					Group: (*v1alpha2.Group)(&v1alpha2.GroupVersion.Group),
					Kind:  v1alpha2.Kind("HTTPRoute"),
				}},
				Conditions: []metav1.Condition{
					{
						Type:   string(v1alpha2.ListenerConditionReady),
						Status: metav1.ConditionTrue,
						Reason: string(v1alpha2.ListenerReasonReady),
					},
				},
			}}

			kubernetes.GatewayStatusMustHaveListeners(t, s.Client, s.TimeoutConfig, gwNN, listeners)
		})
	},
}
<commit_msg>conformance: fix GatewaySecretReferenceGrantAllInNamespace resources<commit_after>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tests

import (
	"testing"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"

	"sigs.k8s.io/gateway-api/apis/v1alpha2"
	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
	"sigs.k8s.io/gateway-api/conformance/utils/suite"
)

func init() {
	ConformanceTests = append(ConformanceTests, GatewaySecretReferenceGrantAllInNamespace)
}

var GatewaySecretReferenceGrantAllInNamespace = suite.ConformanceTest{
	ShortName:   "GatewaySecretReferenceGrantAllInNamespace",
	Description: "A Gateway in the gateway-conformance-infra namespace should become ready if the Gateway has a certificateRef for a Secret in the gateway-conformance-web-backend namespace and a ReferenceGrant granting permission to all Secrets in the namespace exists",
	Features:    []suite.SupportedFeature{suite.SupportReferenceGrant},
	Manifests:   []string{"tests/gateway-secret-reference-grant-all-in-namespace.yaml"},
	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
		gwNN := types.NamespacedName{Name: "gateway-secret-reference-grant", Namespace: "gateway-conformance-infra"}

		t.Run("Gateway listener should have a true ResolvedRefs condition and a true Ready condition", func(t *testing.T) {
			listeners := []v1alpha2.ListenerStatus{{
				Name: v1alpha2.SectionName("https"),
				SupportedKinds: []v1alpha2.RouteGroupKind{{
					Group: (*v1alpha2.Group)(&v1alpha2.GroupVersion.Group),
					Kind:  v1alpha2.Kind("HTTPRoute"),
				}},
				Conditions: []metav1.Condition{
					{
						Type:   string(v1alpha2.ListenerConditionReady),
						Status: metav1.ConditionTrue,
						Reason: string(v1alpha2.ListenerReasonReady),
					},
				},
			}}

			kubernetes.GatewayStatusMustHaveListeners(t, s.Client, s.TimeoutConfig, gwNN, listeners)
		})
	},
}
<|endoftext|>


<commit_before>package function_test

import (
	"errors"
	"testing"

	"github.com/apex/apex/function"
	"github.com/apex/apex/mock"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"
)

func TestDelete_Success(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()
	serviceMock := mock_lambdaiface.NewMockLambdaAPI(mockCtrl)

	serviceMock.EXPECT().DeleteFunction(&lambda.DeleteFunctionInput{
		FunctionName: aws.String("testfn"),
	}).Return(&lambda.DeleteFunctionOutput{}, nil)

	fn := &function.Function{
		Config:  function.Config{Name: "testfn"},
		Service: serviceMock,
	}
	err := fn.Delete()

	assert.Nil(t, err)
}

func TestDelete_Failed(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()
	serviceMock := mock_lambdaiface.NewMockLambdaAPI(mockCtrl)

	serviceMock.EXPECT().DeleteFunction(gomock.Any()).Return(nil, errors.New("API err"))

	fn := &function.Function{
		Config:  function.Config{Name: "testfn"},
		Service: serviceMock,
	}
	err := fn.Delete()

	assert.Error(t, err, "API err")
}
<commit_msg>rename function tests<commit_after>package function_test

import (
	"errors"
	"testing"

	"github.com/apex/apex/function"
	"github.com/apex/apex/mock"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"
)

func TestFunction_Delete_success(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()
	serviceMock := mock_lambdaiface.NewMockLambdaAPI(mockCtrl)

	serviceMock.EXPECT().DeleteFunction(&lambda.DeleteFunctionInput{
		FunctionName: aws.String("testfn"),
	}).Return(&lambda.DeleteFunctionOutput{}, nil)

	fn := &function.Function{
		Config:  function.Config{Name: "testfn"},
		Service: serviceMock,
	}
	err := fn.Delete()

	assert.Nil(t, err)
}

func TestFunction_Delete_failed(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()
	serviceMock := mock_lambdaiface.NewMockLambdaAPI(mockCtrl)

	serviceMock.EXPECT().DeleteFunction(gomock.Any()).Return(nil, errors.New("API err"))

	fn := &function.Function{
		Config:  function.Config{Name: "testfn"},
		Service: serviceMock,
	}
	err := fn.Delete()

	assert.Error(t, err, "API err")
}
<|endoftext|>


<commit_before>package utils

import "os"

// FileUtil type
type FileUtil struct {
	// Path to the file
	path string

	os.FileInfo
}

// NewFileUtil creates a file utility from the given path
func NewFileUtil(path string) (*FileUtil, error) {
	info, err := os.Stat(path)
	if err != nil {
		return &FileUtil{}, err
	}

	f := &FileUtil{
		path,
		info,
	}

	return f, nil
}

// Md5 returns the md5 hash of the file's content
func (f *FileUtil) Md5() (string, error) {
	buf, err := iotuil.ReadFile(f.path)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", md5.Sum(buf)), nil
}
<commit_msg>utils: implement Sha1() and Sha256() methods on FileUtil type<commit_after>package utils

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"fmt"
	"os"
)

// FileUtil type
type FileUtil struct {
	// Path to the file
	path string

	os.FileInfo
}

// NewFileUtil creates a file utility from the given path
func NewFileUtil(path string) (*FileUtil, error) {
	info, err := os.Stat(path)
	if err != nil {
		return &FileUtil{}, err
	}

	f := &FileUtil{
		path,
		info,
	}

	return f, nil
}

// Md5 returns the md5 checksum of the file's contents
func (f *FileUtil) Md5() (string, error) {
	buf, err := iotuil.ReadFile(f.path)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", md5.Sum(buf)), nil
}

// Sha1 returns the sha1 checksum of the file's contents
func (f *FileUtil) Sha1() (string, error) {
	buf, err := iotuil.ReadFile(f.path)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", sha1.Sum(buf)), nil
}

// Sha256 returns the sha256 checksum of the file's contents
func (f *FileUtil) Sha256() (string, error) {
	buf, err := iotuil.ReadFile(f.path)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", sha256.Sum256(buf)), nil
}
<|endoftext|>


<commit_before>// Package defaults implements the Pipe interface providing default values
// for missing configuration.
package defaults

import (
	"github.com/apex/log"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/pipeline"
	"github.com/goreleaser/goreleaser/pipeline/archive"
	"github.com/goreleaser/goreleaser/pipeline/artifactory"
	"github.com/goreleaser/goreleaser/pipeline/brew"
	"github.com/goreleaser/goreleaser/pipeline/build"
	"github.com/goreleaser/goreleaser/pipeline/checksums"
	"github.com/goreleaser/goreleaser/pipeline/docker"
	"github.com/goreleaser/goreleaser/pipeline/fpm"
	"github.com/goreleaser/goreleaser/pipeline/release"
	"github.com/goreleaser/goreleaser/pipeline/sign"
	"github.com/goreleaser/goreleaser/pipeline/snapshot"
)

// Pipe that sets the defaults
type Pipe struct{}

func (Pipe) String() string {
	return "setting defaults for:"
}

var defaulters = []pipeline.Defaulter{
	snapshot.Pipe{},
	release.Pipe{},
	archive.Pipe{},
	build.Pipe{},
	fpm.Pipe{},
	checksums.Pipe{},
	sign.Pipe{},
	docker.Pipe{},
	artifactory.Pipe{},
	brew.Pipe{},
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Dist == "" {
		ctx.Config.Dist = "dist"
	}
	for _, defaulter := range defaulters {
		log.Info(defaulter.String())
		if err := defaulter.Default(ctx); err != nil {
			return err
		}
	}
	if ctx.Config.ProjectName == "" {
		ctx.Config.ProjectName = ctx.Config.Release.GitHub.Name
	}
	return nil
}
<commit_msg>feat: defaulter on snap pipe<commit_after>// Package defaults implements the Pipe interface providing default values
// for missing configuration.
package defaults

import (
	"github.com/apex/log"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/pipeline"
	"github.com/goreleaser/goreleaser/pipeline/archive"
	"github.com/goreleaser/goreleaser/pipeline/artifactory"
	"github.com/goreleaser/goreleaser/pipeline/brew"
	"github.com/goreleaser/goreleaser/pipeline/build"
	"github.com/goreleaser/goreleaser/pipeline/checksums"
	"github.com/goreleaser/goreleaser/pipeline/docker"
	"github.com/goreleaser/goreleaser/pipeline/fpm"
	"github.com/goreleaser/goreleaser/pipeline/release"
	"github.com/goreleaser/goreleaser/pipeline/sign"
	"github.com/goreleaser/goreleaser/pipeline/snapcraft"
	"github.com/goreleaser/goreleaser/pipeline/snapshot"
)

// Pipe that sets the defaults
type Pipe struct{}

func (Pipe) String() string {
	return "setting defaults for:"
}

var defaulters = []pipeline.Defaulter{
	snapshot.Pipe{},
	release.Pipe{},
	archive.Pipe{},
	build.Pipe{},
	fpm.Pipe{},
	snapcraft.Pipe{},
	checksums.Pipe{},
	sign.Pipe{},
	docker.Pipe{},
	artifactory.Pipe{},
	brew.Pipe{},
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Dist == "" {
		ctx.Config.Dist = "dist"
	}
	for _, defaulter := range defaulters {
		log.Info(defaulter.String())
		if err := defaulter.Default(ctx); err != nil {
			return err
		}
	}
	if ctx.Config.ProjectName == "" {
		ctx.Config.ProjectName = ctx.Config.Release.GitHub.Name
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tapmanager

import (
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/containernetworking/cni/pkg/ns"
	cnitypes "github.com/containernetworking/cni/pkg/types"
	cnicurrent "github.com/containernetworking/cni/pkg/types/current"
	"github.com/davecgh/go-spew/spew"
	"github.com/golang/glog"

	"github.com/Mirantis/virtlet/pkg/cni"
	"github.com/Mirantis/virtlet/pkg/dhcp"
	"github.com/Mirantis/virtlet/pkg/nettools"
)

// PodNetworkDesc contains the data that are required by TapFDSource
// to set up a tap device for a VM
type PodNetworkDesc struct {
	// PodId specifies the id of the pod
	PodId string `json:"podId"`
	// PodNs specifies the namespace of the pod
	PodNs string `json:"podNs"`
	// PodName specifies the name of the pod
	PodName string `json:"podName"`
	// DNS specifies DNS settings for the pod
	DNS *cnitypes.DNS
}

// GetFDPayload contains the data that are required by TapFDSource
// to recover the tap device that was already configured, or create a new one
// if CNIConfig is nil
type GetFDPayload struct {
	// Description specifies pod network description for already
	// prepared network configuration
	Description *PodNetworkDesc `json:"podNetworkDesc"`
	// CNIConfig specifies CNI configuration used to configure retaken
	// environment
	CNIConfig *cnicurrent.Result `json:"cniConfig"`
}

type podNetwork struct {
	pnd        PodNetworkDesc
	csn        *nettools.ContainerSideNetwork
	dhcpServer *dhcp.Server
	doneCh     chan error
}

// TapFDSource sets up and tears down Virtlet VM network.
// It implements FDSource interface
type TapFDSource struct {
	sync.Mutex

	cniClient *cni.Client
	fdMap     map[string]*podNetwork
}

var _ FDSource = &TapFDSource{}

// NewTapFDSource returns a TapFDSource for the specified CNI plugin &
// config dir
func NewTapFDSource(cniPluginsDir, cniConfigsDir string) (*TapFDSource, error) {
	cniClient, err := cni.NewClient(cniPluginsDir, cniConfigsDir)
	if err != nil {
		return nil, err
	}

	return &TapFDSource{
		cniClient: cniClient,
		fdMap:     make(map[string]*podNetwork),
	}, nil
}

// GetFD implements GetFD method of FDSource interface
func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) {
	var payload GetFDPayload
	if err := json.Unmarshal(data, &payload); err != nil {
		return 0, nil, fmt.Errorf("error unmarshalling GetFD payload: %v", err)
	}
	pnd := payload.Description

	recover := payload.CNIConfig != nil

	if !recover {
		if err := cni.CreateNetNS(pnd.PodId); err != nil {
			return 0, nil, fmt.Errorf("error creating new netns for pod %s (%s): %v", pnd.PodName, pnd.PodId, err)
		}

		netConfig, err := s.cniClient.AddSandboxToNetwork(pnd.PodId, pnd.PodName, pnd.PodNs)
		if err != nil {
			return 0, nil, fmt.Errorf("error adding pod %s (%s) to CNI network: %v", pnd.PodName, pnd.PodId, err)
		}
		glog.V(3).Infof("CNI configuration for pod %s (%s): %s", pnd.PodName, pnd.PodId, spew.Sdump(netConfig))

		if payload.Description.DNS != nil {
			netConfig.DNS.Nameservers = pnd.DNS.Nameservers
			netConfig.DNS.Search = pnd.DNS.Search
			netConfig.DNS.Options = pnd.DNS.Options
		}
		payload.CNIConfig = netConfig
	}

	netConfig := payload.CNIConfig

	netNSPath := cni.PodNetNSPath(pnd.PodId)
	vmNS, err := ns.GetNS(netNSPath)
	if err != nil {
		return 0, nil, fmt.Errorf("failed to open network namespace at %q: %v", netNSPath, err)
	}

	var csn *nettools.ContainerSideNetwork
	var dhcpServer *dhcp.Server
	doneCh := make(chan error)
	if err := vmNS.Do(func(ns.NetNS) error {
		var err error
		if recover {
			csn, err = nettools.RecreateContainerSideNetwork(netConfig)
		} else {
			csn, err = nettools.SetupContainerSideNetwork(netConfig)
		}
		if err != nil {
			return err
		}

		// NOTE: older CNI plugins don't include the hardware address
		// in Result, but it's needed for Cloud-Init based
		// network setup, so we add it here if it's missing
		ensureCNIInterfaceHwAddress(netConfig, csn)

		// TODO: now CNIConfig should always contain interface mac address, so there
		// is no reason to pass it as separate field in dhcp.Config,
		// dhcp.NewServer should need only CNIConfig, instead of dhcp.Config
		// TODO: set up DHCP server for all the interfaces defined in CNIConfig
		dhcpConfg := &dhcp.Config{
			CNIResult:           *csn.Result,
			PeerHardwareAddress: csn.HardwareAddr,
		}
		dhcpServer = dhcp.NewServer(dhcpConfg)
		if err := dhcpServer.SetupListener("0.0.0.0"); err != nil {
			return fmt.Errorf("Failed to set up dhcp listener: %v", err)
		}
		go func() {
			doneCh <- vmNS.Do(func(ns.NetNS) error {
				err := dhcpServer.Serve()
				if err != nil {
					glog.Errorf("dhcp server error: %v", err)
				}
				return err
			})
		}()
		// FIXME: there's some very small possibility for a race here
		// (happens if the VM makes DHCP request before DHCP server is ready)
		// For now, let's make the probability of such problem even smaller
		time.Sleep(500 * time.Millisecond)
		return nil
	}); err != nil {
		return 0, nil, err
	}

	respData, err := json.Marshal(netConfig)
	if err != nil {
		return 0, nil, fmt.Errorf("error marshalling net config: %v", err)
	}

	s.Lock()
	defer s.Unlock()
	s.fdMap[key] = &podNetwork{
		pnd:        *pnd,
		csn:        csn,
		dhcpServer: dhcpServer,
		doneCh:     doneCh,
	}
	return int(csn.TapFile.Fd()), respData, nil
}

// Release implements Release method of FDSource interface
func (s *TapFDSource) Release(key string) error {
	s.Lock()
	defer s.Unlock()
	pn, found := s.fdMap[key]
	if !found {
		return fmt.Errorf("bad fd key: %q", key)
	}

	netNSPath := cni.PodNetNSPath(pn.pnd.PodId)

	vmNS, err := ns.GetNS(netNSPath)
	if err != nil {
		return fmt.Errorf("failed to open network namespace at %q: %v", netNSPath, err)
	}

	if err := vmNS.Do(func(ns.NetNS) error {
		if err := pn.dhcpServer.Close(); err != nil {
			return fmt.Errorf("failed to stop dhcp server: %v", err)
		}
		<-pn.doneCh
		if err := pn.csn.Teardown(); err != nil {
			return err
		}
		return nil
	}); err != nil {
		return err
	}

	if err := s.cniClient.RemoveSandboxFromNetwork(pn.pnd.PodId, pn.pnd.PodName, pn.pnd.PodNs); err != nil {
		return fmt.Errorf("error removing pod sandbox %q from CNI network: %v", pn.pnd.PodId, err)
	}

	if err := cni.DestroyNetNS(pn.pnd.PodId); err != nil {
		return fmt.Errorf("error when removing network namespace for pod sandbox %q: %v", pn.pnd.PodId, err)
	}

	delete(s.fdMap, key)
	return nil
}

// GetInfo implements GetInfo method of FDSource interface
func (s *TapFDSource) GetInfo(key string) ([]byte, error) {
	s.Lock()
	defer s.Unlock()
	pn, found := s.fdMap[key]
	if !found {
		return nil, fmt.Errorf("bad fd key: %q", key)
	}
	return pn.csn.HardwareAddr, nil
}

func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) {
	// If there's no interface info in netConfig, we can assume that we're dealing
	// with an old-style CNI plugin which only supports a single network interface
	if len(netConfig.Interfaces) > 0 {
		return
	}

	iface := &cnicurrent.Interface{
		Name: "cni0",
		Mac:  csn.HardwareAddr.String(),
	}
	netConfig.Interfaces = []*cnicurrent.Interface{iface}

	for _, IP := range netConfig.IPs {
		IP.Interface = 0
	}
}
<commit_msg>Calico support PoC<commit_after>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tapmanager

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/containernetworking/cni/pkg/ns"
	cnitypes "github.com/containernetworking/cni/pkg/types"
	cnicurrent "github.com/containernetworking/cni/pkg/types/current"
	"github.com/davecgh/go-spew/spew"
	"github.com/golang/glog"

	"github.com/Mirantis/virtlet/pkg/cni"
	"github.com/Mirantis/virtlet/pkg/dhcp"
	"github.com/Mirantis/virtlet/pkg/nettools"
)

const (
	podIpFile = "/pod-ip"
)

// PodNetworkDesc contains the data that are required by TapFDSource
// to set up a tap device for a VM
type PodNetworkDesc struct {
	// PodId specifies the id of the pod
	PodId string `json:"podId"`
	// PodNs specifies the namespace of the pod
	PodNs string `json:"podNs"`
	// PodName specifies the name of the pod
	PodName string `json:"podName"`
	// DNS specifies DNS settings for the pod
	DNS *cnitypes.DNS
}

// GetFDPayload contains the data that are required by TapFDSource
// to recover the tap device that was already configured, or create a new one
// if CNIConfig is nil
type GetFDPayload struct {
	// Description specifies pod network description for already
	// prepared network configuration
	Description *PodNetworkDesc `json:"podNetworkDesc"`
	// CNIConfig specifies CNI configuration used to configure retaken
	// environment
	CNIConfig *cnicurrent.Result `json:"cniConfig"`
}

type podNetwork struct {
	pnd        PodNetworkDesc
	csn        *nettools.ContainerSideNetwork
	dhcpServer *dhcp.Server
	doneCh     chan error
}

// TapFDSource sets up and tears down Virtlet VM network.
// It implements FDSource interface
type TapFDSource struct {
	sync.Mutex

	cniClient *cni.Client
	fdMap     map[string]*podNetwork
}

var _ FDSource = &TapFDSource{}

// NewTapFDSource returns a TapFDSource for the specified CNI plugin &
// config dir
func NewTapFDSource(cniPluginsDir, cniConfigsDir string) (*TapFDSource, error) {
	cniClient, err := cni.NewClient(cniPluginsDir, cniConfigsDir)
	if err != nil {
		return nil, err
	}

	return &TapFDSource{
		cniClient: cniClient,
		fdMap:     make(map[string]*podNetwork),
	}, nil
}

// GetFD implements GetFD method of FDSource interface
func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) {
	var payload GetFDPayload
	if err := json.Unmarshal(data, &payload); err != nil {
		return 0, nil, fmt.Errorf("error unmarshalling GetFD payload: %v", err)
	}
	pnd := payload.Description

	recover := payload.CNIConfig != nil

	if !recover {
		if err := cni.CreateNetNS(pnd.PodId); err != nil {
			return 0, nil, fmt.Errorf("error creating new netns for pod %s (%s): %v", pnd.PodName, pnd.PodId, err)
		}

		netConfig, err := s.cniClient.AddSandboxToNetwork(pnd.PodId, pnd.PodName, pnd.PodNs)
		if err != nil {
			return 0, nil, fmt.Errorf("error adding pod %s (%s) to CNI network: %v", pnd.PodName, pnd.PodId, err)
		}
		glog.V(3).Infof("CNI configuration for pod %s (%s): %s", pnd.PodName, pnd.PodId, spew.Sdump(netConfig))

		if payload.Description.DNS != nil {
			netConfig.DNS.Nameservers = pnd.DNS.Nameservers
			netConfig.DNS.Search = pnd.DNS.Search
			netConfig.DNS.Options = pnd.DNS.Options
		}
		payload.CNIConfig = netConfig
	}

	netConfig := payload.CNIConfig

	// serialize the original config before modifying it
	// (in case if dummyGateway is set)
	respData, err := json.Marshal(netConfig)
	if err != nil {
		return 0, nil, fmt.Errorf("error marshalling net config: %v", err)
	}

	// TODO: do this upon startup
	var dummyGateway net.IP
	// TODO: detect Calico here
	// TODO: get pod IP address from an env var set by cmd/virtlet/virtlet.go
	// TODO: add descriptive comments why that's needed
	// (incl. the fact that Virtlet runs in host network ns so it can't just grab
	ipBytes, err := ioutil.ReadFile(podIpFile)
	if err == nil {
		ipStr := strings.TrimSpace(string(ipBytes))
		dummyGateway = net.ParseIP(ipStr)
		if dummyGateway == nil {
			return 0, nil, fmt.Errorf("error parsing pod IP: %q", ipStr)
		}
	}

	if dummyGateway != nil {
		// TODO: better diagnostics
		if len(netConfig.IPs) != 1 {
			return 0, nil, errors.New("didn't expect more than one IP config")
		}
		if netConfig.IPs[0].Version != "4" {
			return 0, nil, errors.New("IPv4 config was expected")
		}
		// TODO: calculate network mask based on the pod IP and the gateway IP
		netConfig.IPs[0].Address.Mask = net.IPv4Mask(255, 255, 255, 0)
		netConfig.IPs[0].Gateway = dummyGateway
		netConfig.Routes = []*cnitypes.Route{
			{
				Dst: net.IPNet{
					IP:   net.IP{0, 0, 0, 0},
					Mask: net.IPMask{0, 0, 0, 0},
				},
				GW: dummyGateway,
			},
		}
	}

	netNSPath := cni.PodNetNSPath(pnd.PodId)
	vmNS, err := ns.GetNS(netNSPath)
	if err != nil {
		return 0, nil, fmt.Errorf("failed to open network namespace at %q: %v", netNSPath, err)
	}

	var csn *nettools.ContainerSideNetwork
	var dhcpServer *dhcp.Server
	doneCh := make(chan error)
	if err := vmNS.Do(func(ns.NetNS) error {
		var err error
		if recover {
			csn, err = nettools.RecreateContainerSideNetwork(netConfig)
		} else {
			csn, err = nettools.SetupContainerSideNetwork(netConfig)
		}
		if err != nil {
			return err
		}

		// NOTE: older CNI plugins don't include the hardware address
		// in Result, but it's needed for Cloud-Init based
		// network setup, so we add it here if it's missing
		ensureCNIInterfaceHwAddress(netConfig, csn)

		// TODO: now CNIConfig should always contain interface mac address, so there
		// is no reason to pass it as separate field in dhcp.Config,
		// dhcp.NewServer should need only CNIConfig, instead of dhcp.Config
		// TODO: set up DHCP server for all the interfaces defined in CNIConfig
		dhcpConfg := &dhcp.Config{
			CNIResult:           *csn.Result,
			PeerHardwareAddress: csn.HardwareAddr,
		}
		dhcpServer = dhcp.NewServer(dhcpConfg)
		if err := dhcpServer.SetupListener("0.0.0.0"); err != nil {
			return fmt.Errorf("Failed to set up dhcp listener: %v", err)
		}
		go func() {
			doneCh <- vmNS.Do(func(ns.NetNS) error {
				err := dhcpServer.Serve()
				if err != nil {
					glog.Errorf("dhcp server error: %v", err)
				}
				return err
			})
		}()
		// FIXME: there's some very small possibility for a race here
		// (happens if the VM makes DHCP request before DHCP server is ready)
		// For now, let's make the probability of such problem even smaller
		time.Sleep(500 * time.Millisecond)
		return nil
	}); err != nil {
		return 0, nil, err
	}

	s.Lock()
	defer s.Unlock()
	s.fdMap[key] = &podNetwork{
		pnd:        *pnd,
		csn:        csn,
		dhcpServer: dhcpServer,
		doneCh:     doneCh,
	}
	return int(csn.TapFile.Fd()), respData, nil
}

// Release implements Release method of FDSource interface
func (s *TapFDSource) Release(key string) error {
	s.Lock()
	defer s.Unlock()
	pn, found := s.fdMap[key]
	if !found {
		return fmt.Errorf("bad fd key: %q", key)
	}

	netNSPath := cni.PodNetNSPath(pn.pnd.PodId)

	vmNS, err := ns.GetNS(netNSPath)
	if err != nil {
		return fmt.Errorf("failed to open network namespace at %q: %v", netNSPath, err)
	}

	if err := vmNS.Do(func(ns.NetNS) error {
		if err := pn.dhcpServer.Close(); err != nil {
			return fmt.Errorf("failed to stop dhcp server: %v", err)
		}
		<-pn.doneCh
		if err := pn.csn.Teardown(); err != nil {
			return err
		}
		return nil
	}); err != nil {
		return err
	}

	if err := s.cniClient.RemoveSandboxFromNetwork(pn.pnd.PodId, pn.pnd.PodName, pn.pnd.PodNs); err != nil {
		return fmt.Errorf("error removing pod sandbox %q from CNI network: %v", pn.pnd.PodId, err)
	}

	if err := cni.DestroyNetNS(pn.pnd.PodId); err != nil {
		return fmt.Errorf("error when removing network namespace for pod sandbox %q: %v", pn.pnd.PodId, err)
	}

	delete(s.fdMap, key)
	return nil
}

// GetInfo implements GetInfo method of FDSource interface
func (s *TapFDSource) GetInfo(key string) ([]byte, error) {
	s.Lock()
	defer s.Unlock()
	pn, found := s.fdMap[key]
	if !found {
		return nil, fmt.Errorf("bad fd key: %q", key)
	}
	return pn.csn.HardwareAddr, nil
}

func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) {
	// If there's no interface info in netConfig, we can assume that we're dealing
	// with an old-style CNI plugin which only supports a single network interface
	if len(netConfig.Interfaces) > 0 {
		return
	}

	iface := &cnicurrent.Interface{
		Name: "cni0",
		Mac:  csn.HardwareAddr.String(),
	}
	netConfig.Interfaces = []*cnicurrent.Interface{iface}

	for _, IP := range netConfig.IPs {
		IP.Interface = 0
	}
}
<|endoftext|>


<commit_before>// Copyright © 2019 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"strings"

	"github.com/banzaicloud/bank-vaults/pkg/vault"
	corev1 "k8s.io/api/core/v1"
)

func configMapNeedsMutation(configMap *corev1.ConfigMap) bool {
	for _, value := range configMap.Data {
		if strings.HasPrefix(value, "vault:") {
			return true
		}
	}
	return false
}

func mutateConfigMap(configMap *corev1.ConfigMap, vaultConfig vaultConfig, ns string) error {

	// do an early exit and don't construct the Vault client if not needed
	if !configMapNeedsMutation(configMap) {
		return nil
	}

	vaultClient, err := newVaultClient(vaultConfig)
	if err != nil {
		return fmt.Errorf("failed to create vault client: %v", err)
	}

	defer vaultClient.Close()

	for key, value := range configMap.Data {
		if strings.HasPrefix(value, "vault:") {
			data := map[string]string{
				key: string(value),
			}
			err := mutateConfigMapData(configMap, data, vaultClient)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func mutateConfigMapData(configMap *corev1.ConfigMap, data map[string]string, vaultClient *vault.Client) error {
	mapData, err := getDataFromVault(data, vaultClient)
	if err != nil {
		return err
	}
	for key, value := range mapData {
		configMap.Data[key] = value
	}
	return nil
}
<commit_msg>webhook/configmap: mutate BinaryData as well<commit_after>// Copyright © 2019 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/base64"
	"fmt"
	"strings"

	"github.com/banzaicloud/bank-vaults/pkg/vault"
	corev1 "k8s.io/api/core/v1"
)

func configMapNeedsMutation(configMap *corev1.ConfigMap) bool {
	for _, value := range configMap.Data {
		if strings.HasPrefix(value, "vault:") {
			return true
		}
	}
	for _, value := range configMap.BinaryData {
		if strings.HasPrefix(string(value), "vault:") {
			return true
		}
	}
	return false
}

func mutateConfigMap(configMap *corev1.ConfigMap, vaultConfig vaultConfig, ns string) error {

	// do an early exit and don't construct the Vault client if not needed
	if !configMapNeedsMutation(configMap) {
		return nil
	}

	vaultClient, err := newVaultClient(vaultConfig)
	if err != nil {
		return fmt.Errorf("failed to create vault client: %v", err)
	}

	defer vaultClient.Close()

	for key, value := range configMap.Data {
		if strings.HasPrefix(value, "vault:") {
			data := map[string]string{
				key: string(value),
			}
			err := mutateConfigMapData(configMap, data, vaultClient)
			if err != nil {
				return err
			}
		}
	}

	for key, value := range configMap.BinaryData {
		if strings.HasPrefix(string(value), "vault:") {
			binaryData := map[string]string{
				key: string(value),
			}
			err := mutateConfigMapBinaryData(configMap, binaryData, vaultClient)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func mutateConfigMapData(configMap *corev1.ConfigMap, data map[string]string, vaultClient *vault.Client) error {
	mapData, err := getDataFromVault(data, vaultClient)
	if err != nil {
		return err
	}
	for key, value := range mapData {
		configMap.Data[key] = value
	}
	return nil
}

func mutateConfigMapBinaryData(configMap *corev1.ConfigMap, data map[string]string, vaultClient *vault.Client) error {
	mapData, err := getDataFromVault(data, vaultClient)
	if err != nil {
		return err
	}
	for key, value := range mapData {
		// binary data are stored in base64 inside vault
		// we need to decode base64 since k8s will encode this data too
		valueBytes, err := base64.StdEncoding.DecodeString(value)
		if err != nil {
			return err
		}
		configMap.BinaryData[key] = valueBytes
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/alecthomas/template"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

var HELM_BIN = "/bin/helm"
var KUBECONFIG = "/root/.kube/kubeconfig"
var CONFIG = "/root/.kube/config"

type (
	// Config maps the params we need to run Helm
	Config struct {
		APIServer     string   `json:"api_server"`
		Token         string   `json:"token"`
		HelmCommand   []string `json:"helm_command"`
		SkipTLSVerify bool     `json:"tls_skip_verify"`
		Namespace     string   `json:"namespace"`
		Release       string   `json:"release"`
		Chart         string   `json:"chart"`
		Values        string   `json:"values"`
		ValuesFiles   string   `json:"values_files"`
		Debug         bool     `json:"debug"`
		DryRun        bool     `json:"dry_run"`
		Secrets       []string `json:"secrets"`
		Prefix        string   `json:"prefix"`
		TillerNs      string   `json:"tiller_ns"`
	}
	// Plugin default
	Plugin struct {
		Config Config
	}
)

func setHelmHelp(p *Plugin) {
	p.Config.HelmCommand = []string{""}
}
func setDeleteEventCommand(p *Plugin) {
	upgrade := make([]string, 2)
	upgrade[0] = "delete"
	upgrade[1] = p.Config.Release

	p.Config.HelmCommand = upgrade
}

func setPushEventCommand(p *Plugin) {
	upgrade := make([]string, 2)
	upgrade[0] = "upgrade"
	upgrade[1] = "--install"

	if p.Config.Release != "" {
		upgrade = append(upgrade, p.Config.Release)
	}
	upgrade = append(upgrade, p.Config.Chart)
	if p.Config.Values != "" {
		upgrade = append(upgrade, "--set")
		upgrade = append(upgrade, p.Config.Values)
	}
	if p.Config.ValuesFiles != "" {
  	for _, valuesFile := range strings.Split(p.Config.ValuesFiles, ",") {
      upgrade = append(upgrade, "--values")
      upgrade = append(upgrade, valuesFile)
    }
	}
	if p.Config.Namespace != "" {
		upgrade = append(upgrade, "--namespace")
		upgrade = append(upgrade, p.Config.Namespace)
	}
	if p.Config.DryRun {
		upgrade = append(upgrade, "--dry-run")
	}
	if p.Config.Debug {
		upgrade = append(upgrade, "--debug")
	}
	p.Config.HelmCommand = upgrade

}

func setHelmCommand(p *Plugin) {
	buildEvent := os.Getenv("DRONE_BUILD_EVENT")
	switch buildEvent {
	case "push":
		setPushEventCommand(p)
	case "delete":
		setDeleteEventCommand(p)
	default:
		setHelmHelp(p)
	}

}

func doHelmInit(p *Plugin) []string {
	init := make([]string, 1)
	init[0] = "init"
	if p.Config.TillerNs != "" {
		init = append(init, "--tiller-namespace")
		init = append(init, p.Config.TillerNs)
	}
	return init

}

// Exec default method
func (p *Plugin) Exec() error {
	resolveSecrets(p)
	if p.Config.APIServer == "" {
		return fmt.Errorf("Error: API Server is needed to deploy.")
	}
	if p.Config.Token == "" {
		return fmt.Errorf("Error: Token is needed to deploy.")
	}
	initialiseKubeconfig(&p.Config, KUBECONFIG, CONFIG)

	if p.Config.Debug {
		p.debug()
	}

	init := doHelmInit(p)
	err := runCommand(init)
	if err != nil {
		return fmt.Errorf("Error running helm command: " + strings.Join(init[:], " "))
	}
	setHelmCommand(p)

	if p.Config.Debug {
		log.Println("helm command: " + strings.Join(p.Config.HelmCommand[:], " "))
	}
	err = runCommand(p.Config.HelmCommand)
	if err != nil {
		return fmt.Errorf("Error running helm command: " + strings.Join(p.Config.HelmCommand[:], " "))
	}
	return nil
}

func initialiseKubeconfig(params *Config, source string, target string) error {
	t, _ := template.ParseFiles(source)
	f, err := os.Create(target)
	err = t.Execute(f, params)
	f.Close()
	return err
}

func runCommand(params []string) error {
	cmd := new(exec.Cmd)
	cmd = exec.Command(HELM_BIN, params...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err := cmd.Run()
	return err
}

func resolveSecrets(p *Plugin) {
	p.Config.Values = resolveEnvVar(p.Config.Values, p.Config.Prefix)
	p.Config.APIServer = resolveEnvVar("${API_SERVER}", p.Config.Prefix)
	p.Config.Token = resolveEnvVar("${KUBERNETES_TOKEN}", p.Config.Prefix)
}

// getEnvVars will return [${TAG} {TAG} TAG]
func getEnvVars(envvars string) [][]string {
	re := regexp.MustCompile(`\$(\{?(\w+)\}?)\.?`)
	extracted := re.FindAllStringSubmatch(envvars, -1)
	return extracted
}

func resolveEnvVar(key string, prefix string) string {
	envvars := getEnvVars(key)
	return replaceEnvvars(envvars, prefix, key)
}

func replaceEnvvars(envvars [][]string, prefix string, s string) string {
	for _, envvar := range envvars {
		envvarName := envvar[0]
		envvarKey := envvar[2]
		envval := os.Getenv(prefix + "_" + envvarKey)
		if envval == "" {
			envval = os.Getenv(envvarKey)
		}
		if strings.Contains(s, envvarName) {
			s = strings.Replace(s, envvarName, envval, -1)
		}
	}
	return s
}

func (p *Plugin) debug() {
	fmt.Println(p)
	// debug env vars
	for _, e := range os.Environ() {
		fmt.Println("-Var:--", e)
	}
	// debug plugin obj
	fmt.Printf("Api server: %s \n", p.Config.APIServer)
	fmt.Printf("Values: %s \n", p.Config.Values)
	fmt.Printf("Secrets: %s \n", p.Config.Secrets)
	fmt.Printf("ValuesFiles: %s \n", p.Config.ValuesFiles)

	kubeconfig, err := ioutil.ReadFile(KUBECONFIG)
	if err == nil {
		fmt.Println(string(kubeconfig))
	}
	config, err := ioutil.ReadFile(CONFIG)
	if err == nil {
		fmt.Println(string(config))
	}

}
<commit_msg>Added back tiller namespace option for upgrade command. Now that main branch is using Helm 2.2 this will work.<commit_after>package main

import (
	"fmt"
	"github.com/alecthomas/template"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

var HELM_BIN = "/bin/helm"
var KUBECONFIG = "/root/.kube/kubeconfig"
var CONFIG = "/root/.kube/config"

type (
	// Config maps the params we need to run Helm
	Config struct {
		APIServer     string   `json:"api_server"`
		Token         string   `json:"token"`
		HelmCommand   []string `json:"helm_command"`
		SkipTLSVerify bool     `json:"tls_skip_verify"`
		Namespace     string   `json:"namespace"`
		Release       string   `json:"release"`
		Chart         string   `json:"chart"`
		Values        string   `json:"values"`
		ValuesFiles   string   `json:"values_files"`
		Debug         bool     `json:"debug"`
		DryRun        bool     `json:"dry_run"`
		Secrets       []string `json:"secrets"`
		Prefix        string   `json:"prefix"`
		TillerNs      string   `json:"tiller_ns"`
	}
	// Plugin default
	Plugin struct {
		Config Config
	}
)

func setHelmHelp(p *Plugin) {
	p.Config.HelmCommand = []string{""}
}
func setDeleteEventCommand(p *Plugin) {
	upgrade := make([]string, 2)
	upgrade[0] = "delete"
	upgrade[1] = p.Config.Release

	p.Config.HelmCommand = upgrade
}

func setPushEventCommand(p *Plugin) {
	upgrade := make([]string, 2)
	upgrade[0] = "upgrade"
	upgrade[1] = "--install"

	if p.Config.Release != "" {
		upgrade = append(upgrade, p.Config.Release)
	}
	upgrade = append(upgrade, p.Config.Chart)
	if p.Config.Values != "" {
		upgrade = append(upgrade, "--set")
		upgrade = append(upgrade, p.Config.Values)
	}
	if p.Config.ValuesFiles != "" {
  	for _, valuesFile := range strings.Split(p.Config.ValuesFiles, ",") {
      upgrade = append(upgrade, "--values")
      upgrade = append(upgrade, valuesFile)
    }
	}
	if p.Config.Namespace != "" {
		upgrade = append(upgrade, "--namespace")
		upgrade = append(upgrade, p.Config.Namespace)
	}
	if p.Config.TillerNs != "" {
		upgrade = append(upgrade, "--tiller-namespace")
		upgrade = append(upgrade, p.Config.TillerNs)
	}
	if p.Config.DryRun {
		upgrade = append(upgrade, "--dry-run")
	}
	if p.Config.Debug {
		upgrade = append(upgrade, "--debug")
	}
	p.Config.HelmCommand = upgrade

}

func setHelmCommand(p *Plugin) {
	buildEvent := os.Getenv("DRONE_BUILD_EVENT")
	switch buildEvent {
	case "push":
		setPushEventCommand(p)
	case "delete":
		setDeleteEventCommand(p)
	default:
		setHelmHelp(p)
	}

}

func doHelmInit(p *Plugin) []string {
	init := make([]string, 1)
	init[0] = "init"
	if p.Config.TillerNs != "" {
		init = append(init, "--tiller-namespace")
		init = append(init, p.Config.TillerNs)
	}
	return init

}

// Exec default method
func (p *Plugin) Exec() error {
	resolveSecrets(p)
	if p.Config.APIServer == "" {
		return fmt.Errorf("Error: API Server is needed to deploy.")
	}
	if p.Config.Token == "" {
		return fmt.Errorf("Error: Token is needed to deploy.")
	}
	initialiseKubeconfig(&p.Config, KUBECONFIG, CONFIG)

	if p.Config.Debug {
		p.debug()
	}

	init := doHelmInit(p)
	err := runCommand(init)
	if err != nil {
		return fmt.Errorf("Error running helm command: " + strings.Join(init[:], " "))
	}
	setHelmCommand(p)

	if p.Config.Debug {
		log.Println("helm command: " + strings.Join(p.Config.HelmCommand[:], " "))
	}
	err = runCommand(p.Config.HelmCommand)
	if err != nil {
		return fmt.Errorf("Error running helm command: " + strings.Join(p.Config.HelmCommand[:], " "))
	}
	return nil
}

func initialiseKubeconfig(params *Config, source string, target string) error {
	t, _ := template.ParseFiles(source)
	f, err := os.Create(target)
	err = t.Execute(f, params)
	f.Close()
	return err
}

func runCommand(params []string) error {
	cmd := new(exec.Cmd)
	cmd = exec.Command(HELM_BIN, params...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err := cmd.Run()
	return err
}

func resolveSecrets(p *Plugin) {
	p.Config.Values = resolveEnvVar(p.Config.Values, p.Config.Prefix)
	p.Config.APIServer = resolveEnvVar("${API_SERVER}", p.Config.Prefix)
	p.Config.Token = resolveEnvVar("${KUBERNETES_TOKEN}", p.Config.Prefix)
}

// getEnvVars will return [${TAG} {TAG} TAG]
func getEnvVars(envvars string) [][]string {
	re := regexp.MustCompile(`\$(\{?(\w+)\}?)\.?`)
	extracted := re.FindAllStringSubmatch(envvars, -1)
	return extracted
}

func resolveEnvVar(key string, prefix string) string {
	envvars := getEnvVars(key)
	return replaceEnvvars(envvars, prefix, key)
}

func replaceEnvvars(envvars [][]string, prefix string, s string) string {
	for _, envvar := range envvars {
		envvarName := envvar[0]
		envvarKey := envvar[2]
		envval := os.Getenv(prefix + "_" + envvarKey)
		if envval == "" {
			envval = os.Getenv(envvarKey)
		}
		if strings.Contains(s, envvarName) {
			s = strings.Replace(s, envvarName, envval, -1)
		}
	}
	return s
}

func (p *Plugin) debug() {
	fmt.Println(p)
	// debug env vars
	for _, e := range os.Environ() {
		fmt.Println("-Var:--", e)
	}
	// debug plugin obj
	fmt.Printf("Api server: %s \n", p.Config.APIServer)
	fmt.Printf("Values: %s \n", p.Config.Values)
	fmt.Printf("Secrets: %s \n", p.Config.Secrets)
	fmt.Printf("ValuesFiles: %s \n", p.Config.ValuesFiles)

	kubeconfig, err := ioutil.ReadFile(KUBECONFIG)
	if err == nil {
		fmt.Println(string(kubeconfig))
	}
	config, err := ioutil.ReadFile(CONFIG)
	if err == nil {
		fmt.Println(string(config))
	}

}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"net/http"
	"time"

	ctxu "github.com/docker/distribution/context"
	"github.com/docker/distribution/registry/api/errcode"
	"github.com/docker/distribution/registry/auth"
	"github.com/docker/notary/tuf/signed"
	"github.com/gorilla/mux"
	"golang.org/x/net/context"
)

// ContextHandler defines an alterate HTTP handler interface which takes in
// a context for authorization and returns an HTTP application error.
type ContextHandler func(ctx context.Context, w http.ResponseWriter, r *http.Request) error

// rootHandler is an implementation of an HTTP request handler which handles
// authorization and calling out to the defined alternate http handler.
type rootHandler struct {
	handler ContextHandler
	auth    auth.AccessController
	actions []string
	context context.Context
	trust   signed.CryptoService
	//cachePool redis.Pool
}

// RootHandlerFactory creates a new rootHandler factory  using the given
// Context creator and authorizer.  The returned factory allows creating
// new rootHandlers from the alternate http handler contextHandler and
// a scope.
func RootHandlerFactory(auth auth.AccessController, ctx context.Context, trust signed.CryptoService) func(ContextHandler, ...string) *rootHandler {
	return func(handler ContextHandler, actions ...string) *rootHandler {
		return &rootHandler{
			handler: handler,
			auth:    auth,
			actions: actions,
			context: ctx,
			trust:   trust,
		}
	}
}

// ServeHTTP serves an HTTP request and implements the http.Handler interface.
func (root *rootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	ctx := ctxu.WithRequest(root.context, r)
	log := ctxu.GetRequestLogger(ctx)
	ctx, w = ctxu.WithResponseWriter(ctx, w)
	ctx = ctxu.WithLogger(ctx, log)
	ctx = context.WithValue(ctx, "repo", vars["imageName"])
	ctx = context.WithValue(ctx, "cryptoService", root.trust)

	defer func() {
		ctxu.GetResponseLogger(ctx).Info("response completed")
	}()

	if root.auth != nil {
		access := buildAccessRecords(vars["imageName"], root.actions...)
		var authCtx context.Context
		var err error
		if authCtx, err = root.auth.Authorized(ctx, access...); err != nil {
			if challenge, ok := err.(auth.Challenge); ok {
				// Let the challenge write the response.
				challenge.SetHeaders(w)

				if err := errcode.ServeJSON(w, errcode.ErrorCodeUnauthorized.WithDetail(access)); err != nil {
					log.Errorf("failed to serve challenge response: %s", err.Error())
				}
				return
			}
			errcode.ServeJSON(w, errcode.ErrorCodeUnauthorized)
			return
		}
		ctx = authCtx
	}
	if err := root.handler(ctx, w, r); err != nil {
		if httpErr, ok := err.(errcode.ErrorCoder); ok {
			// info level logging for non-5XX http errors
			httpErrCode := httpErr.ErrorCode().Descriptor().HTTPStatusCode
			if (httpErrCode < http.StatusOK || httpErrCode >= http.StatusMultipleChoices) && httpErrCode < http.StatusInternalServerError {
				log.Info(httpErr)
			} else if httpErrCode >= http.StatusInternalServerError {
				// error level logging for 5XX http errors
				log.Error(httpErr)
			}
		}
		e := errcode.ServeJSON(w, err)
		if e != nil {
			log.Error(e)
		}
		return
	}
}

func buildAccessRecords(repo string, actions ...string) []auth.Access {
	requiredAccess := make([]auth.Access, 0, len(actions))
	for _, action := range actions {
		requiredAccess = append(requiredAccess, auth.Access{
			Resource: auth.Resource{
				Type: "repository",
				Name: repo,
			},
			Action: action,
		})
	}
	return requiredAccess
}

// CacheControlConfig is an interface for something that knows how to set cache
// control headers
type CacheControlConfig interface {
	// SetHeaders will actually set the cache control headers on a Headers object
	SetHeaders(headers http.Header)
}

// NewCacheControlConfig returns CacheControlConfig interface for either setting
// cache control or disabling cache control entirely
func NewCacheControlConfig(maxAgeInSeconds int, mustRevalidate bool) CacheControlConfig {
	if maxAgeInSeconds > 0 {
		return PublicCacheControl{MustReValidate: mustRevalidate, MaxAgeInSeconds: maxAgeInSeconds}
	}
	return NoCacheControl{}
}

// PublicCacheControl is a set of options that we will set to enable cache control
type PublicCacheControl struct {
	MustReValidate  bool
	MaxAgeInSeconds int
}

// SetHeaders sets the public headers with an optional must-revalidate header
func (p PublicCacheControl) SetHeaders(headers http.Header) {
	cacheControlValue := fmt.Sprintf("public, max-age=%v, s-maxage=%v",
		p.MaxAgeInSeconds, p.MaxAgeInSeconds)

	if p.MustReValidate {
		cacheControlValue = fmt.Sprintf("%s, must-revalidate", cacheControlValue)
	}
	headers.Set("Cache-Control", cacheControlValue)
	// delete the Pragma directive, because the only valid value in HTTP is
	// "no-cache"
	headers.Del("Pragma")
	if headers.Get("Last-Modified") == "" {
		SetLastModifiedHeader(headers, time.Time{})
	}
}

// NoCacheControl is an object which represents a directive to cache nothing
type NoCacheControl struct{}

// SetHeaders sets the public headers cache-control headers and pragma to no-cache
func (n NoCacheControl) SetHeaders(headers http.Header) {
	headers.Set("Cache-Control", "max-age=0, no-cache, no-store")
	headers.Set("Pragma", "no-cache")
}

// cacheControlResponseWriter wraps an existing response writer, and if Write is
// called, will try to set the cache control headers if it can
type cacheControlResponseWriter struct {
	http.ResponseWriter
	config     CacheControlConfig
	statusCode int
}

// WriteHeader stores the header before writing it, so we can tell if it's been set
// to a non-200 status code
func (c *cacheControlResponseWriter) WriteHeader(statusCode int) {
	c.statusCode = statusCode
	c.ResponseWriter.WriteHeader(statusCode)
}

// Write will set the cache headers if they haven't already been set and if the status
// code has either not been set or set to 200
func (c *cacheControlResponseWriter) Write(data []byte) (int, error) {
	if c.statusCode == http.StatusOK || c.statusCode == 0 {
		headers := c.ResponseWriter.Header()
		if headers.Get("Cache-Control") == "" {
			c.config.SetHeaders(headers)
		}
	}
	return c.ResponseWriter.Write(data)
}

type cacheControlHandler struct {
	http.Handler
	config CacheControlConfig
}

func (c cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	c.Handler.ServeHTTP(&cacheControlResponseWriter{ResponseWriter: w, config: c.config}, r)
}

// WrapWithCacheHandler wraps another handler in one that can add cache control headers
// given a 200 response
func WrapWithCacheHandler(ccc CacheControlConfig, handler http.Handler) http.Handler {
	if ccc != nil {
		return cacheControlHandler{Handler: handler, config: ccc}
	}
	return handler
}

// SetLastModifiedHeader takes a time and uses it to set the LastModified header using
// the right date format
func SetLastModifiedHeader(headers http.Header, lmt time.Time) {
	headers.Set("Last-Modified", lmt.Format(time.RFC1123))
}
<commit_msg>Simplify logging on httpErr<commit_after>package utils

import (
	"fmt"
	"net/http"
	"time"

	ctxu "github.com/docker/distribution/context"
	"github.com/docker/distribution/registry/api/errcode"
	"github.com/docker/distribution/registry/auth"
	"github.com/docker/notary/tuf/signed"
	"github.com/gorilla/mux"
	"golang.org/x/net/context"
)

// ContextHandler defines an alterate HTTP handler interface which takes in
// a context for authorization and returns an HTTP application error.
type ContextHandler func(ctx context.Context, w http.ResponseWriter, r *http.Request) error

// rootHandler is an implementation of an HTTP request handler which handles
// authorization and calling out to the defined alternate http handler.
type rootHandler struct {
	handler ContextHandler
	auth    auth.AccessController
	actions []string
	context context.Context
	trust   signed.CryptoService
	//cachePool redis.Pool
}

// RootHandlerFactory creates a new rootHandler factory  using the given
// Context creator and authorizer.  The returned factory allows creating
// new rootHandlers from the alternate http handler contextHandler and
// a scope.
func RootHandlerFactory(auth auth.AccessController, ctx context.Context, trust signed.CryptoService) func(ContextHandler, ...string) *rootHandler {
	return func(handler ContextHandler, actions ...string) *rootHandler {
		return &rootHandler{
			handler: handler,
			auth:    auth,
			actions: actions,
			context: ctx,
			trust:   trust,
		}
	}
}

// ServeHTTP serves an HTTP request and implements the http.Handler interface.
func (root *rootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	ctx := ctxu.WithRequest(root.context, r)
	log := ctxu.GetRequestLogger(ctx)
	ctx, w = ctxu.WithResponseWriter(ctx, w)
	ctx = ctxu.WithLogger(ctx, log)
	ctx = context.WithValue(ctx, "repo", vars["imageName"])
	ctx = context.WithValue(ctx, "cryptoService", root.trust)

	defer func() {
		ctxu.GetResponseLogger(ctx).Info("response completed")
	}()

	if root.auth != nil {
		access := buildAccessRecords(vars["imageName"], root.actions...)
		var authCtx context.Context
		var err error
		if authCtx, err = root.auth.Authorized(ctx, access...); err != nil {
			if challenge, ok := err.(auth.Challenge); ok {
				// Let the challenge write the response.
				challenge.SetHeaders(w)

				if err := errcode.ServeJSON(w, errcode.ErrorCodeUnauthorized.WithDetail(access)); err != nil {
					log.Errorf("failed to serve challenge response: %s", err.Error())
				}
				return
			}
			errcode.ServeJSON(w, errcode.ErrorCodeUnauthorized)
			return
		}
		ctx = authCtx
	}
	if err := root.handler(ctx, w, r); err != nil {
		if httpErr, ok := err.(errcode.ErrorCoder); ok {
			// info level logging for non-5XX http errors
			httpErrCode := httpErr.ErrorCode().Descriptor().HTTPStatusCode
			if httpErrCode >= http.StatusInternalServerError {
				// error level logging for 5XX http errors
				log.Error(httpErr)
			} else {
				log.Info(httpErr)
			}
		}
		e := errcode.ServeJSON(w, err)
		if e != nil {
			log.Error(e)
		}
		return
	}
}

func buildAccessRecords(repo string, actions ...string) []auth.Access {
	requiredAccess := make([]auth.Access, 0, len(actions))
	for _, action := range actions {
		requiredAccess = append(requiredAccess, auth.Access{
			Resource: auth.Resource{
				Type: "repository",
				Name: repo,
			},
			Action: action,
		})
	}
	return requiredAccess
}

// CacheControlConfig is an interface for something that knows how to set cache
// control headers
type CacheControlConfig interface {
	// SetHeaders will actually set the cache control headers on a Headers object
	SetHeaders(headers http.Header)
}

// NewCacheControlConfig returns CacheControlConfig interface for either setting
// cache control or disabling cache control entirely
func NewCacheControlConfig(maxAgeInSeconds int, mustRevalidate bool) CacheControlConfig {
	if maxAgeInSeconds > 0 {
		return PublicCacheControl{MustReValidate: mustRevalidate, MaxAgeInSeconds: maxAgeInSeconds}
	}
	return NoCacheControl{}
}

// PublicCacheControl is a set of options that we will set to enable cache control
type PublicCacheControl struct {
	MustReValidate  bool
	MaxAgeInSeconds int
}

// SetHeaders sets the public headers with an optional must-revalidate header
func (p PublicCacheControl) SetHeaders(headers http.Header) {
	cacheControlValue := fmt.Sprintf("public, max-age=%v, s-maxage=%v",
		p.MaxAgeInSeconds, p.MaxAgeInSeconds)

	if p.MustReValidate {
		cacheControlValue = fmt.Sprintf("%s, must-revalidate", cacheControlValue)
	}
	headers.Set("Cache-Control", cacheControlValue)
	// delete the Pragma directive, because the only valid value in HTTP is
	// "no-cache"
	headers.Del("Pragma")
	if headers.Get("Last-Modified") == "" {
		SetLastModifiedHeader(headers, time.Time{})
	}
}

// NoCacheControl is an object which represents a directive to cache nothing
type NoCacheControl struct{}

// SetHeaders sets the public headers cache-control headers and pragma to no-cache
func (n NoCacheControl) SetHeaders(headers http.Header) {
	headers.Set("Cache-Control", "max-age=0, no-cache, no-store")
	headers.Set("Pragma", "no-cache")
}

// cacheControlResponseWriter wraps an existing response writer, and if Write is
// called, will try to set the cache control headers if it can
type cacheControlResponseWriter struct {
	http.ResponseWriter
	config     CacheControlConfig
	statusCode int
}

// WriteHeader stores the header before writing it, so we can tell if it's been set
// to a non-200 status code
func (c *cacheControlResponseWriter) WriteHeader(statusCode int) {
	c.statusCode = statusCode
	c.ResponseWriter.WriteHeader(statusCode)
}

// Write will set the cache headers if they haven't already been set and if the status
// code has either not been set or set to 200
func (c *cacheControlResponseWriter) Write(data []byte) (int, error) {
	if c.statusCode == http.StatusOK || c.statusCode == 0 {
		headers := c.ResponseWriter.Header()
		if headers.Get("Cache-Control") == "" {
			c.config.SetHeaders(headers)
		}
	}
	return c.ResponseWriter.Write(data)
}

type cacheControlHandler struct {
	http.Handler
	config CacheControlConfig
}

func (c cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	c.Handler.ServeHTTP(&cacheControlResponseWriter{ResponseWriter: w, config: c.config}, r)
}

// WrapWithCacheHandler wraps another handler in one that can add cache control headers
// given a 200 response
func WrapWithCacheHandler(ccc CacheControlConfig, handler http.Handler) http.Handler {
	if ccc != nil {
		return cacheControlHandler{Handler: handler, config: ccc}
	}
	return handler
}

// SetLastModifiedHeader takes a time and uses it to set the LastModified header using
// the right date format
func SetLastModifiedHeader(headers http.Header, lmt time.Time) {
	headers.Set("Last-Modified", lmt.Format(time.RFC1123))
}
<|endoftext|>