<commit_before>package lba

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"sync"

	"github.com/garyburd/redigo/redis"
)

// MetaRedisProvider is used by the LBA,
// to retreive a Redis Meta Connection
type MetaRedisProvider interface {
	MetaRedisConnection() (redis.Conn, error)
}

//NewLBA creates a new LBA
func NewLBA(volumeID string, blockCount, cacheLimitInBytes int64, provider MetaRedisProvider) (lba *LBA, err error) {
	if provider == nil {
		return nil, errors.New("NewLBA requires a non-nil MetaRedisProvider")
	}

	muxCount := blockCount / NumberOfRecordsPerLBAShard
	if blockCount%NumberOfRecordsPerLBAShard > 0 {
		muxCount++
	}

	lba = &LBA{
		provider: provider,
		volumeID: volumeID,
		shardMux: make([]sync.Mutex, muxCount),
	}

	lba.cache, err = newShardCache(cacheLimitInBytes, lba.onCacheEviction)

	return
}

// LBA implements the functionality to lookup block keys through the logical block index.
// The data is persisted to an external metadataserver in shards of n keys,
// where n = NumberOfRecordsPerLBAShard.
type LBA struct {
	cache *shardCache

	// One mutex per shard, allows us to only lock
	// on a per-shard basis. Even with 65k block, that's still only a ~500 element mutex array.
	// We stil need to lock on a per-shard basis,
	// as otherwise we might have a race condition where for example
	// 2 operations might create a new shard, and thus we would miss an operation.
	shardMux []sync.Mutex

	provider MetaRedisProvider
	volumeID string
}

//Set the content hash for a specific block.
// When a key is updated, the shard containing this blockindex is marked as dirty and will be
// stored in the external metadataserver when Flush is called.
func (lba *LBA) Set(blockIndex int64, h Hash) (err error) {
	//Fetch the appropriate shard
	shard, err := func(shardIndex int64) (shard *shard, err error) {
		lba.shardMux[shardIndex].Lock()
		defer lba.shardMux[shardIndex].Unlock()

		shard, err = lba.getShard(shardIndex)
		if err != nil {
			return
		}
		if shard == nil {
			shard = newShard()
			// store the new shard in the cache,
			// otherwise it will be forgotten...
			lba.cache.Add(shardIndex, shard)
		}

		return
	}(blockIndex / NumberOfRecordsPerLBAShard)

	if err != nil {
		return
	}

	//Update the hash
	hashIndex := blockIndex % NumberOfRecordsPerLBAShard
	shard.Set(hashIndex, h)

	return
}

//Delete the content hash for a specific block.
// When a key is updated, the shard containing this blockindex is marked as dirty and will be
// stored in the external metadaserver when Flush is called
// Deleting means actually that the nilhash will be set for this blockindex.
func (lba *LBA) Delete(blockIndex int64) (err error) {
	err = lba.Set(blockIndex, nil)
	return
}

//Get returns the hash for a block, nil if no hash registered
// If the shard containing this blockindex is not present, it is fetched from the external metadaserver
func (lba *LBA) Get(blockIndex int64) (h Hash, err error) {
	shard, err := func(shardIndex int64) (*shard, error) {
		lba.shardMux[shardIndex].Lock()
		defer lba.shardMux[shardIndex].Unlock()

		return lba.getShard(shardIndex)
	}(blockIndex / NumberOfRecordsPerLBAShard)

	if err != nil || shard == nil {
		return
	}

	// get the hash
	hashIndex := blockIndex % NumberOfRecordsPerLBAShard
	h = shard.Get(hashIndex)

	return
}

//Flush stores all dirty shards to the external metadaserver
func (lba *LBA) Flush() (err error) {
	err = lba.storeCacheInExternalStorage()
	return
}

func (lba *LBA) getShard(index int64) (shard *shard, err error) {
	shard, ok := lba.cache.Get(index)
	if !ok {
		shard, err = lba.getShardFromExternalStorage(index)
		if err != nil {
			return
		}

		if shard != nil {
			lba.cache.Add(index, shard)
		}
	}

	return
}

// in case a shard gets evicted from cache,
// this method will be called, and we'll serialize the shard immediately,
// unless it isn't dirty
func (lba *LBA) onCacheEviction(index int64, shard *shard) {
	if !shard.Dirty() {
		return
	}

	var err error

	// the given shard can be nil in case it was deleted by the user,
	// in that case we will remove the shard from the external storage as well
	// otherwise we serialize the shard before it gets thrown into the void
	if shard != nil {
		err = lba.storeShardInExternalStorage(index, shard)
	} else {
		err = lba.deleteShardFromExternalStorage(index)
	}

	if err != nil {
		log.Printf("[ERROR] error during eviction of shard %d: %s", index, err)
	}
}

func (lba *LBA) getShardFromExternalStorage(index int64) (shard *shard, err error) {
	conn, err := lba.provider.MetaRedisConnection()
	if err != nil {
		return
	}
	defer conn.Close()
	reply, err := conn.Do("HGET", lba.volumeID, index)
	if err != nil || reply == nil {
		return
	}

	shardBytes, err := redis.Bytes(reply, err)
	if err != nil {
		return
	}

	shard, err = shardFromBytes(shardBytes)
	return
}

func (lba *LBA) storeCacheInExternalStorage() (err error) {
	conn, err := lba.provider.MetaRedisConnection()
	if err != nil {
		return
	}
	defer conn.Close()

	if err = conn.Send("MULTI"); err != nil {
		return
	}

	lba.cache.Serialize(func(index int64, bytes []byte) (err error) {
		if bytes != nil {
			err = conn.Send("HSET", lba.volumeID, index, bytes)
		} else {
			err = conn.Send("HDEL", lba.volumeID, index)
		}
		return
	})

	// Write all sets in output buffer to Redis at once
	_, err = conn.Do("EXEC")
	if err != nil {
		// no need to evict, already serialized them
		evict := false
		// clear cache, as we serialized them all
		lba.cache.Clear(evict)
	}
	return
}

func (lba *LBA) storeShardInExternalStorage(index int64, shard *shard) (err error) {
	if !shard.Dirty() {
		return // only store a dirty shard
	}

	var buffer bytes.Buffer
	if err = shard.Write(&buffer); err != nil {
		err = fmt.Errorf("couldn't serialize evicted shard %d: %s", index, err)
		return
	}

	conn, err := lba.provider.MetaRedisConnection()
	if err != nil {
		return
	}
	defer conn.Close()

	_, err = conn.Do("HSET", lba.volumeID, index, buffer.Bytes())
	if err != nil {
		shard.UnsetDirty()
	}

	return
}

func (lba *LBA) deleteShardFromExternalStorage(index int64) (err error) {
	conn, err := lba.provider.MetaRedisConnection()
	if err != nil {
		return
	}
	defer conn.Close()

	_, err = conn.Do("HDEL", lba.volumeID, index)

	return
}
<commit_msg>replace multi/exec LBA call with Flush<commit_after>package lba

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"sync"

	"github.com/garyburd/redigo/redis"
)

// MetaRedisProvider is used by the LBA,
// to retreive a Redis Meta Connection
type MetaRedisProvider interface {
	MetaRedisConnection() (redis.Conn, error)
}

//NewLBA creates a new LBA
func NewLBA(volumeID string, blockCount, cacheLimitInBytes int64, provider MetaRedisProvider) (lba *LBA, err error) {
	if provider == nil {
		return nil, errors.New("NewLBA requires a non-nil MetaRedisProvider")
	}

	muxCount := blockCount / NumberOfRecordsPerLBAShard
	if blockCount%NumberOfRecordsPerLBAShard > 0 {
		muxCount++
	}

	lba = &LBA{
		provider: provider,
		volumeID: volumeID,
		shardMux: make([]sync.Mutex, muxCount),
	}

	lba.cache, err = newShardCache(cacheLimitInBytes, lba.onCacheEviction)

	return
}

// LBA implements the functionality to lookup block keys through the logical block index.
// The data is persisted to an external metadataserver in shards of n keys,
// where n = NumberOfRecordsPerLBAShard.
type LBA struct {
	cache *shardCache

	// One mutex per shard, allows us to only lock
	// on a per-shard basis. Even with 65k block, that's still only a ~500 element mutex array.
	// We stil need to lock on a per-shard basis,
	// as otherwise we might have a race condition where for example
	// 2 operations might create a new shard, and thus we would miss an operation.
	shardMux []sync.Mutex

	provider MetaRedisProvider
	volumeID string
}

//Set the content hash for a specific block.
// When a key is updated, the shard containing this blockindex is marked as dirty and will be
// stored in the external metadataserver when Flush is called.
func (lba *LBA) Set(blockIndex int64, h Hash) (err error) {
	//Fetch the appropriate shard
	shard, err := func(shardIndex int64) (shard *shard, err error) {
		lba.shardMux[shardIndex].Lock()
		defer lba.shardMux[shardIndex].Unlock()

		shard, err = lba.getShard(shardIndex)
		if err != nil {
			return
		}
		if shard == nil {
			shard = newShard()
			// store the new shard in the cache,
			// otherwise it will be forgotten...
			lba.cache.Add(shardIndex, shard)
		}

		return
	}(blockIndex / NumberOfRecordsPerLBAShard)

	if err != nil {
		return
	}

	//Update the hash
	hashIndex := blockIndex % NumberOfRecordsPerLBAShard
	shard.Set(hashIndex, h)

	return
}

//Delete the content hash for a specific block.
// When a key is updated, the shard containing this blockindex is marked as dirty and will be
// stored in the external metadaserver when Flush is called
// Deleting means actually that the nilhash will be set for this blockindex.
func (lba *LBA) Delete(blockIndex int64) (err error) {
	err = lba.Set(blockIndex, nil)
	return
}

//Get returns the hash for a block, nil if no hash registered
// If the shard containing this blockindex is not present, it is fetched from the external metadaserver
func (lba *LBA) Get(blockIndex int64) (h Hash, err error) {
	shard, err := func(shardIndex int64) (*shard, error) {
		lba.shardMux[shardIndex].Lock()
		defer lba.shardMux[shardIndex].Unlock()

		return lba.getShard(shardIndex)
	}(blockIndex / NumberOfRecordsPerLBAShard)

	if err != nil || shard == nil {
		return
	}

	// get the hash
	hashIndex := blockIndex % NumberOfRecordsPerLBAShard
	h = shard.Get(hashIndex)

	return
}

//Flush stores all dirty shards to the external metadaserver
func (lba *LBA) Flush() (err error) {
	err = lba.storeCacheInExternalStorage()
	return
}

func (lba *LBA) getShard(index int64) (shard *shard, err error) {
	shard, ok := lba.cache.Get(index)
	if !ok {
		shard, err = lba.getShardFromExternalStorage(index)
		if err != nil {
			return
		}

		if shard != nil {
			lba.cache.Add(index, shard)
		}
	}

	return
}

// in case a shard gets evicted from cache,
// this method will be called, and we'll serialize the shard immediately,
// unless it isn't dirty
func (lba *LBA) onCacheEviction(index int64, shard *shard) {
	if !shard.Dirty() {
		return
	}

	var err error

	// the given shard can be nil in case it was deleted by the user,
	// in that case we will remove the shard from the external storage as well
	// otherwise we serialize the shard before it gets thrown into the void
	if shard != nil {
		err = lba.storeShardInExternalStorage(index, shard)
	} else {
		err = lba.deleteShardFromExternalStorage(index)
	}

	if err != nil {
		log.Printf("[ERROR] error during eviction of shard %d: %s", index, err)
	}
}

func (lba *LBA) getShardFromExternalStorage(index int64) (shard *shard, err error) {
	conn, err := lba.provider.MetaRedisConnection()
	if err != nil {
		return
	}
	defer conn.Close()
	reply, err := conn.Do("HGET", lba.volumeID, index)
	if err != nil || reply == nil {
		return
	}

	shardBytes, err := redis.Bytes(reply, err)
	if err != nil {
		return
	}

	shard, err = shardFromBytes(shardBytes)
	return
}

func (lba *LBA) storeCacheInExternalStorage() (err error) {
	conn, err := lba.provider.MetaRedisConnection()
	if err != nil {
		return
	}
	defer conn.Close()

	lba.cache.Serialize(func(index int64, bytes []byte) (err error) {
		if bytes != nil {
			err = conn.Send("HSET", lba.volumeID, index, bytes)
		} else {
			err = conn.Send("HDEL", lba.volumeID, index)
		}
		return
	})

	// Write all sets in output buffer to Redis at once
	err = conn.Flush()
	if err != nil {
		// no need to evict, already serialized them
		evict := false
		// clear cache, as we serialized them all
		lba.cache.Clear(evict)
	}
	return
}

func (lba *LBA) storeShardInExternalStorage(index int64, shard *shard) (err error) {
	if !shard.Dirty() {
		return // only store a dirty shard
	}

	var buffer bytes.Buffer
	if err = shard.Write(&buffer); err != nil {
		err = fmt.Errorf("couldn't serialize evicted shard %d: %s", index, err)
		return
	}

	conn, err := lba.provider.MetaRedisConnection()
	if err != nil {
		return
	}
	defer conn.Close()

	_, err = conn.Do("HSET", lba.volumeID, index, buffer.Bytes())
	if err != nil {
		shard.UnsetDirty()
	}

	return
}

func (lba *LBA) deleteShardFromExternalStorage(index int64) (err error) {
	conn, err := lba.provider.MetaRedisConnection()
	if err != nil {
		return
	}
	defer conn.Close()

	_, err = conn.Do("HDEL", lba.volumeID, index)

	return
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "net/http"
    "strings"
    "google.golang.org/appengine"
)

func init() {
    http.HandleFunc("/", IndexHandler)
}

func IndexHandler(w http.ResponseWriter, r *http.Request) {
    switch r.URL.Path {
        case "/":
            switch r.Method {
                case http.MethodGet:
                    GetIndex(w, r)
                default:
                    NotFound(w, r)
            }
        default:
            NotFound(w, r)
    }
}

func GetIndex(w http.ResponseWriter, r *http.Request) {
    w.WriteHeader(http.StatusOK)
    w.Header().Set("Content-Type", "text/plain")
    fmt.Fprint(w, strings.Split(r.RemoteAddr, ":")[0])
}

func NotFound(w http.ResponseWriter, r *http.Request) {
    w.WriteHeader(http.StatusNotFound)
    w.Header().Set("Content-Type", "text/plain")
    fmt.Fprintf(w, "%d Not Found", http.StatusNotFound)
}

func main() {
    appengine.Main()
}
<commit_msg>Add inline documentation<commit_after>package main

import (
    "fmt"
    "net/http"
    "strings"
    "google.golang.org/appengine"
)

// Trivially routes requests to the correct response function.
func IndexHandler(w http.ResponseWriter, r *http.Request) {
    switch r.URL.Path {
        case "/":
            switch r.Method {
                case http.MethodGet:
                    GetIndex(w, r)
                default:
                    NotFound(w, r)
            }
        default:
            NotFound(w, r)
    }
}

// Returns the client's IP address.
func GetIndex(w http.ResponseWriter, r *http.Request) {
    w.WriteHeader(http.StatusOK)
    w.Header().Set("Content-Type", "text/plain")

    // RemoteAddr is formatted as host:port, so we just trim off the port here
    // and return the IP.
    fmt.Fprint(w, strings.Split(r.RemoteAddr, ":")[0])
}

// Returns a 404 Not Found page.
func NotFound(w http.ResponseWriter, r *http.Request) {
    w.WriteHeader(http.StatusNotFound)
    w.Header().Set("Content-Type", "text/plain")
    fmt.Fprintf(w, "%d Not Found", http.StatusNotFound)
}

// appengine.Main() expects packages to register HTTP handlers in their init()
// functions.
func init() {
    http.HandleFunc("/", IndexHandler)
}

func main() {
    // Starts listening on port 8080 (or $PORT), and never returns.
    // https://godoc.org/google.golang.org/appengine#Main
    appengine.Main()
}
<|endoftext|>


<commit_before>package k8s_test

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/onsi/gomega/gexec"

	. "github.com/concourse/concourse/topgun"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Kubernetes credential management", func() {
	var (
		proxySession *gexec.Session
		releaseName  string
		atcEndpoint  string
		namespace    string
		username     = "test"
		password     = "test"
	)

	BeforeEach(func() {
		releaseName = fmt.Sprintf("topgun-k8s-cm-%d-%d", GinkgoRandomSeed(), GinkgoParallelNode())
		namespace = releaseName

		deployConcourseChart(releaseName, "--set=worker.replicas=1")

		waitAllPodsInNamespaceToBeReady(namespace)

		By("Creating the web proxy")
		proxySession, atcEndpoint = startPortForwarding(namespace, "service/" + releaseName+"-web", "8080")

		By("Logging in")
		fly.Login(username, password, atcEndpoint)

		By("Waiting for a running worker")
		Eventually(func() []Worker {
			return getRunningWorkers(fly.GetWorkers())
		}, 2*time.Minute, 10*time.Second).
			ShouldNot(HaveLen(0))
	})

	Context("/api/v1/info/creds", func() {
		var parsedResponse struct {
			Kubernetes struct {
				ConfigPath      string `json:"config_path"`
				InClusterConfig bool   `json:"in_cluster_config"`
				NamespaceConfig string `json:"namespace_config"`
			} `json:"kubernetes"`
		}

		JustBeforeEach(func() {
			token, err := FetchToken(atcEndpoint, username, password)
			Expect(err).ToNot(HaveOccurred())

			body, err := RequestCredsInfo(atcEndpoint, token.AccessToken)
			Expect(err).ToNot(HaveOccurred())

			err = json.Unmarshal(body, &parsedResponse)
			Expect(err).ToNot(HaveOccurred())
		})

		It("Contains kubernetes config", func() {
			Expect(parsedResponse.Kubernetes.ConfigPath).To(BeEmpty())
			Expect(parsedResponse.Kubernetes.InClusterConfig).To(BeTrue())
			Expect(parsedResponse.Kubernetes.NamespaceConfig).To(Equal(releaseName + "-"))
		})
	})

	Context("Consuming per-team k8s secrets", func() {
		BeforeEach(func() {
			// ((foo)) --> bar
			createCredentialSecret(releaseName, "foo", "main", map[string]string{"value": "bar"})

			// ((caz.baz)) --> zaz
			createCredentialSecret(releaseName, "caz", "main", map[string]string{"baz": "zaz"})

			fly.Run("set-pipeline", "-n", "-c", "../pipelines/minimal-credential-management.yml", "-p", "pipeline")
			session := fly.Start("get-pipeline", "-p", "pipeline")
			Wait(session)

			Expect(string(session.Out.Contents())).ToNot(ContainSubstring("bar"))
			Expect(string(session.Out.Contents())).ToNot(ContainSubstring("zaz"))

			fly.Run("unpause-pipeline", "-p", "pipeline")
		})

		It("Gets credentials set by consuming k8s secrets", func() {
			session := fly.Start("trigger-job", "-j", "pipeline/unit", "-w")
			Wait(session)

			Expect(string(session.Out.Contents())).To(ContainSubstring("bar"))
			Expect(string(session.Out.Contents())).To(ContainSubstring("zaz"))
		})
	})

	AfterEach(func() {
		helmDestroy(releaseName)
		Wait(proxySession.Interrupt())
		Wait(Start(nil, "kubectl", "delete", "namespace", namespace, "--wait=false"))
	})

})

func createCredentialSecret(releaseName, secretName, team string, kv map[string]string) {
	args := []string{
		"create",
		"secret",
		"generic",
		secretName,
		"--namespace=" + releaseName + "-" + team,
	}

	for key, value := range kv {
		args = append(args, "--from-literal="+key+"="+value)
	}

	Wait(Start(nil, "kubectl", args...))
}
<commit_msg>topgun/k8s: Test external team namespaces<commit_after>package k8s_test

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/onsi/gomega/gexec"

	. "github.com/concourse/concourse/topgun"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Kubernetes credential management", func() {
	var (
		proxySession *gexec.Session
		releaseName  string
		atcEndpoint  string
		namespace    string
		username     = "test"
		password     = "test"
		extraArgs    []string
	)

	BeforeEach(func(){
		releaseName = fmt.Sprintf("topgun-k8s-cm-%d-%d", GinkgoRandomSeed(), GinkgoParallelNode())
		namespace = releaseName
	})

	JustBeforeEach(func() {

		deployConcourseChart(releaseName, append([]string{"--set=worker.replicas=1"}, extraArgs...)...)

		waitAllPodsInNamespaceToBeReady(namespace)

		By("Creating the web proxy")
		proxySession, atcEndpoint = startPortForwarding(namespace, "service/" + releaseName+"-web", "8080")

		By("Logging in")
		fly.Login(username, password, atcEndpoint)

		By("Waiting for a running worker")
		Eventually(func() []Worker {
			return getRunningWorkers(fly.GetWorkers())
		}, 2*time.Minute, 10*time.Second).
			ShouldNot(HaveLen(0))
	})

	Context("/api/v1/info/creds", func() {
		var parsedResponse struct {
			Kubernetes struct {
				ConfigPath      string `json:"config_path"`
				InClusterConfig bool   `json:"in_cluster_config"`
				NamespaceConfig string `json:"namespace_config"`
			} `json:"kubernetes"`
		}

		JustBeforeEach(func() {
			token, err := FetchToken(atcEndpoint, username, password)
			Expect(err).ToNot(HaveOccurred())

			body, err := RequestCredsInfo(atcEndpoint, token.AccessToken)
			Expect(err).ToNot(HaveOccurred())

			err = json.Unmarshal(body, &parsedResponse)
			Expect(err).ToNot(HaveOccurred())
		})

		It("Contains kubernetes config", func() {
			Expect(parsedResponse.Kubernetes.ConfigPath).To(BeEmpty())
			Expect(parsedResponse.Kubernetes.InClusterConfig).To(BeTrue())
			Expect(parsedResponse.Kubernetes.NamespaceConfig).To(Equal(releaseName + "-"))
		})
	})

	Context("Consuming per-team k8s secrets", func() {
		JustBeforeEach(func() {
			// ((foo)) --> bar
			createCredentialSecret(releaseName, "foo", "main", map[string]string{"value": "bar"})

			// ((caz.baz)) --> zaz
			createCredentialSecret(releaseName, "caz", "main", map[string]string{"baz": "zaz"})

			fly.Run("set-pipeline", "-n", "-c", "../pipelines/minimal-credential-management.yml", "-p", "pipeline")
			session := fly.Start("get-pipeline", "-p", "pipeline")
			Wait(session)

			Expect(string(session.Out.Contents())).ToNot(ContainSubstring("bar"))
			Expect(string(session.Out.Contents())).ToNot(ContainSubstring("zaz"))

			fly.Run("unpause-pipeline", "-p", "pipeline")
		})

		It("Gets credentials set by consuming k8s secrets", func() {
			session := fly.Start("trigger-job", "-j", "pipeline/unit", "-w")
			Wait(session)

			Expect(string(session.Out.Contents())).To(ContainSubstring("bar"))
			Expect(string(session.Out.Contents())).To(ContainSubstring("zaz"))
		})
	})

	Context("consuming creds from pre-created namespace for a team", func() {
		BeforeEach(func() {
			Run(nil, "kubectl", "create", "namespace", releaseName+"-main")
			extraArgs = []string{
				"--set=concourse.web.kubernetes.createTeamNamespaces=false",
			}
		})
		JustBeforeEach(func() {
			// ((foo)) --> bar
			createCredentialSecret(releaseName, "foo", "main", map[string]string{"value": "bar"})

			// ((caz.baz)) --> zaz
			createCredentialSecret(releaseName, "caz", "main", map[string]string{"baz": "zaz"})

			fly.Run("set-pipeline", "-n", "-c", "../pipelines/minimal-credential-management.yml", "-p", "pipeline")
			session := fly.Start("get-pipeline", "-p", "pipeline")
			Wait(session)

			Expect(string(session.Out.Contents())).ToNot(ContainSubstring("bar"))
			Expect(string(session.Out.Contents())).ToNot(ContainSubstring("zaz"))

			fly.Run("unpause-pipeline", "-p", "pipeline")
		})

		It("Gets credentials set by consuming k8s secrets", func() {
			session := fly.Start("trigger-job", "-j", "pipeline/unit", "-w")
			Wait(session)

			Expect(string(session.Out.Contents())).To(ContainSubstring("bar"))
			Expect(string(session.Out.Contents())).To(ContainSubstring("zaz"))
		})

		AfterEach(func(){
			Run(nil, "kubectl", "delete", "namespace", releaseName+"-main", "--wait=false")
		})
	})

	AfterEach(func() {
		helmDestroy(releaseName)
		Wait(proxySession.Interrupt())
		Run(nil, "kubectl", "delete", "namespace", namespace, "--wait=false")
	})

})

func createCredentialSecret(releaseName, secretName, team string, kv map[string]string) {
	args := []string{
		"create",
		"secret",
		"generic",
		secretName,
		"--namespace=" + releaseName + "-" + team,
	}

	for key, value := range kv {
		args = append(args, "--from-literal="+key+"="+value)
	}

	Run(nil, "kubectl", args...)
}
<|endoftext|>


<commit_before>package main

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import (
	"crypto/tls"
	"database/sql"
	"fmt"
	"net/http"
	"net/http/httputil"
)

// Routes returns the routes, and a catchall route for when no route matches.
func Routes(d ServerData) ([]Route, http.Handler, error) {
	rd, err := routeData(d)
	if err != nil {
		return nil, nil, err
	}
	return []Route{
		{1.2, http.MethodGet, "cdns/{cdn}/configs/monitoring$", wrapHeaders(wrapAuth(monitoringHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, MonitoringPrivLevel))},
		{1.2, http.MethodGet, "cdns/{cdn}/configs/monitoring.json$", wrapHeaders(wrapAuth(monitoringHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, MonitoringPrivLevel))},
		{1.2, http.MethodGet, "servers$", wrapHeaders(wrapAuthWithData(serversHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, ServersPrivLevel))},
		{1.2, http.MethodGet, "servers.json$", wrapHeaders(wrapAuthWithData(serversHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, ServersPrivLevel))},
		{1.2, http.MethodGet, "cdns$", wrapHeaders(wrapAuthWithData(cdnsHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, CdnsPrivLevel))},
		{1.2, http.MethodGet, "cdns.json$", wrapHeaders(wrapAuthWithData(cdnsHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, CdnsPrivLevel))},
	}, rootHandler(d), nil
}

type RouteData struct {
	PrivLevelStmt *sql.Stmt
}

func routeData(d ServerData) (RouteData, error) {
	rd := RouteData{}
	err := error(nil)

	if rd.PrivLevelStmt, err = preparePrivLevelStmt(d.DB); err != nil {
		return rd, fmt.Errorf("Error preparing db priv level query: ", err)
	}

	return rd, nil
}

// getRootHandler returns the / handler for the service, which reverse-proxies the old Perl Traffic Ops
func rootHandler(d ServerData) http.Handler {
	// debug
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	rp := httputil.NewSingleHostReverseProxy(d.TOURL)
	rp.Transport = tr

	loggingProxyHandler := wrapAccessLog(d.TOSecret, rp)
	return loggingProxyHandler
}
<commit_msg>changes the routes to have -wip on the end to depict they are still in development<commit_after>package main

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import (
	"crypto/tls"
	"database/sql"
	"fmt"
	"net/http"
	"net/http/httputil"
)

// Routes returns the routes, and a catchall route for when no route matches.
func Routes(d ServerData) ([]Route, http.Handler, error) {
	rd, err := routeData(d)
	if err != nil {
		return nil, nil, err
	}
	return []Route{
		{1.2, http.MethodGet, "cdns/{cdn}/configs/monitoring$", wrapHeaders(wrapAuth(monitoringHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, MonitoringPrivLevel))},
		{1.2, http.MethodGet, "cdns/{cdn}/configs/monitoring.json$", wrapHeaders(wrapAuth(monitoringHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, MonitoringPrivLevel))},
		{1.2, http.MethodGet, "servers-wip$", wrapHeaders(wrapAuthWithData(serversHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, ServersPrivLevel))},
		{1.2, http.MethodGet, "servers.json-wip$", wrapHeaders(wrapAuthWithData(serversHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, ServersPrivLevel))},
		{1.2, http.MethodGet, "cdns-wip$", wrapHeaders(wrapAuthWithData(cdnsHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, CdnsPrivLevel))},
		{1.2, http.MethodGet, "cdns.json-wip$", wrapHeaders(wrapAuthWithData(cdnsHandler(d.DB), d.Insecure, d.TOSecret, rd.PrivLevelStmt, CdnsPrivLevel))},
	}, rootHandler(d), nil
}

type RouteData struct {
	PrivLevelStmt *sql.Stmt
}

func routeData(d ServerData) (RouteData, error) {
	rd := RouteData{}
	err := error(nil)

	if rd.PrivLevelStmt, err = preparePrivLevelStmt(d.DB); err != nil {
		return rd, fmt.Errorf("Error preparing db priv level query: ", err)
	}

	return rd, nil
}

// getRootHandler returns the / handler for the service, which reverse-proxies the old Perl Traffic Ops
func rootHandler(d ServerData) http.Handler {
	// debug
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	rp := httputil.NewSingleHostReverseProxy(d.TOURL)
	rp.Transport = tr

	loggingProxyHandler := wrapAccessLog(d.TOSecret, rp)
	return loggingProxyHandler
}
<|endoftext|>


<commit_before>package dc

import (
	"fmt"

	"github.com/CenturyLinkCloud/clc-sdk/api"
)

func New(client api.HTTP) *Service {
	return &Service{
		client: client,
		config: client.Config(),
	}
}

type Service struct {
	client api.HTTP
	config *api.Config
}

func (s *Service) Get(id string) (*Response, error) {
	url := fmt.Sprintf("%s/datacenters/%s/%s?groupLinks=true", s.config.BaseURL, s.config.Alias, id)
	dc := &Response{}
	err := s.client.Get(url, dc)
	return dc, err
}

func (s *Service) GetAll() ([]*Response, error) {
	url := fmt.Sprintf("%s/datacenters/%s", s.config.BaseURL, s.config.Alias)
	dcs := make([]*Response, 0)
	err := s.client.Get(url, &dcs)
	return dcs, err
}

type Response struct {
	ID    string    `json:"id"`
	Name  string    `json:"name"`
	Links api.Links `json:"links"`
}
<commit_msg>add DC capabilities<commit_after>package dc

import (
	"fmt"

	"github.com/CenturyLinkCloud/clc-sdk/api"
)

func New(client api.HTTP) *Service {
	return &Service{
		client: client,
		config: client.Config(),
	}
}

type Service struct {
	client api.HTTP
	config *api.Config
}

func (s *Service) Get(id string) (*Response, error) {
	url := fmt.Sprintf("%s/datacenters/%s/%s?groupLinks=true", s.config.BaseURL, s.config.Alias, id)
	dc := &Response{}
	err := s.client.Get(url, dc)
	return dc, err
}

func (s *Service) GetAll() ([]*Response, error) {
	url := fmt.Sprintf("%s/datacenters/%s", s.config.BaseURL, s.config.Alias)
	dcs := make([]*Response, 0)
	err := s.client.Get(url, &dcs)
	return dcs, err
}

func (s *Service) GetCapabilities(id string) (*CapabilitiesResponse, error) {
	url := fmt.Sprintf("%s/datacenters/%s/%s/deploymentCapabilities", s.config.BaseURL, s.config.Alias, id)
	c := &CapabilitiesResponse{}
	err := s.client.Get(url, c)
	return c, err
}

type Response struct {
	ID    string    `json:"id"`
	Name  string    `json:"name"`
	Links api.Links `json:"links"`
}

type CapabilitiesResponse struct {
	SupportsPremiumStorage     bool `json:"supportsPremiumStorage"`
	SupportsBareMetalServers   bool `json:"supportsBareMetalServers"`
	SupportsSharedLoadBalancer bool `json:"supportsSharedLoadBalancer"`
	Templates                  []struct {
		Name               string   `json:"name"`
		Description        string   `json:"description"`
		StorageSizeGB      string   `json:"storageSizeGB"`
		Capabilities       []string `json:"capabilities"`
		ReservedDrivePaths []string `json:"reservedDrivePaths"`
	} `json:"templates"`
	DeployableNetworks []struct {
		Name      string `json:"name"`
		NetworkId string `json:"networkId"`
		Type      string `json:"type"`
		AccountID string `json:"accountID"`
	} `json:deployableNetworks`
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// The resource package provides the functionality of the "resources"
// feature in Juju.
package resource

import (
	"time"

	"github.com/juju/errors"
	"gopkg.in/juju/charm.v6-unstable/resource"
)

// Resource defines a single resource within Juju state.
type Resource struct {
	resource.Resource

	// Username is the ID of the user that added the revision
	// to the model (whether implicitly or explicitly).
	Username string

	// Timestamp indicates when the resource was added to the model.
	Timestamp time.Time
}

// Validate ensures that the spec is valid.
func (res Resource) Validate() error {
	if err := res.Resource.Validate(); err != nil {
		return errors.Annotate(err, "bad info")
	}

	// TODO(ericsnow) Require that Username be set if timestamp is?

	if res.Timestamp.IsZero() {
		if res.Username != "" {
			return errors.NewNotValid(nil, "missing timestamp")
		}
	}

	return nil
}
<commit_msg>Combine two if statements.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// The resource package provides the functionality of the "resources"
// feature in Juju.
package resource

import (
	"time"

	"github.com/juju/errors"
	"gopkg.in/juju/charm.v6-unstable/resource"
)

// Resource defines a single resource within Juju state.
type Resource struct {
	resource.Resource

	// Username is the ID of the user that added the revision
	// to the model (whether implicitly or explicitly).
	Username string

	// Timestamp indicates when the resource was added to the model.
	Timestamp time.Time
}

// Validate ensures that the spec is valid.
func (res Resource) Validate() error {
	if err := res.Resource.Validate(); err != nil {
		return errors.Annotate(err, "bad info")
	}

	// TODO(ericsnow) Require that Username be set if timestamp is?

	if res.Timestamp.IsZero() && res.Username != "" {
		return errors.NewNotValid(nil, "missing timestamp")
	}

	return nil
}
<|endoftext|>


<commit_before>package resource

import (
	"fmt"
	"log"
	"os"
)

// Resource is the interface type for resources
type Resource interface {
	// ID returns the unique identifier of the resource
	ID() string

	// Validate validates the resource
	Validate() error

	// Returns the resources before which this resource shoud be processed
	WantBefore() []string

	// Returns the resources after which this resource should be processed
	WantAfter() []string

	// Evaluates the resource
	Evaluate() (State, error)

	// Creates the resource
	Create() error

	// Deletes the resource
	Delete() error

	// Updates the resource
	Update() error

	// Log logs events
	Log(format string, a ...interface{})
}

// Config type contains various settings used by the resources
type Config struct {
	// The site repo which contains module and data files
	SiteRepo string

	// Logger used by the resources to log events
	Logger *log.Logger
}

// DefaultLogger is the default logger instance used for
// logging events from the resources
var DefaultLogger = log.New(os.Stdout, "", log.LstdFlags)

// DefaultConfig is the default configuration used by the resources
var DefaultConfig = &Config{
	Logger: DefaultLogger,
}

// Log logs an event using the default resource logger
func Log(format string, a ...interface{}) {
	DefaultConfig.Logger.Printf(format, a...)
}

// BaseResource is the base resource type for all resources
// The purpose of this type is to be embedded into other resources
// Partially implements the Resource interface
type BaseResource struct {
	// Type of the resource
	Type string `luar:"-"`

	// Name of the resource
	Name string `luar:"-"`

	// Desired state of the resource
	State string `luar:"state"`

	// Resources before which this resource should be processed
	Before []string `luar:"before"`

	// Resources after which this resource should be processed
	After []string `luar:"after"`
}

// ID returns the unique resource id
func (br *BaseResource) ID() string {
	return fmt.Sprintf("%s[%s]", br.Type, br.Name)
}

// WantBefore returns the resources before which this resource
// should be processed
func (br *BaseResource) WantBefore() []string {
	return br.Before
}

// WantAfter returns the resources after which this resource
// should be processed
func (br *BaseResource) WantAfter() []string {
	return br.After
}

// Log writes to the default config writer object and
// prepends the resource id to the output
func (br *BaseResource) Log(format string, a ...interface{}) {
	f := fmt.Sprintf("%s %s", br.ID(), format)
	Log(f, a...)
}
<commit_msg>resource: implement Validate() on BaseResource type<commit_after>package resource

import (
	"errors"
	"fmt"
	"log"
	"os"
)

// Resource is the interface type for resources
type Resource interface {
	// ID returns the unique identifier of the resource
	ID() string

	// Validate validates the resource
	Validate() error

	// Returns the resources before which this resource shoud be processed
	WantBefore() []string

	// Returns the resources after which this resource should be processed
	WantAfter() []string

	// Evaluates the resource
	Evaluate() (State, error)

	// Creates the resource
	Create() error

	// Deletes the resource
	Delete() error

	// Updates the resource
	Update() error

	// Log logs events
	Log(format string, a ...interface{})
}

// Config type contains various settings used by the resources
type Config struct {
	// The site repo which contains module and data files
	SiteRepo string

	// Logger used by the resources to log events
	Logger *log.Logger
}

// DefaultLogger is the default logger instance used for
// logging events from the resources
var DefaultLogger = log.New(os.Stdout, "", log.LstdFlags)

// DefaultConfig is the default configuration used by the resources
var DefaultConfig = &Config{
	Logger: DefaultLogger,
}

// Log logs an event using the default resource logger
func Log(format string, a ...interface{}) {
	DefaultConfig.Logger.Printf(format, a...)
}

// BaseResource is the base resource type for all resources
// The purpose of this type is to be embedded into other resources
// Partially implements the Resource interface
type BaseResource struct {
	// Type of the resource
	Type string `luar:"-"`

	// Name of the resource
	Name string `luar:"-"`

	// Desired state of the resource
	State string `luar:"state"`

	// Resources before which this resource should be processed
	Before []string `luar:"before"`

	// Resources after which this resource should be processed
	After []string `luar:"after"`
}

// ID returns the unique resource id
func (br *BaseResource) ID() string {
	return fmt.Sprintf("%s[%s]", br.Type, br.Name)
}

// Validate validates the resource
func (br *BaseResource) Validate() error {
	if br.Type == "" {
		return errors.New("Invalid resource type")
	}

	if br.Name == "" {
		return errors.New("Invalid resource name")
	}
}

// WantBefore returns the resources before which this resource
// should be processed
func (br *BaseResource) WantBefore() []string {
	return br.Before
}

// WantAfter returns the resources after which this resource
// should be processed
func (br *BaseResource) WantAfter() []string {
	return br.After
}

// Log writes to the default config writer object and
// prepends the resource id to the output
func (br *BaseResource) Log(format string, a ...interface{}) {
	f := fmt.Sprintf("%s %s", br.ID(), format)
	Log(f, a...)
}
<|endoftext|>


<commit_before>package resource

import (
	"fmt"

	"github.com/hashicorp/hcl/hcl/ast"
)

// Resource states
const (
	ResourceStateUnknown = "unknown"
	ResourceStatePresent = "present"
	ResourceStateAbsent  = "absent"
	ResourceStateUpdate  = "update"
)

// Provider is used to create new resources from an HCL AST object item
type Provider func(item *ast.ObjectItem) (Resource, error)

// Registry contains all known resource types and their providers
var registry = make(map[string]Provider)

// Register registers a resource type and it's provider
func Register(name string, p Provider) error {
	_, ok := registry[name]
	if ok {
		return fmt.Errorf("Resource provider for '%s' is already registered", name)
	}

	registry[name] = p

	return nil
}

// Get retrieves the provider for a given resource type
func Get(name string) (Provider, bool) {
	p, ok := registry[name]

	return p, ok
}

// State type represents the current and wanted states of a resource
type State struct {
	// Current state of the resource
	Current string

	// Wanted state of the resource
	Want string
}

// Resource is the base interface type for all resources
type Resource interface {
	// Type of the resource
	Type() string

	// ID returns the unique identifier of a resource
	ID() string

	// Returns the wanted resources/dependencies
	Want() []string

	// Evaluates the resource and returns it's state
	Evaluate() (State, error)

	// Creates the resource
	Create() error

	// Deletes the resource
	Delete() error

	// Updates the resource
	Update() error
}

// BaseResource is the base resource type for all resources
// The purpose of this type is to be embedded into other resources
// Partially implements the Resource interface
type BaseResource struct {
	// Name of the resource
	Name string `json:"name"`

	// Desired state of the resource
	State string `json:"state"`

	// Type of the resource
	ResourceType string `json:"-"`

	// Resource dependencies
	WantResource []string `json:"want,omitempty" hcl:"want"`
}

// Type returns the resource type name
func (b *BaseResource) Type() string {
	return b.ResourceType
}

// ID returns the unique resource id
func (b *BaseResource) ID() string {
	return fmt.Sprintf("%s[%s]", b.ResourceType, b.Name)
}

// Want returns the wanted resources/dependencies
func (b *BaseResource) Want() []string {
	return b.WantResource
}
<commit_msg>Resource interface requires the implementation of Validate() method<commit_after>package resource

import (
	"fmt"

	"github.com/hashicorp/hcl/hcl/ast"
)

// Provider is used to create new resources from an HCL AST object item
type Provider func(item *ast.ObjectItem) (Resource, error)

// Registry contains all known resource types and their providers
var registry = make(map[string]Provider)

// Register registers a resource type and it's provider
func Register(name string, p Provider) error {
	_, ok := registry[name]
	if ok {
		return fmt.Errorf("Resource provider for '%s' is already registered", name)
	}

	registry[name] = p

	return nil
}

// Get retrieves the provider for a given resource type
func Get(name string) (Provider, bool) {
	p, ok := registry[name]

	return p, ok
}

// State type represents the current and wanted states of a resource
type State struct {
	// Current state of the resource
	Current string

	// Wanted state of the resource
	Want string
}

// Resource is the base interface type for all resources
type Resource interface {
	// Type of the resource
	Type() string

	// ID returns the unique identifier of a resource
	ID() string

	// Validates the resource
	Validate() error

	// Returns the wanted resources/dependencies
	Want() []string

	// Evaluates the resource and returns it's state
	Evaluate() (State, error)

	// Creates the resource
	Create() error

	// Deletes the resource
	Delete() error

	// Updates the resource
	Update() error
}

// BaseResource is the base resource type for all resources
// The purpose of this type is to be embedded into other resources
// Partially implements the Resource interface
type BaseResource struct {
	// Name of the resource
	Name string `json:"name"`

	// Desired state of the resource
	State string `json:"state"`

	// Type of the resource
	ResourceType string `json:"-"`

	// Resource dependencies
	WantResource []string `json:"want,omitempty" hcl:"want"`
}

// Type returns the resource type name
func (b *BaseResource) Type() string {
	return b.ResourceType
}

// ID returns the unique resource id
func (b *BaseResource) ID() string {
	return fmt.Sprintf("%s[%s]", b.ResourceType, b.Name)
}

// Validate checks if the resource contains valid information
func (b *BaseResource) Validate() error {
	if b.Name == "" {
		return fmt.Errorf("Missing name for resource %s", b.ID())
	}

	return nil
}

// Want returns the wanted resources/dependencies
func (b *BaseResource) Want() []string {
	return b.WantResource
}
<|endoftext|>


<commit_before>package main

import (
	"strconv"
	"time"
)

func (f foldersResp) displayTable() [][]string {
	table := make([][]string, 0, len(f))
	for _, folders := range f {
		tableRow := make([]string, 2)
		tableRow[0] = strconv.Itoa(folders.ID)
		tableRow[1] = folders.Title
		table = append(table, tableRow)
	}
	return table
}

func (s sitesResp) displayTable() [][]string {
	table := make([][]string, 0, len(s))
	for _, sites := range s {
		tableRow := make([]string, 2)
		tableRow[0] = strconv.Itoa(sites.ID)
		tableRow[1] = sites.Name
		table = append(table, tableRow)
	}
	return table
}

func (b browsersResp) displayTable() [][]string {
	table := make([][]string, 0, len(b.AvailableBrowsers))
	for _, browsers := range b.AvailableBrowsers {
		tableRow := make([]string, 2)
		tableRow[0] = browsers.Name
		tableRow[1] = browsers.Description
		table = append(table, tableRow)
	}
	return table
}

type foldersResp []folder

type folder struct {
	ID        int          `json:"id"`
	CreatedAt time.Time    `json:"created_at"`
	Title     string       `json:"title"`
	Logic     []logicSlice `json:"logic"`
	TestCount int          `json:"test_count"`
}
type logicSlice struct {
	Tag       string `json:"tag"`
	Inclusive bool   `json:"inclusive"`
}

type browsersResp struct {
	AvailableBrowsers []browser `json:"available_browsers"`
}
type browser struct {
	Name           string `json:"name"`
	Description    string `json:"description"`
	Category       string `json:"category"`
	BrowserVersion string `json:"browser_version"`
	OsVersion      string `json:"os_version"`
	Default        bool   `json:"default"`
}
type sitesResp []sites
type sites struct {
	ID        int       `json:"id"`
	CreatedAt time.Time `json:"created_at"`
	Name      string    `json:"name"`
	Default   bool      `json:"default"`
}
<commit_msg>remove unnecessary fields from API JSON responses<commit_after>package main

import "strconv"

func (f foldersResp) displayTable() [][]string {
	table := make([][]string, 0, len(f))
	for _, folders := range f {
		tableRow := make([]string, 2)
		tableRow[0] = strconv.Itoa(folders.ID)
		tableRow[1] = folders.Title
		table = append(table, tableRow)
	}
	return table
}

func (s sitesResp) displayTable() [][]string {
	table := make([][]string, 0, len(s))
	for _, sites := range s {
		tableRow := make([]string, 2)
		tableRow[0] = strconv.Itoa(sites.ID)
		tableRow[1] = sites.Name
		table = append(table, tableRow)
	}
	return table
}

func (b browsersResp) displayTable() [][]string {
	table := make([][]string, 0, len(b.AvailableBrowsers))
	for _, browsers := range b.AvailableBrowsers {
		tableRow := make([]string, 2)
		tableRow[0] = browsers.Name
		tableRow[1] = browsers.Description
		table = append(table, tableRow)
	}
	return table
}

type foldersResp []folder

type folder struct {
	ID    int    `json:"id"`
	Title string `json:"title"`
}

type browsersResp struct {
	AvailableBrowsers []browser `json:"available_browsers"`
}

type browser struct {
	Name        string `json:"name"`
	Description string `json:"description"`
}

type sitesResp []sites

type sites struct {
	ID   int    `json:"id"`
	Name string `json:"name"`
}
<|endoftext|>


<commit_before>// Package response defines the how the default microservice response must look and behave like.
package response

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/LUSHDigital/microservice-core-golang/pagination"
)

// Standard response statuses.
const (
	StatusOk   = "ok"
	StatusFail = "fail"
)

// ResponseInterface - Interface for microservice responses.
type ResponseInterface interface {
	// ExtractData returns a particular item of data from the response.
	ExtractData(srcKey string, dst interface{}) error

	// GetCode returns the response code.
	GetCode() int
}

// Response - A standardised response format for a microservice.
type Response struct {
	Status  string `json:"status"`         // Can be 'ok' or 'fail'
	Code    int    `json:"code"`           // Any valid HTTP response code
	Message string `json:"message"`        // Any relevant message (optional)
	Data    *Data  `json:"data,omitempty"` // Data to pass along to the response (optional)
}

// New returns a new Response for a microservice endpoint
// This ensures that all API endpoints return data in a standardised format:
//
//    {
//       "status": "ok or fail",
//       "code": any HTTP response code,
//       "message": "any relevant message (optional)",
//       "data": {[
//          ...
//       ]}
//    }
func New(code int, message string, data *Data) *Response {
	var status string
	switch {
	case code >= http.StatusOK && code < http.StatusBadRequest:
		status = StatusOk
	default:
		status = StatusFail
	}
	return &Response{
		Code:    code,
		Status:  status,
		Message: message,
		Data:    data,
	}
}

// WriteTo - pick a response writer to write the default json response to.
func (r *Response) WriteTo(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(r.Code)
	json.NewEncoder(w).Encode(r)
}

// ExtractData returns a particular item of data from the response.
func (r *Response) ExtractData(srcKey string, dst interface{}) error {
	if !r.Data.Valid() {
		return fmt.Errorf("invalid data provided: %v", r.Data)
	}
	for key, value := range r.Data.Map() {
		if key != srcKey {
			continue
		}

		// Get the raw JSON just for the endpoints.
		rawJSON, err := json.Marshal(value)
		if err != nil {
			return err
		}

		// Decode the raw JSON.
		json.Unmarshal(rawJSON, &dst)
	}

	return nil
}

// GetCode returns the response code.
func (r *Response) GetCode() int {
	return r.Code
}

// PaginatedResponse - A paginated response format for a microservice.
type PaginatedResponse struct {
	Status     string               `json:"status"`         // Can be 'ok' or 'fail'
	Code       int                  `json:"code"`           // Any valid HTTP response code
	Message    string               `json:"message"`        // Any relevant message (optional)
	Data       *Data                `json:"data,omitempty"` // Data to pass along to the response (optional)
	Pagination *pagination.Response `json:"pagination"`     // Pagination data
}

// NewPaginated returns a new PaginatedResponse for a microservice endpoint
func NewPaginated(paginator *pagination.Paginator, code int, message string, data *Data) *PaginatedResponse {
	var status string
	switch {
	case code >= http.StatusOK && code < http.StatusBadRequest:
		status = StatusOk
	default:
		status = StatusFail
	}
	return &PaginatedResponse{
		Code:       code,
		Status:     status,
		Message:    message,
		Data:       data,
		Pagination: paginator.PrepareResponse(),
	}
}

func (p *PaginatedResponse) WriteTo(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(p.Code)
	json.NewEncoder(w).Encode(p)
}

// ExtractData returns a particular item of data from the response.
func (p *PaginatedResponse) ExtractData(srcKey string, dst interface{}) error {
	if !p.Data.Valid() {
		return fmt.Errorf("invalid data provided: %v", p.Data)
	}
	for key, value := range p.Data.Map() {
		if key != srcKey {
			continue
		}

		// Get the raw JSON just for the endpoints.
		rawJSON, err := json.Marshal(value)
		if err != nil {
			return err
		}

		// Decode the raw JSON.
		json.Unmarshal(rawJSON, &dst)
	}

	return nil
}

// GetCode returns the response code.
func (p *PaginatedResponse) GetCode() int {
	return p.Code
}

// Data represents the collection data the the response will return to the consumer.
// Type ends up being the name of the key containing the collection of Content
type Data struct {
	Type    string
	Content interface{}
}

// UnmarshalJSON implements the Unmarshaler interface
// this implementation will fill the type in the case we're been provided a valid single collection
// and set the content to the contents of said collection.
// for every other options, it behaves like normal.
// Despite the fact that we are not suposed to marshal without a type set,
// this is purposefuly left open to unmarshal without a collection name set, in case you may want to set it later,
// and for interop with other systems which may not send the collection properly.
func (d *Data) UnmarshalJSON(b []byte) error {
	if err := json.Unmarshal(b, &d.Content); err != nil {
		log.Printf("cannot unmarshal data: %v", err)
	}

	data, ok := d.Content.(map[string]interface{})
	if ok {
		// count how many collections were provided
		var count int
		for _, value := range data {
			if _, ok := value.(map[string]interface{}); ok {
				count++
			}
			if _, ok := value.([]interface{}); ok {
				count++
			}
		}
		if count > 1 {
			// we can stop there since this is not a single collection
			return nil
		}

		for key, value := range data {
			if _, ok := value.(map[string]interface{}); ok {
				d.Type = key
				d.Content = data[key]
			} else if _, ok := value.([]interface{}); ok {
				d.Type = key
				d.Content = data[key]
			}
		}
	}

	return nil
}

// Valid ensures the Data passed to the response is correct (it must contain a Type along with the data).
func (d *Data) Valid() bool {
	if d.Type != "" {
		return true
	}
	return false
}

// MarshalJSON implements the Marshaler interface and is there to ensure the output
// is correct when we return data to the consumer
func (d *Data) MarshalJSON() ([]byte, error) {
	return json.Marshal(d.Map())
}

// Map returns a version of the data as a map
func (d *Data) Map() map[string]interface{} {
	if !d.Valid() {
		return nil
	}
	d.Type = strings.Replace(strings.ToLower(d.Type), " ", "-", -1)

	return map[string]interface{}{
		d.Type: d.Content,
	}
}
<commit_msg>Adds prepared responses for common use-cases<commit_after>// Package response defines the how the default microservice response must look and behave like.
package response

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/LUSHDigital/microservice-core-golang/pagination"
	"database/sql"
	"github.com/VividCortex/mysqlerr"
	"github.com/go-sql-driver/mysql"
)

// Standard response statuses.
const (
	StatusOk   = "ok"
	StatusFail = "fail"
)

// ResponseInterface - Interface for microservice responses.
type ResponseInterface interface {
	// ExtractData returns a particular item of data from the response.
	ExtractData(srcKey string, dst interface{}) error

	// GetCode returns the response code.
	GetCode() int
}

// Response - A standardised response format for a microservice.
type Response struct {
	Status  string `json:"status"`         // Can be 'ok' or 'fail'
	Code    int    `json:"code"`           // Any valid HTTP response code
	Message string `json:"message"`        // Any relevant message (optional)
	Data    *Data  `json:"data,omitempty"` // Data to pass along to the response (optional)
}

// New returns a new Response for a microservice endpoint
// This ensures that all API endpoints return data in a standardised format:
//
//    {
//       "status": "ok or fail",
//       "code": any HTTP response code,
//       "message": "any relevant message (optional)",
//       "data": {[
//          ...
//       ]}
//    }
func New(code int, message string, data *Data) *Response {
	var status string
	switch {
	case code >= http.StatusOK && code < http.StatusBadRequest:
		status = StatusOk
	default:
		status = StatusFail
	}
	return &Response{
		Code:    code,
		Status:  status,
		Message: message,
		Data:    data,
	}
}

// SQLError returns a prepared 422 Unprocessable Entity response if the error passed is of type sql.ErrNoRows,
// otherwise, returns a 500 Internal Server Error prepared response.
func SQLError(err error) *Response {
	if err == sql.ErrNoRows {
		return New(http.StatusUnprocessableEntity, "no data found", nil)
	}
	if driverErr, ok := err.(*mysql.MySQLError); ok {
		if driverErr.Number == mysqlerr.ER_DUP_ENTRY {
			return New(http.StatusUnprocessableEntity, "duplicate entry.", nil)
		}
	}
	return New(http.StatusInternalServerError, fmt.Sprintf("db error: %v", err), nil)
}

// JSONError returns a prepared 422 Unprocessable Entity response if the error passed is of type *json.SyntaxError,
// otherwise, returns a 500 Internal Server Error prepared response.
func JSONError(err error) *Response {
	if syn, ok := err.(*json.SyntaxError); ok {
		return New(http.StatusUnprocessableEntity, fmt.Sprintf("invalid json: %v", syn), nil)
	}
	return New(http.StatusInternalServerError, fmt.Sprintf("json error: %v", err), nil)
}

// ParamError returns a prepared 422 Unprocessable Entity response, including the name of
// the failing parameter in the message field of the response object.
func ParamError(name string) *Response {
	return New(http.StatusUnprocessableEntity, fmt.Sprintf("invalid or missing parameter: %v", name), nil)
}

// ValidationError returns a prepared 422 Unprocessable Entity response, including the name of
// the failing validation/validator in the message field of the response object.
func ValidationError(err error, name string) *Response {
	return New(http.StatusUnprocessableEntity, fmt.Sprintf("validation error on %s: %v", name, err), nil)
}

// NotFoundErr returns a prepared 404 Not Found response, including the message passed by the user
// in the message field of the response object.
func NotFoundErr(msg string) *Response {
	return New(http.StatusNotFound, msg, nil)
}

// InternalError returns a prepared 500 Internal Server Error, including the error
// message in the message field of the response object
func InternalError(err error) *Response {
	return New(http.StatusInternalServerError, fmt.Sprintf("internal server error: %v", err), nil)
}

// WriteTo - pick a response writer to write the default json response to.
func (r *Response) WriteTo(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(r.Code)
	json.NewEncoder(w).Encode(r)
}

// ExtractData returns a particular item of data from the response.
func (r *Response) ExtractData(srcKey string, dst interface{}) error {
	if !r.Data.Valid() {
		return fmt.Errorf("invalid data provided: %v", r.Data)
	}
	for key, value := range r.Data.Map() {
		if key != srcKey {
			continue
		}

		// Get the raw JSON just for the endpoints.
		rawJSON, err := json.Marshal(value)
		if err != nil {
			return err
		}

		// Decode the raw JSON.
		json.Unmarshal(rawJSON, &dst)
	}

	return nil
}

// GetCode returns the response code.
func (r *Response) GetCode() int {
	return r.Code
}

// PaginatedResponse - A paginated response format for a microservice.
type PaginatedResponse struct {
	Status     string               `json:"status"`         // Can be 'ok' or 'fail'
	Code       int                  `json:"code"`           // Any valid HTTP response code
	Message    string               `json:"message"`        // Any relevant message (optional)
	Data       *Data                `json:"data,omitempty"` // Data to pass along to the response (optional)
	Pagination *pagination.Response `json:"pagination"`     // Pagination data
}

// NewPaginated returns a new PaginatedResponse for a microservice endpoint
func NewPaginated(paginator *pagination.Paginator, code int, message string, data *Data) *PaginatedResponse {
	var status string
	switch {
	case code >= http.StatusOK && code < http.StatusBadRequest:
		status = StatusOk
	default:
		status = StatusFail
	}
	return &PaginatedResponse{
		Code:       code,
		Status:     status,
		Message:    message,
		Data:       data,
		Pagination: paginator.PrepareResponse(),
	}
}

func (p *PaginatedResponse) WriteTo(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(p.Code)
	json.NewEncoder(w).Encode(p)
}

// ExtractData returns a particular item of data from the response.
func (p *PaginatedResponse) ExtractData(srcKey string, dst interface{}) error {
	if !p.Data.Valid() {
		return fmt.Errorf("invalid data provided: %v", p.Data)
	}
	for key, value := range p.Data.Map() {
		if key != srcKey {
			continue
		}

		// Get the raw JSON just for the endpoints.
		rawJSON, err := json.Marshal(value)
		if err != nil {
			return err
		}

		// Decode the raw JSON.
		json.Unmarshal(rawJSON, &dst)
	}

	return nil
}

// GetCode returns the response code.
func (p *PaginatedResponse) GetCode() int {
	return p.Code
}

// Data represents the collection data the the response will return to the consumer.
// Type ends up being the name of the key containing the collection of Content
type Data struct {
	Type    string
	Content interface{}
}

// UnmarshalJSON implements the Unmarshaler interface
// this implementation will fill the type in the case we're been provided a valid single collection
// and set the content to the contents of said collection.
// for every other options, it behaves like normal.
// Despite the fact that we are not suposed to marshal without a type set,
// this is purposefuly left open to unmarshal without a collection name set, in case you may want to set it later,
// and for interop with other systems which may not send the collection properly.
func (d *Data) UnmarshalJSON(b []byte) error {
	if err := json.Unmarshal(b, &d.Content); err != nil {
		log.Printf("cannot unmarshal data: %v", err)
	}

	data, ok := d.Content.(map[string]interface{})
	if ok {
		// count how many collections were provided
		var count int
		for _, value := range data {
			if _, ok := value.(map[string]interface{}); ok {
				count++
			}
			if _, ok := value.([]interface{}); ok {
				count++
			}
		}
		if count > 1 {
			// we can stop there since this is not a single collection
			return nil
		}

		for key, value := range data {
			if _, ok := value.(map[string]interface{}); ok {
				d.Type = key
				d.Content = data[key]
			} else if _, ok := value.([]interface{}); ok {
				d.Type = key
				d.Content = data[key]
			}
		}
	}

	return nil
}

// Valid ensures the Data passed to the response is correct (it must contain a Type along with the data).
func (d *Data) Valid() bool {
	if d.Type != "" {
		return true
	}
	return false
}

// MarshalJSON implements the Marshaler interface and is there to ensure the output
// is correct when we return data to the consumer
func (d *Data) MarshalJSON() ([]byte, error) {
	return json.Marshal(d.Map())
}

// Map returns a version of the data as a map
func (d *Data) Map() map[string]interface{} {
	if !d.Valid() {
		return nil
	}
	d.Type = strings.Replace(strings.ToLower(d.Type), " ", "-", -1)

	return map[string]interface{}{
		d.Type: d.Content,
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package runner

import (
	"context"
	"fmt"
	"io"
	"net"
	"os/exec"
	"strings"
	"time"

	gm "github.com/getgauge/gauge/gauge_messages"
	"github.com/getgauge/gauge/manifest"
	"google.golang.org/grpc"
)

const (
	portPrefix = "Listening on port:"
	host       = "127.0.0.1"
)

// GrpcRunner handles grpc messages.
type GrpcRunner struct {
	cmd     *exec.Cmd
	conn    *grpc.ClientConn
	Client  gm.LspServiceClient
	Timeout time.Duration
}

func (r *GrpcRunner) execute(message *gm.Message) (*gm.Message, error) {
	switch message.MessageType {
	case gm.Message_CacheFileRequest:
		r.Client.CacheFile(context.Background(), message.CacheFileRequest)
		return &gm.Message{}, nil
	case gm.Message_StepNamesRequest:
		response, err := r.Client.GetStepNames(context.Background(), message.StepNamesRequest)
		return &gm.Message{StepNamesResponse: response}, err
	case gm.Message_StepPositionsRequest:
		response, err := r.Client.GetStepPositions(context.Background(), message.StepPositionsRequest)
		return &gm.Message{StepPositionsResponse: response}, err
	case gm.Message_ImplementationFileListRequest:
		response, err := r.Client.GetImplementationFiles(context.Background(), &gm.Empty{})
		return &gm.Message{ImplementationFileListResponse: response}, err
	case gm.Message_StubImplementationCodeRequest:
		response, err := r.Client.ImplementStub(context.Background(), message.StubImplementationCodeRequest)
		return &gm.Message{FileDiff: response}, err
	case gm.Message_StepValidateRequest:
		response, err := r.Client.ValidateStep(context.Background(), message.StepValidateRequest)
		return &gm.Message{MessageType: gm.Message_StepValidateResponse, StepValidateResponse: response}, err
	case gm.Message_RefactorRequest:
		response, err := r.Client.Refactor(context.Background(), message.RefactorRequest)
		return &gm.Message{MessageType: gm.Message_RefactorResponse, RefactorResponse: response}, err
	case gm.Message_StepNameRequest:
		response, err := r.Client.GetStepName(context.Background(), message.StepNameRequest)
		return &gm.Message{MessageType: gm.Message_StepNameResponse, StepNameResponse: response}, err
	case gm.Message_ImplementationFileGlobPatternRequest:
		response, err := r.Client.GetGlobPatterns(context.Background(), &gm.Empty{})
		return &gm.Message{MessageType: gm.Message_ImplementationFileGlobPatternRequest, ImplementationFileGlobPatternResponse: response}, err
	case gm.Message_KillProcessRequest:
		_, err := r.Client.KillProcess(context.Background(), message.KillProcessRequest)
		return &gm.Message{}, err
	default:
		return nil, nil
	}
}

// ExecuteMessageWithTimeout process reuqest and give back the response
func (r *GrpcRunner) ExecuteMessageWithTimeout(message *gm.Message) (*gm.Message, error) {
	resChan := make(chan *gm.Message)
	errChan := make(chan error)
	go func() {
		res, err := r.execute(message)
		if err != nil {
			errChan <- err
		} else {
			resChan <- res
		}
	}()

	select {
	case response := <-resChan:
		return response, nil
	case err := <-errChan:
		return nil, err
	case <-time.After(r.Timeout):
		return nil, fmt.Errorf("Request Timed out for message %s", message.GetMessageType().String())
	}
}

func (r *GrpcRunner) ExecuteAndGetStatus(m *gm.Message) *gm.ProtoExecutionResult {
	return nil
}
func (r *GrpcRunner) IsProcessRunning() bool {
	return false
}

// Kill closes the grpc connection and kills the process
func (r *GrpcRunner) Kill() error {
	r.ExecuteMessageWithTimeout(&gm.Message{MessageType: gm.Message_KillProcessRequest, KillProcessRequest: &gm.KillProcessRequest{}})
	if err := r.conn.Close(); err != nil {
		return err
	}
	// TODO: wait for process to exit or kill forcefully after runner kill timeout
	if err := r.cmd.Process.Kill(); err != nil {
		return err
	}
	return nil
}

func (r *GrpcRunner) Connection() net.Conn {
	return nil
}

func (r *GrpcRunner) IsMultithreaded() bool {
	return false
}

func (r *GrpcRunner) Pid() int {
	return 0
}

type customWriter struct {
	file io.Writer
	port chan string
}

func (w customWriter) Write(p []byte) (n int, err error) {
	if strings.Contains(string(p), portPrefix) {
		w.port <- strings.TrimSuffix(strings.Split(string(p), portPrefix)[1], "\n")
	}
	return w.file.Write(p)
}

// ConnectToGrpcRunner makes a connection with grpc server
func ConnectToGrpcRunner(manifest *manifest.Manifest, outFile io.Writer, timeout time.Duration) (*GrpcRunner, error) {
	portChan := make(chan string)
	cmd, _, err := runRunnerCommand(manifest, "0", false, customWriter{file: outFile, port: portChan})
	if err != nil {
		return nil, err
	}
	port := <-portChan
	close(portChan)
	conn, err := grpc.Dial(fmt.Sprintf("%s:%s", host, port), grpc.WithInsecure(), grpc.WithBlock())
	if err != nil {
		return nil, err
	}
	return &GrpcRunner{Client: gm.NewLspServiceClient(conn), cmd: cmd, conn: conn, Timeout: timeout}, nil
}
<commit_msg>Removing carriage returm char from port info line. getgauge/gauge-python#52<commit_after>// Copyright 2018 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package runner

import (
	"context"
	"fmt"
	"io"
	"net"
	"os/exec"
	"strings"
	"time"

	gm "github.com/getgauge/gauge/gauge_messages"
	"github.com/getgauge/gauge/manifest"
	"google.golang.org/grpc"
)

const (
	portPrefix = "Listening on port:"
	host       = "127.0.0.1"
)

// GrpcRunner handles grpc messages.
type GrpcRunner struct {
	cmd     *exec.Cmd
	conn    *grpc.ClientConn
	Client  gm.LspServiceClient
	Timeout time.Duration
}

func (r *GrpcRunner) execute(message *gm.Message) (*gm.Message, error) {
	switch message.MessageType {
	case gm.Message_CacheFileRequest:
		r.Client.CacheFile(context.Background(), message.CacheFileRequest)
		return &gm.Message{}, nil
	case gm.Message_StepNamesRequest:
		response, err := r.Client.GetStepNames(context.Background(), message.StepNamesRequest)
		return &gm.Message{StepNamesResponse: response}, err
	case gm.Message_StepPositionsRequest:
		response, err := r.Client.GetStepPositions(context.Background(), message.StepPositionsRequest)
		return &gm.Message{StepPositionsResponse: response}, err
	case gm.Message_ImplementationFileListRequest:
		response, err := r.Client.GetImplementationFiles(context.Background(), &gm.Empty{})
		return &gm.Message{ImplementationFileListResponse: response}, err
	case gm.Message_StubImplementationCodeRequest:
		response, err := r.Client.ImplementStub(context.Background(), message.StubImplementationCodeRequest)
		return &gm.Message{FileDiff: response}, err
	case gm.Message_StepValidateRequest:
		response, err := r.Client.ValidateStep(context.Background(), message.StepValidateRequest)
		return &gm.Message{MessageType: gm.Message_StepValidateResponse, StepValidateResponse: response}, err
	case gm.Message_RefactorRequest:
		response, err := r.Client.Refactor(context.Background(), message.RefactorRequest)
		return &gm.Message{MessageType: gm.Message_RefactorResponse, RefactorResponse: response}, err
	case gm.Message_StepNameRequest:
		response, err := r.Client.GetStepName(context.Background(), message.StepNameRequest)
		return &gm.Message{MessageType: gm.Message_StepNameResponse, StepNameResponse: response}, err
	case gm.Message_ImplementationFileGlobPatternRequest:
		response, err := r.Client.GetGlobPatterns(context.Background(), &gm.Empty{})
		return &gm.Message{MessageType: gm.Message_ImplementationFileGlobPatternRequest, ImplementationFileGlobPatternResponse: response}, err
	case gm.Message_KillProcessRequest:
		_, err := r.Client.KillProcess(context.Background(), message.KillProcessRequest)
		return &gm.Message{}, err
	default:
		return nil, nil
	}
}

// ExecuteMessageWithTimeout process reuqest and give back the response
func (r *GrpcRunner) ExecuteMessageWithTimeout(message *gm.Message) (*gm.Message, error) {
	resChan := make(chan *gm.Message)
	errChan := make(chan error)
	go func() {
		res, err := r.execute(message)
		if err != nil {
			errChan <- err
		} else {
			resChan <- res
		}
	}()

	select {
	case response := <-resChan:
		return response, nil
	case err := <-errChan:
		return nil, err
	case <-time.After(r.Timeout):
		return nil, fmt.Errorf("Request Timed out for message %s", message.GetMessageType().String())
	}
}

func (r *GrpcRunner) ExecuteAndGetStatus(m *gm.Message) *gm.ProtoExecutionResult {
	return nil
}
func (r *GrpcRunner) IsProcessRunning() bool {
	return false
}

// Kill closes the grpc connection and kills the process
func (r *GrpcRunner) Kill() error {
	r.ExecuteMessageWithTimeout(&gm.Message{MessageType: gm.Message_KillProcessRequest, KillProcessRequest: &gm.KillProcessRequest{}})
	if err := r.conn.Close(); err != nil {
		return err
	}
	// TODO: wait for process to exit or kill forcefully after runner kill timeout
	if err := r.cmd.Process.Kill(); err != nil {
		return err
	}
	return nil
}

func (r *GrpcRunner) Connection() net.Conn {
	return nil
}

func (r *GrpcRunner) IsMultithreaded() bool {
	return false
}

func (r *GrpcRunner) Pid() int {
	return 0
}

type customWriter struct {
	file io.Writer
	port chan string
}

func (w customWriter) Write(p []byte) (n int, err error) {
	if strings.Contains(string(p), portPrefix) {
		text := strings.Replace(string(p), "\r\n", "\n", -1)
		w.port <- strings.TrimSuffix(strings.Split(text, portPrefix)[1], "\n")
	}
	return w.file.Write(p)
}

// ConnectToGrpcRunner makes a connection with grpc server
func ConnectToGrpcRunner(manifest *manifest.Manifest, outFile io.Writer, timeout time.Duration) (*GrpcRunner, error) {
	portChan := make(chan string)
	cmd, _, err := runRunnerCommand(manifest, "0", false, customWriter{file: outFile, port: portChan})
	if err != nil {
		return nil, err
	}
	port := <-portChan
	close(portChan)
	conn, err := grpc.Dial(fmt.Sprintf("%s:%s", host, port), grpc.WithInsecure(), grpc.WithBlock())
	if err != nil {
		return nil, err
	}
	return &GrpcRunner{Client: gm.NewLspServiceClient(conn), cmd: cmd, conn: conn, Timeout: timeout}, nil
}
<|endoftext|>


<commit_before>//  Copyright 2018 Google LLC
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at

//        https://www.apache.org/licenses/LICENSE-2.0

//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//	limitations under the License.

package main

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"syscall"

	"github.com/google/minions/go/overlord/interests"
	minions "github.com/google/minions/proto/minions"
	pb "github.com/google/minions/proto/overlord"
)

// loadFiles builds the File protos for a slice of interests in chunks,
// topping at maximum size and files count. Note we do not support
// content regexps at this point (i.e. we do not check file contents).
func loadFiles(intrs []*minions.Interest, maxKb int, maxFiles int, root string) ([][]*pb.File, error) {
	// Defensively minify the interests: this should have already happened but better safe than sorry.
	intrs = interests.Minify(intrs)

	paths := make(map[string]minions.Interest_DataType)
	// Note we assume a unix filesystem here. Might want to revisit.
	err := filepath.Walk(root, func(path string, f os.FileInfo, e error) error {

		if e != nil {
			// If we don't have permission, skip the directory but don't bail out.
			if strings.Contains(e.Error(), "permission denied") {
				return filepath.SkipDir
			}
			if strings.Contains(e.Error(), "no such file") {
				return filepath.SkipDir
			}

			log.Printf("prevent panic by handling failure accessing a path %q: %v\n", path, e)
			return e
		}
		// For the naive implementation, let's check every file, but really
		// here we need to bail out early instead and return filepath.SkipDir
		// anytime we take a wrong turn.
		if !f.IsDir() {
			// Let's see if we match any interest!
			for _, i := range intrs {
				r, err := regexp.MatchString("^"+i.GetPathRegexp(), path)
				if err == nil && r {
					// NOTE: this overwrites existing datatypes, under the assumption that the
					// minification has taken care of this.
					paths[path] = i.GetDataType()
				}
			}
		}
		return nil
	})

	if err != nil {
		return nil, err
	}

	var files [][]*pb.File
	var fs []*pb.File
	for path, dataType := range paths {
		metadata, err := getMetadata(path)
		if err != nil {
			return nil, err
		}
		f := &pb.File{Metadata: metadata, DataChunks: nil}
		switch dataType {
		case minions.Interest_METADATA:
			break
		case minions.Interest_METADATA_AND_DATA:
			chunks, err := getDataChunks(path)
			if err != nil {
				return nil, err
			}
			f.DataChunks = chunks
			break
		default:
			return nil, errors.New("Unknown interest type")
		}
		fs = append(fs, f)
	}
	files = append(files, fs)

	return files, nil
}

// getMetadata is heavily linux skewed, but so is minions right now.
// It's fairly easy to port to windows by adding the appropriate data
// structure, but not planned for now.
func getMetadata(path string) (*minions.FileMetadata, error) {
	s, err := os.Stat(path)
	if err != nil {
		// I suspect this is too aggressive, as this can fail for a number
		// of reasons, including permissions. It might be wiser to log
		// and proceed, but let's try as is now.
		return nil, err
	}
	sys := s.Sys()
	if sys == nil {
		return nil, errors.New("cannot access OS-specific metadata")
	}

	m := &minions.FileMetadata{Path: path}
	// TODO(paradoxengine): these conversions are all over the place :-(
	m.OwnerUid = int32(sys.(*syscall.Stat_t).Uid)
	m.OwnerGid = int32(sys.(*syscall.Stat_t).Gid)
	m.Permissions = uint32(s.Mode())
	m.Size = s.Size()
	return m, nil
}

// getDataChunks splits the file at the path in a set of chunks.
func getDataChunks(path string) ([]*pb.DataChunk, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	var chunks []*pb.DataChunk
	// Arbitrary size of each data chunk.
	var chunkSize = 1024 * 1024 * 2
	dataLen := len(data)
	for i := 0; i < dataLen; i += chunkSize {
		var chunk []byte
		if i+chunkSize >= dataLen {
			chunk = data[i:]
		} else {
			chunk = data[i : i+chunkSize]
		}
		chunks = append(chunks, &pb.DataChunk{
			Offset: int64(i),
			Data:   chunk,
		})
	}
	return chunks, nil
}
<commit_msg>Note that file size and count limits are not implemented yet<commit_after>//  Copyright 2018 Google LLC
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at

//        https://www.apache.org/licenses/LICENSE-2.0

//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//	limitations under the License.

package main

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"syscall"

	"github.com/google/minions/go/overlord/interests"
	minions "github.com/google/minions/proto/minions"
	pb "github.com/google/minions/proto/overlord"
)

// loadFiles builds the File protos for a slice of interests in chunks,
// topping at maximum size and files count. Note we do not support
// content regexps at this point (i.e. we do not check file contents).
func loadFiles(intrs []*minions.Interest, maxKb int, maxFiles int, root string) ([][]*pb.File, error) {
	// Defensively minify the interests: this should have already happened but better safe than sorry.
	intrs = interests.Minify(intrs)

	paths := make(map[string]minions.Interest_DataType)
	// Note we assume a unix filesystem here. Might want to revisit.
	err := filepath.Walk(root, func(path string, f os.FileInfo, e error) error {

		if e != nil {
			// If we don't have permission, skip the directory but don't bail out.
			if strings.Contains(e.Error(), "permission denied") {
				return filepath.SkipDir
			}
			if strings.Contains(e.Error(), "no such file") {
				return filepath.SkipDir
			}

			log.Printf("prevent panic by handling failure accessing a path %q: %v\n", path, e)
			return e
		}
		// For the naive implementation, let's check every file, but really
		// here we need to bail out early instead and return filepath.SkipDir
		// anytime we take a wrong turn.
		if !f.IsDir() {
			// Let's see if we match any interest!
			for _, i := range intrs {
				r, err := regexp.MatchString("^"+i.GetPathRegexp(), path)
				if err == nil && r {
					// NOTE: this overwrites existing datatypes, under the assumption that the
					// minification has taken care of this.
					paths[path] = i.GetDataType()
				}
			}
		}
		return nil
	})

	if err != nil {
		return nil, err
	}

	var files [][]*pb.File
	var fs []*pb.File
	// TODO(paradoxengine): implement limits on count of files and size.
	for path, dataType := range paths {
		metadata, err := getMetadata(path)
		if err != nil {
			return nil, err
		}
		f := &pb.File{Metadata: metadata, DataChunks: nil}
		switch dataType {
		case minions.Interest_METADATA:
			break
		case minions.Interest_METADATA_AND_DATA:
			chunks, err := getDataChunks(path)
			if err != nil {
				return nil, err
			}
			f.DataChunks = chunks
			break
		default:
			return nil, errors.New("Unknown interest type")
		}
		fs = append(fs, f)
	}
	files = append(files, fs)

	return files, nil
}

// getMetadata is heavily linux skewed, but so is minions right now.
// It's fairly easy to port to windows by adding the appropriate data
// structure, but not planned for now.
func getMetadata(path string) (*minions.FileMetadata, error) {
	s, err := os.Stat(path)
	if err != nil {
		// I suspect this is too aggressive, as this can fail for a number
		// of reasons, including permissions. It might be wiser to log
		// and proceed, but let's try as is now.
		return nil, err
	}
	sys := s.Sys()
	if sys == nil {
		return nil, errors.New("cannot access OS-specific metadata")
	}

	m := &minions.FileMetadata{Path: path}
	// TODO(paradoxengine): these conversions are all over the place :-(
	m.OwnerUid = int32(sys.(*syscall.Stat_t).Uid)
	m.OwnerGid = int32(sys.(*syscall.Stat_t).Gid)
	m.Permissions = uint32(s.Mode())
	m.Size = s.Size()
	return m, nil
}

// getDataChunks splits the file at the path in a set of chunks.
func getDataChunks(path string) ([]*pb.DataChunk, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	var chunks []*pb.DataChunk
	// Arbitrary size of each data chunk.
	var chunkSize = 1024 * 1024 * 2
	dataLen := len(data)
	for i := 0; i < dataLen; i += chunkSize {
		var chunk []byte
		if i+chunkSize >= dataLen {
			chunk = data[i:]
		} else {
			chunk = data[i : i+chunkSize]
		}
		chunks = append(chunks, &pb.DataChunk{
			Offset: int64(i),
			Data:   chunk,
		})
	}
	return chunks, nil
}
<|endoftext|>


<commit_before>package aa

import (
	"encoding/json"
	"fmt"

	"github.com/codegangsta/cli"
	"github.com/mikebeyer/clc-sdk/sdk/clc"
)

// Commands exports the cli commands for the status package
func Commands(client *clc.Client) cli.Command {
	return cli.Command{
		Name:        "anti-alias",
		Aliases:     []string{"aa"},
		Usage:       "anti-alias api",
		Subcommands: []cli.Command{get(client)},
	}
}

func get(client *clc.Client) cli.Command {
	return cli.Command{
		Name:    "get",
		Aliases: []string{"g"},
		Usage:   "get aa policy",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "all", Usage: "retrieve all policies"},
			cli.StringFlag{Name: "alias, a", Usage: "account alias"},
		},
		Action: func(c *cli.Context) {
			if c.Bool("all") || c.Args().First() == "" {
				policies, err := client.AA.GetAll()
				if err != nil {
					fmt.Printf("unable to retrieve aa policies")
					return
				}
				b, err := json.MarshalIndent(policies, "", "  ")
				if err != nil {
					fmt.Printf("%s", err)
					return
				}
				fmt.Printf("%s\n", b)
			}

			policy, err := client.AA.Get(c.Args().First())
			if err != nil {
				fmt.Printf("unable to retrieve aa policy: [%s]", c.Args().First())
				return
			}
			b, err := json.MarshalIndent(policy, "", "  ")
			if err != nil {
				fmt.Printf("%s", err)
				return
			}
			fmt.Printf("%s\n", b)
		},
	}
}
<commit_msg>updated cli to support new AA featues.  uats to support<commit_after>package aa

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/codegangsta/cli"
	"github.com/mikebeyer/clc-sdk/sdk/clc"
)

// Commands exports the cli commands for the status package
func Commands(client *clc.Client) cli.Command {
	return cli.Command{
		Name:    "anti-alias",
		Aliases: []string{"aa"},
		Usage:   "anti-alias api",
		Subcommands: []cli.Command{
			get(client),
			create(client),
			delete(client),
		},
	}
}

func get(client *clc.Client) cli.Command {
	return cli.Command{
		Name:    "get",
		Aliases: []string{"g"},
		Usage:   "get aa policy",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "all", Usage: "retrieve all policies"},
			cli.StringFlag{Name: "alias, a", Usage: "account alias"},
		},
		Action: func(c *cli.Context) {
			if c.Bool("all") || c.Args().First() == "" {
				policies, err := client.AA.GetAll()
				if err != nil {
					fmt.Printf("unable to retrieve aa policies\n")
					return
				}
				b, err := json.MarshalIndent(policies, "", "  ")
				if err != nil {
					fmt.Printf("%s", err)
					return
				}
				fmt.Printf("%s\n", b)
				return
			}

			policy, err := client.AA.Get(c.Args().First())
			if err != nil {
				fmt.Printf("unable to retrieve aa policy: [%s]\n", c.Args().First())
				return
			}
			b, err := json.MarshalIndent(policy, "", "  ")
			if err != nil {
				fmt.Printf("%s", err)
				return
			}
			fmt.Printf("%s\n", b)
		},
	}
}

func create(client *clc.Client) cli.Command {
	return cli.Command{
		Name:    "create",
		Aliases: []string{"c"},
		Usage:   "create aa policy",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "name, n", Usage: "policy name [required]"},
			cli.StringFlag{Name: "location, l", Usage: "policy location [required]"},
		},
		Action: func(c *cli.Context) {
			name := c.String("name")
			loc := c.String("location")
			if name == "" || loc == "" {
				fmt.Printf("missing required flags to create policy. [use --help to show required flags]\n")
				return
			}

			policy, err := client.AA.Create(name, loc)
			if err != nil {
				fmt.Printf("failed to create policy %s in %s", name, loc)
				return
			}
			b, err := json.MarshalIndent(policy, "", "  ")
			if err != nil {
				fmt.Printf("%s", err)
				return
			}
			fmt.Printf("%s\n", b)
		},
	}
}

func delete(client *clc.Client) cli.Command {
	return cli.Command{
		Name:    "delete",
		Aliases: []string{"d"},
		Usage:   "delete aa policy",
		Before: func(c *cli.Context) error {
			if c.Args().First() == "" {
				fmt.Println("usage: delete [id]")
				return errors.New("")
			}
			return nil
		},
		Action: func(c *cli.Context) {
			err := client.AA.Delete(c.Args().First())
			if err != nil {
				fmt.Printf("unable to delete aa policy: [%s]\n", c.Args().First())
				return
			}
			fmt.Printf("deleted aa policy: %s\n", c.Args().First())
		},
	}
}
<|endoftext|>