<commit_before>package main

import (
	"fmt"
	"github.com/timakin/ts/loader"
	"github.com/codegangsta/cli"
)

var Commands = []cli.Command{
	commandAll,
	commandHack,
	commandPH,
	commandRE,
	commandHN,
	commandGH,
	commandMS,
	commandTNW,
	commandDN,
	commandFB,
	commandEJ,
	commandRD,
	commandA16Z,
	commandHatena,
}

var commandAll = cli.Command{
	Name:  "pop",
	Usage: "",
	Description: "Show today's news from major tech news sites, HN, PH, and subreddit of /programming.",
	Action: doAll,
}

var commandHack = cli.Command{
	Name:  "hack",
	Usage: "",
	Description: `
`,
	Action: doHack,
}

var commandPH = cli.Command{
	Name:  "ph",
	Usage: "",
	Description: `
`,
	Action: doPH,
}

var commandHN = cli.Command{
	Name:  "hn",
	Usage: "",
	Description: `
`,
	Action: doHN,
}

var commandGH = cli.Command{
	Name:  "github",
	Usage: "",
	Description: `
`,
	Action: doGH,
}

var commandRE = cli.Command{
	Name:  "reddit",
	Usage: "",
	Description: `
`,
	Action: doRE,
}

var commandTC = cli.Command{
	Name:  "tc",
	Usage: "",
	Description: `
`,
	Action: doTC,
}

var commandMS = cli.Command{
	Name:  "ms",
	Usage: "",
	Description: `
`,
	Action: doMS,
}

var commandTNW = cli.Command{
	Name:  "tnw",
	Usage: "",
	Description: `
`,
	Action: doTNW,
}

var commandDN = cli.Command{
	Name:  "dn",
	Usage: "",
	Description: `
`,
	Action: doDN,
}

var commandFB = cli.Command{
	Name:  "forbes",
	Usage: "",
	Description: `
`,
	Action: doFB,
}

var commandEJ = cli.Command{
	Name:  "echojs",
	Usage: "",
	Description: `
`,
	Action: doEJ,
}

var commandRD = cli.Command{
	Name:  "rdaily",
	Usage: "",
	Description: `
`,
	Action: doRD,
}

var commandA16Z = cli.Command{
	Name:  "a16z",
	Usage: "",
	Description: `
`,
	Action: doA16Z,
}

var commandHatena = cli.Command{
	Name:  "hatena",
	Usage: "",
	Description: `
`,
	Action: doHatena,
}

func pp(str string) {
  fmt.Printf(str)
}

func displayRSSFeed(name string, uri string) {
	pp("[" + name + "]\n")
	loader.GetRSSFeed(uri)
}

func displayRSSFeedWithDescription(name string, uri string) {
	pp("[" + name + "]\n")
	loader.GetRSSFeedWithDescription(uri)
}

func doAll(c *cli.Context) {
		pp("▁ ▂ ▄ ▅ ▆ ▇ █ тecнѕтacĸ █ ▇ ▆ ▅ ▄ ▂ ▁\n\n")
		ph := make(chan loader.ResultData)
		re := make(chan loader.ResultData)
		go loader.GetPHFeed(ph)
		go loader.GetRedditFeed(re)
		phres := <- ph
		reres := <- re
		var PHData loader.Feed = &phres
		var REData loader.Feed = &reres
		PHData.Display()
		REData.Display()
		displayRSSFeed("HackerNews", "https://news.ycombinator.com/rss")
		displayRSSFeedWithDescription("Github Trends", "http://github-trends.ryotarai.info/rss/github_trends_all_daily.rss")
		displayRSSFeed("TechCrunch", "http://feeds.feedburner.com/TechCrunch/")
		displayRSSFeed("Mashable", "http://feeds.mashable.com/Mashable")
		displayRSSFeed("The Next Web", "http://feeds2.feedburner.com/thenextweb")
		displayRSSFeed("Designer News", "https://news.layervault.com/?format=rss")
		displayRSSFeed("Forbes - Tech", "http://www.forbes.com/technology/feed/")
		displayRSSFeed("EchoJS", "http://www.echojs.com/rss")
		displayRSSFeed("RubyDaily", "http://feeds.rubydaily.org/RubyDaily")
		displayRSSFeed("A16Z", "http://a16z.com/feed/")
}

func doHack(c *cli.Context) {
	re := make(chan loader.ResultData)
	go loader.GetRedditFeed(re)
	reres := <- re
	var REData loader.Feed = &reres
	REData.Display()
	displayRSSFeed("HackerNews", "https://news.ycombinator.com/rss")
	displayRSSFeedWithDescription("Github Trends", "http://github-trends.ryotarai.info/rss/github_trends_all_daily.rss")
	displayRSSFeed("EchoJS", "http://www.echojs.com/rss")
	displayRSSFeed("RubyDaily", "http://feeds.rubydaily.org/RubyDaily")
}

func doPH(c *cli.Context) {
	ph := make(chan loader.ResultData)
	go loader.GetPHFeed(ph)
	phres := <- ph
	var PHData loader.Feed = &phres
	PHData.Display()
}

func doRE(c *cli.Context) {
	re := make(chan loader.ResultData)
	go loader.GetRedditFeed(re)
	reres := <- re
	var REData loader.Feed = &reres
	REData.Display()
}

func doHN(c *cli.Context) {
	displayRSSFeed("HackerNews", "https://news.ycombinator.com/rss")
}

func doGH(c *cli.Context) {
	displayRSSFeedWithDescription("Github Trends", "http://github-trends.ryotarai.info/rss/github_trends_all_daily.rss")
}

func doTC(c *cli.Context) {
	displayRSSFeed("TechCrunch", "http://feeds.feedburner.com/TechCrunch/")
}

func doMS(c *cli.Context) {
	displayRSSFeed("Mashable", "http://feeds.mashable.com/Mashable")
}

func doTNW(c *cli.Context) {
	displayRSSFeed("The Next Web", "http://feeds2.feedburner.com/thenextweb")
}

func doDN(c *cli.Context) {
	displayRSSFeed("Designer News", "https://news.layervault.com/?format=rss")
}

func doFB(c *cli.Context) {
	displayRSSFeed("Forbes - Tech", "http://www.forbes.com/technology/feed/")
}

func doEJ(c *cli.Context) {
	displayRSSFeed("EchoJS", "http://www.echojs.com/rss")
}

func doRD(c *cli.Context) {
	displayRSSFeed("RubyDaily", "http://feeds.rubydaily.org/RubyDaily")
}

func doA16Z(c *cli.Context) {
	displayRSSFeed("A16Z", "http://a16z.com/feed/")
}

func doHatena(c *cli.Context) {
	displayRSSFeed("Hatena", "http://b.hatena.ne.jp/search/tag?q=%E3%83%97%E3%83%AD%E3%82%B0%E3%83%A9%E3%83%9F%E3%83%B3%E3%82%B0&users=10&mode=rss")
}
<commit_msg>Fix TechCrunch feed command setting<commit_after>package main

import (
	"fmt"
	"github.com/timakin/ts/loader"
	"github.com/codegangsta/cli"
)

var Commands = []cli.Command{
	commandAll,
	commandHack,
	commandPH,
	commandTC,
	commandRE,
	commandHN,
	commandGH,
	commandMS,
	commandTNW,
	commandDN,
	commandFB,
	commandEJ,
	commandRD,
	commandA16Z,
	commandHatena,
}

var commandAll = cli.Command{
	Name:  "pop",
	Usage: "",
	Description: "Show today's news from major tech news sites, HN, PH, and subreddit of /programming.",
	Action: doAll,
}

var commandHack = cli.Command{
	Name:  "hack",
	Usage: "",
	Description: `
`,
	Action: doHack,
}

var commandPH = cli.Command{
	Name:  "ph",
	Usage: "",
	Description: `
`,
	Action: doPH,
}

var commandHN = cli.Command{
	Name:  "hn",
	Usage: "",
	Description: `
`,
	Action: doHN,
}

var commandGH = cli.Command{
	Name:  "github",
	Usage: "",
	Description: `
`,
	Action: doGH,
}

var commandRE = cli.Command{
	Name:  "reddit",
	Usage: "",
	Description: `
`,
	Action: doRE,
}

var commandTC = cli.Command{
	Name:  "tc",
	Usage: "",
	Description: `
`,
	Action: doTC,
}

var commandMS = cli.Command{
	Name:  "ms",
	Usage: "",
	Description: `
`,
	Action: doMS,
}

var commandTNW = cli.Command{
	Name:  "tnw",
	Usage: "",
	Description: `
`,
	Action: doTNW,
}

var commandDN = cli.Command{
	Name:  "dn",
	Usage: "",
	Description: `
`,
	Action: doDN,
}

var commandFB = cli.Command{
	Name:  "forbes",
	Usage: "",
	Description: `
`,
	Action: doFB,
}

var commandEJ = cli.Command{
	Name:  "echojs",
	Usage: "",
	Description: `
`,
	Action: doEJ,
}

var commandRD = cli.Command{
	Name:  "rdaily",
	Usage: "",
	Description: `
`,
	Action: doRD,
}

var commandA16Z = cli.Command{
	Name:  "a16z",
	Usage: "",
	Description: `
`,
	Action: doA16Z,
}

var commandHatena = cli.Command{
	Name:  "hatena",
	Usage: "",
	Description: `
`,
	Action: doHatena,
}

func pp(str string) {
  fmt.Printf(str)
}

func displayRSSFeed(name string, uri string) {
	pp("[" + name + "]\n")
	loader.GetRSSFeed(uri)
}

func displayRSSFeedWithDescription(name string, uri string) {
	pp("[" + name + "]\n")
	loader.GetRSSFeedWithDescription(uri)
}

func doAll(c *cli.Context) {
		pp("▁ ▂ ▄ ▅ ▆ ▇ █ тecнѕтacĸ █ ▇ ▆ ▅ ▄ ▂ ▁\n\n")
		ph := make(chan loader.ResultData)
		re := make(chan loader.ResultData)
		go loader.GetPHFeed(ph)
		go loader.GetRedditFeed(re)
		phres := <- ph
		reres := <- re
		var PHData loader.Feed = &phres
		var REData loader.Feed = &reres
		PHData.Display()
		REData.Display()
		displayRSSFeed("HackerNews", "https://news.ycombinator.com/rss")
		displayRSSFeedWithDescription("Github Trends", "http://github-trends.ryotarai.info/rss/github_trends_all_daily.rss")
		displayRSSFeed("TechCrunch", "http://feeds.feedburner.com/TechCrunch/")
		displayRSSFeed("Mashable", "http://feeds.mashable.com/Mashable")
		displayRSSFeed("The Next Web", "http://feeds2.feedburner.com/thenextweb")
		displayRSSFeed("Designer News", "https://news.layervault.com/?format=rss")
		displayRSSFeed("Forbes - Tech", "http://www.forbes.com/technology/feed/")
		displayRSSFeed("EchoJS", "http://www.echojs.com/rss")
		displayRSSFeed("RubyDaily", "http://feeds.rubydaily.org/RubyDaily")
		displayRSSFeed("A16Z", "http://a16z.com/feed/")
}

func doHack(c *cli.Context) {
	re := make(chan loader.ResultData)
	go loader.GetRedditFeed(re)
	reres := <- re
	var REData loader.Feed = &reres
	REData.Display()
	displayRSSFeed("HackerNews", "https://news.ycombinator.com/rss")
	displayRSSFeedWithDescription("Github Trends", "http://github-trends.ryotarai.info/rss/github_trends_all_daily.rss")
	displayRSSFeed("EchoJS", "http://www.echojs.com/rss")
	displayRSSFeed("RubyDaily", "http://feeds.rubydaily.org/RubyDaily")
}

func doPH(c *cli.Context) {
	ph := make(chan loader.ResultData)
	go loader.GetPHFeed(ph)
	phres := <- ph
	var PHData loader.Feed = &phres
	PHData.Display()
}

func doRE(c *cli.Context) {
	re := make(chan loader.ResultData)
	go loader.GetRedditFeed(re)
	reres := <- re
	var REData loader.Feed = &reres
	REData.Display()
}

func doHN(c *cli.Context) {
	displayRSSFeed("HackerNews", "https://news.ycombinator.com/rss")
}

func doGH(c *cli.Context) {
	displayRSSFeedWithDescription("Github Trends", "http://github-trends.ryotarai.info/rss/github_trends_all_daily.rss")
}

func doTC(c *cli.Context) {
	displayRSSFeed("TechCrunch", "http://feeds.feedburner.com/TechCrunch/")
}

func doMS(c *cli.Context) {
	displayRSSFeed("Mashable", "http://feeds.mashable.com/Mashable")
}

func doTNW(c *cli.Context) {
	displayRSSFeed("The Next Web", "http://feeds2.feedburner.com/thenextweb")
}

func doDN(c *cli.Context) {
	displayRSSFeed("Designer News", "https://news.layervault.com/?format=rss")
}

func doFB(c *cli.Context) {
	displayRSSFeed("Forbes - Tech", "http://www.forbes.com/technology/feed/")
}

func doEJ(c *cli.Context) {
	displayRSSFeed("EchoJS", "http://www.echojs.com/rss")
}

func doRD(c *cli.Context) {
	displayRSSFeed("RubyDaily", "http://feeds.rubydaily.org/RubyDaily")
}

func doA16Z(c *cli.Context) {
	displayRSSFeed("A16Z", "http://a16z.com/feed/")
}

func doHatena(c *cli.Context) {
	displayRSSFeed("Hatena", "http://b.hatena.ne.jp/search/tag?q=%E3%83%97%E3%83%AD%E3%82%B0%E3%83%A9%E3%83%9F%E3%83%B3%E3%82%B0&users=10&mode=rss")
}
<|endoftext|>


<commit_before>package restful

// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

import (
	"compress/gzip"
	"compress/zlib"
	"errors"
	"io"
	"net/http"
	"strings"
)

// OBSOLETE : use restful.DefaultContainer.EnableContentEncoding(true) to change this setting.
var EnableContentEncoding = false

// CompressingResponseWriter is a http.ResponseWriter that can perform content encoding (gzip and zlib)
type CompressingResponseWriter struct {
	writer     http.ResponseWriter
	compressor io.WriteCloser
	encoding   string
}

// Header is part of http.ResponseWriter interface
func (c *CompressingResponseWriter) Header() http.Header {
	return c.writer.Header()
}

// WriteHeader is part of http.ResponseWriter interface
func (c *CompressingResponseWriter) WriteHeader(status int) {
	c.writer.WriteHeader(status)
}

// Write is part of http.ResponseWriter interface
// It is passed through the compressor
func (c *CompressingResponseWriter) Write(bytes []byte) (int, error) {
	if c.isCompressorClosed() {
		return -1, errors.New("Compressing error: tried to write data using closed compressor")
	}
	return c.compressor.Write(bytes)
}

// CloseNotify is part of http.CloseNotifier interface
func (c *CompressingResponseWriter) CloseNotify() <-chan bool {
	return c.writer.(http.CloseNotifier).CloseNotify()
}

// Close the underlying compressor
func (c *CompressingResponseWriter) Close() error {
	if c.isCompressorClosed() {
		return errors.New("Compressing error: tried to close already closed compressor")
	}

	c.compressor.Close()
	if ENCODING_GZIP == c.encoding {
		currentCompressorProvider.ReleaseGzipWriter(c.compressor.(*gzip.Writer))
	}
	if ENCODING_DEFLATE == c.encoding {
		currentCompressorProvider.ReleaseZlibWriter(c.compressor.(*zlib.Writer))
	}
	// gc hint needed?
	c.compressor = nil
	return nil
}

func (c *CompressingResponseWriter) isCompressorClosed() bool {
	return nil == c.compressor
}

// WantsCompressedResponse reads the Accept-Encoding header to see if and which encoding is requested.
func wantsCompressedResponse(httpRequest *http.Request) (bool, string) {
	header := httpRequest.Header.Get(HEADER_AcceptEncoding)
	gi := strings.Index(header, ENCODING_GZIP)
	zi := strings.Index(header, ENCODING_DEFLATE)
	// use in order of appearance
	if gi == -1 {
		return zi != -1, ENCODING_DEFLATE
	} else if zi == -1 {
		return gi != -1, ENCODING_GZIP
	} else {
		if gi < zi {
			return true, ENCODING_GZIP
		}
		return true, ENCODING_DEFLATE
	}
}

// NewCompressingResponseWriter create a CompressingResponseWriter for a known encoding = {gzip,deflate}
func NewCompressingResponseWriter(httpWriter http.ResponseWriter, encoding string) (*CompressingResponseWriter, error) {
	httpWriter.Header().Set(HEADER_ContentEncoding, encoding)
	c := new(CompressingResponseWriter)
	c.writer = httpWriter
	var err error
	if ENCODING_GZIP == encoding {
		w := currentCompressorProvider.AcquireGzipWriter()
		w.Reset(httpWriter)
		c.compressor = w
		c.encoding = ENCODING_GZIP
	} else if ENCODING_DEFLATE == encoding {
		w := currentCompressorProvider.AcquireZlibWriter()
		w.Reset(httpWriter)
		c.compressor = w
		c.encoding = ENCODING_DEFLATE
	} else {
		return nil, errors.New("Unknown encoding:" + encoding)
	}
	return c, err
}
<commit_msg>Implement http.Hijacjer interface on CompressingResponseWriter<commit_after>package restful

// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

import (
	"bufio"
	"compress/gzip"
	"compress/zlib"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
)

// OBSOLETE : use restful.DefaultContainer.EnableContentEncoding(true) to change this setting.
var EnableContentEncoding = false

// CompressingResponseWriter is a http.ResponseWriter that can perform content encoding (gzip and zlib)
type CompressingResponseWriter struct {
	writer     http.ResponseWriter
	compressor io.WriteCloser
	encoding   string
}

// Header is part of http.ResponseWriter interface
func (c *CompressingResponseWriter) Header() http.Header {
	return c.writer.Header()
}

// WriteHeader is part of http.ResponseWriter interface
func (c *CompressingResponseWriter) WriteHeader(status int) {
	c.writer.WriteHeader(status)
}

// Write is part of http.ResponseWriter interface
// It is passed through the compressor
func (c *CompressingResponseWriter) Write(bytes []byte) (int, error) {
	if c.isCompressorClosed() {
		return -1, errors.New("Compressing error: tried to write data using closed compressor")
	}
	return c.compressor.Write(bytes)
}

// CloseNotify is part of http.CloseNotifier interface
func (c *CompressingResponseWriter) CloseNotify() <-chan bool {
	return c.writer.(http.CloseNotifier).CloseNotify()
}

// Close the underlying compressor
func (c *CompressingResponseWriter) Close() error {
	if c.isCompressorClosed() {
		return errors.New("Compressing error: tried to close already closed compressor")
	}

	c.compressor.Close()
	if ENCODING_GZIP == c.encoding {
		currentCompressorProvider.ReleaseGzipWriter(c.compressor.(*gzip.Writer))
	}
	if ENCODING_DEFLATE == c.encoding {
		currentCompressorProvider.ReleaseZlibWriter(c.compressor.(*zlib.Writer))
	}
	// gc hint needed?
	c.compressor = nil
	return nil
}

func (c *CompressingResponseWriter) isCompressorClosed() bool {
	return nil == c.compressor
}

// Hijack implements the Hijacker interface
// This is especially useful when combining Container.EnabledContentEncoding
// in combination with websockets (for instance gorilla/websocket)
func (c *CompressingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hijacker, ok := c.writer.(http.Hijacker)
	if !ok {
		return nil, nil, fmt.Errorf("ResponseWriter doesn't support Hijacker interface")
	}
	return hijacker.Hijack()
}

// WantsCompressedResponse reads the Accept-Encoding header to see if and which encoding is requested.
func wantsCompressedResponse(httpRequest *http.Request) (bool, string) {
	header := httpRequest.Header.Get(HEADER_AcceptEncoding)
	gi := strings.Index(header, ENCODING_GZIP)
	zi := strings.Index(header, ENCODING_DEFLATE)
	// use in order of appearance
	if gi == -1 {
		return zi != -1, ENCODING_DEFLATE
	} else if zi == -1 {
		return gi != -1, ENCODING_GZIP
	} else {
		if gi < zi {
			return true, ENCODING_GZIP
		}
		return true, ENCODING_DEFLATE
	}
}

// NewCompressingResponseWriter create a CompressingResponseWriter for a known encoding = {gzip,deflate}
func NewCompressingResponseWriter(httpWriter http.ResponseWriter, encoding string) (*CompressingResponseWriter, error) {
	httpWriter.Header().Set(HEADER_ContentEncoding, encoding)
	c := new(CompressingResponseWriter)
	c.writer = httpWriter
	var err error
	if ENCODING_GZIP == encoding {
		w := currentCompressorProvider.AcquireGzipWriter()
		w.Reset(httpWriter)
		c.compressor = w
		c.encoding = ENCODING_GZIP
	} else if ENCODING_DEFLATE == encoding {
		w := currentCompressorProvider.AcquireZlibWriter()
		w.Reset(httpWriter)
		c.compressor = w
		c.encoding = ENCODING_DEFLATE
	} else {
		return nil, errors.New("Unknown encoding:" + encoding)
	}
	return c, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"sync"
)

var mu sync.Mutex
var count int

func main() {
	log.Print("Server running...")
	http.HandleFunc("/", handler)
	http.HandleFunc("/count", counter)
	log.Fatal(http.ListenAndServe("localhost:8000", nil))
}

func handler(w http.ResponseWriter, r *http.Request) {
	mu.Lock()
	count++
	mu.Unlock()
	fmt.Fprintf(w, "%s %s %s\n", r.Method, r.URL, r.Proto)
	for k, v := range r.Header {
		fmt.Fprintf(w, "Header[%q]: %q\n", k, v)
	}
	fmt.Fprintf(w, "Host: %q\n", r.Host)
	fmt.Fprintf(w, "RemoteAddr: %q\n", r.RemoteAddr)
	if err := r.ParseForm(); err != nil {
		log.Print(err)
	}
	for k, v := range r.Form {
		fmt.Fprintf(w, "Form[%q]: %q\n", k, v)
	}
}

func counter(w http.ResponseWriter, r *http.Request) {
	mu.Lock()
	fmt.Fprintf(w, "Count: %d\n", count)
	mu.Unlock()
}
<commit_msg>Log request info rather than echoing.<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"sync"
)

var mu sync.Mutex
var count int

func main() {
	log.Print("Server running...")
	http.HandleFunc("/", handler)
	http.HandleFunc("/count", counter)
	log.Fatal(http.ListenAndServe("localhost:8000", nil))
}

func handler(w http.ResponseWriter, r *http.Request) {
	mu.Lock()
	count++
	mu.Unlock()
	log.Printf("%s %s %s\n", r.Method, r.URL, r.Proto)
	for k, v := range r.Header {
		log.Printf("Header[%q]: %q\n", k, v)
	}
	log.Printf("Host: %q\n", r.Host)
	log.Printf("RemoteAddr: %q\n", r.RemoteAddr)
	if err := r.ParseForm(); err != nil {
		log.Print(err)
	}
	for k, v := range r.Form {
		log.Printf("Form[%q]: %q\n", k, v)
	}
}

func counter(w http.ResponseWriter, r *http.Request) {
	mu.Lock()
	fmt.Fprintf(w, "Count: %d\n", count)
	mu.Unlock()
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"text/template"

	services "github.com/maximilien/softlayer-go/services"
	softlayer "github.com/maximilien/softlayer-go/softlayer"
)

const (
	SOFTLAYER_API_URL  = "api.softlayer.com/rest/v3"
	TEMPLATE_ROOT_PATH = "templates"
)

type softLayerClient struct {
	username string
	apiKey   string

	templatePath string

	httpClient *http.Client

	softLayerServices map[string]softlayer.Service
}

func NewSoftLayerClient(username, apiKey string) *softLayerClient {
	pwd, _ := os.Getwd()
	slc := &softLayerClient{
		username: username,
		apiKey:   apiKey,

		templatePath: filepath.Join(pwd, TEMPLATE_ROOT_PATH),

		httpClient: &http.Client{
			Transport: &http.Transport{
				Proxy: http.ProxyFromEnvironment,
			},
		},

		softLayerServices: map[string]softlayer.Service{},
	}

	slc.initSoftLayerServices()

	return slc
}

//softlayer.Client interface methods

func (slc *softLayerClient) GetService(serviceName string) (softlayer.Service, error) {
	slService, ok := slc.softLayerServices[serviceName]
	if !ok {
		return nil, errors.New(fmt.Sprintf("softlayer-go does not support service '%s'", serviceName))
	}

	return slService, nil
}

func (slc *softLayerClient) GetSoftLayer_Account_Service() (softlayer.SoftLayer_Account_Service, error) {
	slService, err := slc.GetService("SoftLayer_Account")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Account_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Virtual_Guest_Service() (softlayer.SoftLayer_Virtual_Guest_Service, error) {
	slService, err := slc.GetService("SoftLayer_Virtual_Guest")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Virtual_Guest_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Virtual_Disk_Image_Service() (softlayer.SoftLayer_Virtual_Disk_Image_Service, error) {
	slService, err := slc.GetService("SoftLayer_Virtual_Disk_Image")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Virtual_Disk_Image_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Security_Ssh_Key_Service() (softlayer.SoftLayer_Security_Ssh_Key_Service, error) {
	slService, err := slc.GetService("SoftLayer_Security_Ssh_Key")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Security_Ssh_Key_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Product_Package_Service() (softlayer.SoftLayer_Product_Package_Service, error) {
	slService, err := slc.GetService("SoftLayer_Product_Package")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Product_Package_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Virtual_Guest_Block_Device_Template_Group_Service() (softlayer.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Service, error) {
	slService, err := slc.GetService("SoftLayer_Virtual_Guest_Block_Device_Template_Group_Service")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Network_Storage_Service() (softlayer.SoftLayer_Network_Storage_Service, error) {
	slService, err := slc.GetService("SoftLayer_Network_Storage")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Network_Storage_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Product_Order_Service() (softlayer.SoftLayer_Product_Order_Service, error) {
	slService, err := slc.GetService("SoftLayer_Product_Order")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Product_Order_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Billing_Item_Cancellation_Request_Service() (softlayer.SoftLayer_Billing_Item_Cancellation_Request_Service, error) {
	slService, err := slc.GetService("SoftLayer_Billing_Item_Cancellation_Request")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Billing_Item_Cancellation_Request_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Hardware_Service() (softlayer.SoftLayer_Hardware_Service, error) {
	slService, err := slc.GetService("SoftLayer_Hardware")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Hardware_Service), nil
}

//Public methods

func (slc *softLayerClient) DoRawHttpRequestWithObjectMask(path string, masks []string, requestType string, requestBody *bytes.Buffer) ([]byte, error) {
	url := fmt.Sprintf("https://%s:%s@%s/%s", slc.username, slc.apiKey, SOFTLAYER_API_URL, path)

	url += "?objectMask="
	for i := 0; i < len(masks); i++ {
		url += masks[i]
		if i != len(masks)-1 {
			url += ";"
		}
	}

	return slc.makeHttpRequest(url, requestType, requestBody)
}

func (slc *softLayerClient) DoRawHttpRequest(path string, requestType string, requestBody *bytes.Buffer) ([]byte, error) {
	url := fmt.Sprintf("https://%s:%s@%s/%s", slc.username, slc.apiKey, SOFTLAYER_API_URL, path)
	return slc.makeHttpRequest(url, requestType, requestBody)
}

func (slc *softLayerClient) GenerateRequestBody(templateData interface{}) (*bytes.Buffer, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	bodyTemplate := template.Must(template.ParseFiles(filepath.Join(cwd, slc.templatePath)))
	body := new(bytes.Buffer)
	bodyTemplate.Execute(body, templateData)

	return body, nil
}

func (slc *softLayerClient) HasErrors(body map[string]interface{}) error {
	if errString, ok := body["error"]; !ok {
		return nil
	} else {
		return errors.New(errString.(string))
	}
}

func (slc *softLayerClient) CheckForHttpResponseErrors(data []byte) error {
	var decodedResponse map[string]interface{}
	err := json.Unmarshal(data, &decodedResponse)
	if err != nil {
		return err
	}

	if err := slc.HasErrors(decodedResponse); err != nil {
		return err
	}

	return nil
}

//Private methods

func (slc *softLayerClient) initSoftLayerServices() {
	slc.softLayerServices["SoftLayer_Account"] = services.NewSoftLayer_Account_Service(slc)
	slc.softLayerServices["SoftLayer_Virtual_Guest"] = services.NewSoftLayer_Virtual_Guest_Service(slc)
	slc.softLayerServices["SoftLayer_Virtual_Disk_Image"] = services.NewSoftLayer_Virtual_Disk_Image_Service(slc)
	slc.softLayerServices["SoftLayer_Security_Ssh_Key"] = services.NewSoftLayer_Security_Ssh_Key_Service(slc)
	slc.softLayerServices["SoftLayer_Product_Package"] = services.NewSoftLayer_Product_Package_Service(slc)
	slc.softLayerServices["SoftLayer_Network_Storage"] = services.NewSoftLayer_Network_Storage_Service(slc)
	slc.softLayerServices["SoftLayer_Product_Order"] = services.NewSoftLayer_Product_Order_Service(slc)
	slc.softLayerServices["SoftLayer_Billing_Item_Cancellation_Request"] = services.NewSoftLayer_Billing_Item_Cancellation_Request_Service(slc)
	slc.softLayerServices["SoftLayer_Virtual_Guest_Block_Device_Template_Group"] = services.NewSoftLayer_Virtual_Guest_Block_Device_Template_Group_Service(slc)
	slc.softLayerServices["SoftLayer_Hardware"] = services.NewSoftLayer_Hardware_Service(slc)
}

func (slc *softLayerClient) makeHttpRequest(url string, requestType string, requestBody *bytes.Buffer) ([]byte, error) {
	req, err := http.NewRequest(requestType, url, requestBody)
	if err != nil {
		return nil, err
	}

	bs, err := httputil.DumpRequest(req, true)
	if err != nil {
		return nil, err
	}

	fmt.Fprintf(os.Stderr, "\n---\n[softlayer-go] Request:\n%s\n", string(bs))

	resp, err := slc.httpClient.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	bs, err = httputil.DumpResponse(resp, true)
	if err != nil {
		return nil, err
	}

	fmt.Fprintf(os.Stderr, "[softlayer-go] Response:\n%s\n", string(bs))

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return responseBody, nil
}
<commit_msg>added ability to switch SL Go to non-verbose by defining environmen SL_GO_NON_VERBOSE<commit_after>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"text/template"

	services "github.com/maximilien/softlayer-go/services"
	softlayer "github.com/maximilien/softlayer-go/softlayer"
)

const (
	SOFTLAYER_API_URL  = "api.softlayer.com/rest/v3"
	TEMPLATE_ROOT_PATH = "templates"
	SL_GO_NON_VERBOSE  = "SL_GO_NON_VERBOSE"
)

type softLayerClient struct {
	username string
	apiKey   string

	templatePath string

	httpClient *http.Client

	softLayerServices map[string]softlayer.Service

	nonVerbose bool
}

func NewSoftLayerClient(username, apiKey string) *softLayerClient {
	pwd, _ := os.Getwd()
	slc := &softLayerClient{
		username: username,
		apiKey:   apiKey,

		templatePath: filepath.Join(pwd, TEMPLATE_ROOT_PATH),

		httpClient: &http.Client{
			Transport: &http.Transport{
				Proxy: http.ProxyFromEnvironment,
			},
		},

		nonVerbose: checkNonVerbose(),

		softLayerServices: map[string]softlayer.Service{},
	}

	slc.initSoftLayerServices()

	return slc
}

//softlayer.Client interface methods

func (slc *softLayerClient) GetService(serviceName string) (softlayer.Service, error) {
	slService, ok := slc.softLayerServices[serviceName]
	if !ok {
		return nil, errors.New(fmt.Sprintf("softlayer-go does not support service '%s'", serviceName))
	}

	return slService, nil
}

func (slc *softLayerClient) GetSoftLayer_Account_Service() (softlayer.SoftLayer_Account_Service, error) {
	slService, err := slc.GetService("SoftLayer_Account")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Account_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Virtual_Guest_Service() (softlayer.SoftLayer_Virtual_Guest_Service, error) {
	slService, err := slc.GetService("SoftLayer_Virtual_Guest")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Virtual_Guest_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Virtual_Disk_Image_Service() (softlayer.SoftLayer_Virtual_Disk_Image_Service, error) {
	slService, err := slc.GetService("SoftLayer_Virtual_Disk_Image")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Virtual_Disk_Image_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Security_Ssh_Key_Service() (softlayer.SoftLayer_Security_Ssh_Key_Service, error) {
	slService, err := slc.GetService("SoftLayer_Security_Ssh_Key")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Security_Ssh_Key_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Product_Package_Service() (softlayer.SoftLayer_Product_Package_Service, error) {
	slService, err := slc.GetService("SoftLayer_Product_Package")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Product_Package_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Virtual_Guest_Block_Device_Template_Group_Service() (softlayer.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Service, error) {
	slService, err := slc.GetService("SoftLayer_Virtual_Guest_Block_Device_Template_Group_Service")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Network_Storage_Service() (softlayer.SoftLayer_Network_Storage_Service, error) {
	slService, err := slc.GetService("SoftLayer_Network_Storage")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Network_Storage_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Product_Order_Service() (softlayer.SoftLayer_Product_Order_Service, error) {
	slService, err := slc.GetService("SoftLayer_Product_Order")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Product_Order_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Billing_Item_Cancellation_Request_Service() (softlayer.SoftLayer_Billing_Item_Cancellation_Request_Service, error) {
	slService, err := slc.GetService("SoftLayer_Billing_Item_Cancellation_Request")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Billing_Item_Cancellation_Request_Service), nil
}

func (slc *softLayerClient) GetSoftLayer_Hardware_Service() (softlayer.SoftLayer_Hardware_Service, error) {
	slService, err := slc.GetService("SoftLayer_Hardware")
	if err != nil {
		return nil, err
	}

	return slService.(softlayer.SoftLayer_Hardware_Service), nil
}

//Public methods

func (slc *softLayerClient) DoRawHttpRequestWithObjectMask(path string, masks []string, requestType string, requestBody *bytes.Buffer) ([]byte, error) {
	url := fmt.Sprintf("https://%s:%s@%s/%s", slc.username, slc.apiKey, SOFTLAYER_API_URL, path)

	url += "?objectMask="
	for i := 0; i < len(masks); i++ {
		url += masks[i]
		if i != len(masks)-1 {
			url += ";"
		}
	}

	return slc.makeHttpRequest(url, requestType, requestBody)
}

func (slc *softLayerClient) DoRawHttpRequest(path string, requestType string, requestBody *bytes.Buffer) ([]byte, error) {
	url := fmt.Sprintf("https://%s:%s@%s/%s", slc.username, slc.apiKey, SOFTLAYER_API_URL, path)
	return slc.makeHttpRequest(url, requestType, requestBody)
}

func (slc *softLayerClient) GenerateRequestBody(templateData interface{}) (*bytes.Buffer, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	bodyTemplate := template.Must(template.ParseFiles(filepath.Join(cwd, slc.templatePath)))
	body := new(bytes.Buffer)
	bodyTemplate.Execute(body, templateData)

	return body, nil
}

func (slc *softLayerClient) HasErrors(body map[string]interface{}) error {
	if errString, ok := body["error"]; !ok {
		return nil
	} else {
		return errors.New(errString.(string))
	}
}

func (slc *softLayerClient) CheckForHttpResponseErrors(data []byte) error {
	var decodedResponse map[string]interface{}
	err := json.Unmarshal(data, &decodedResponse)
	if err != nil {
		return err
	}

	if err := slc.HasErrors(decodedResponse); err != nil {
		return err
	}

	return nil
}

//Private methods

func (slc *softLayerClient) initSoftLayerServices() {
	slc.softLayerServices["SoftLayer_Account"] = services.NewSoftLayer_Account_Service(slc)
	slc.softLayerServices["SoftLayer_Virtual_Guest"] = services.NewSoftLayer_Virtual_Guest_Service(slc)
	slc.softLayerServices["SoftLayer_Virtual_Disk_Image"] = services.NewSoftLayer_Virtual_Disk_Image_Service(slc)
	slc.softLayerServices["SoftLayer_Security_Ssh_Key"] = services.NewSoftLayer_Security_Ssh_Key_Service(slc)
	slc.softLayerServices["SoftLayer_Product_Package"] = services.NewSoftLayer_Product_Package_Service(slc)
	slc.softLayerServices["SoftLayer_Network_Storage"] = services.NewSoftLayer_Network_Storage_Service(slc)
	slc.softLayerServices["SoftLayer_Product_Order"] = services.NewSoftLayer_Product_Order_Service(slc)
	slc.softLayerServices["SoftLayer_Billing_Item_Cancellation_Request"] = services.NewSoftLayer_Billing_Item_Cancellation_Request_Service(slc)
	slc.softLayerServices["SoftLayer_Virtual_Guest_Block_Device_Template_Group"] = services.NewSoftLayer_Virtual_Guest_Block_Device_Template_Group_Service(slc)
	slc.softLayerServices["SoftLayer_Hardware"] = services.NewSoftLayer_Hardware_Service(slc)
}

func (slc *softLayerClient) makeHttpRequest(url string, requestType string, requestBody *bytes.Buffer) ([]byte, error) {
	req, err := http.NewRequest(requestType, url, requestBody)
	if err != nil {
		return nil, err
	}

	bs, err := httputil.DumpRequest(req, true)
	if err != nil {
		return nil, err
	}

	if !slc.nonVerbose {
		fmt.Fprintf(os.Stderr, "\n---\n[softlayer-go] Request:\n%s\n", string(bs))
	}

	resp, err := slc.httpClient.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	bs, err = httputil.DumpResponse(resp, true)
	if err != nil {
		return nil, err
	}

	if !slc.nonVerbose {
		fmt.Fprintf(os.Stderr, "[softlayer-go] Response:\n%s\n", string(bs))
	}

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return responseBody, nil
}

//Private helper methods

func checkNonVerbose() bool {
	slGoNonVerbose := os.Getenv(SL_GO_NON_VERBOSE)
	switch slGoNonVerbose {
	case "yes":
		return true
	case "YES":
		return true
	case "true":
		return true
	case "TRUE":
		return true
	}

	return false
}
<|endoftext|>


<commit_before>package cioutil

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/Sirupsen/logrus"
	"github.com/garyburd/go-oauth/oauth"
)

// ClientRequest defines information that can be used to make a request
type ClientRequest struct {
	Method      string
	Path        string
	FormValues  interface{}
	QueryValues interface{}
}

// DoFormRequest makes the actual request
func (cio Cio) DoFormRequest(request ClientRequest, result interface{}) error {

	// Construct the url
	cioURL := cio.Host + request.Path + QueryString(request.QueryValues)

	// Construct the body
	var bodyReader io.Reader
	bodyValues := FormValues(request.FormValues)
	bodyString := bodyValues.Encode()
	if len(bodyString) > 0 {
		bodyReader = bytes.NewReader([]byte(bodyString))
	}
	logRequest(cio.Log, request.Method, cioURL, bodyValues)

	// Construct the request
	httpReq, err := cio.createRequest(request, cioURL, bodyReader, bodyValues)
	if err != nil {
		return err
	}

	// Send the request
	return cio.sendRequest(httpReq, result, cioURL)
}

// createRequest creates the *http.Request object
func (cio Cio) createRequest(request ClientRequest, cioURL string, bodyReader io.Reader, bodyValues url.Values) (*http.Request, error) {
	// Construct the request
	httpReq, err := http.NewRequest(request.Method, cioURL, bodyReader)
	if err != nil {
		return httpReq, fmt.Errorf("Could not create request: %s", err)
	}

	// oAuth signature
	var client oauth.Client
	client.Credentials = oauth.Credentials{Token: cio.apiKey, Secret: cio.apiSecret}

	// Add headers
	httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	httpReq.Header.Set("Accept", "application/json")
	httpReq.Header.Set("Accept-Charset", "utf-8")
	httpReq.Header.Set("User-Agent", "Golang CIO Library")
	httpReq.Header.Set("Authorization", client.AuthorizationHeader(nil, request.Method, httpReq.URL, bodyValues))

	return httpReq, nil
}

// sendRequest sends the *http.Request
func (cio Cio) sendRequest(httpReq *http.Request, result interface{}, cioURL string) error {
	// Create the HTTP client
	httpClient := &http.Client{
		Transport: http.DefaultTransport,
		Timeout:   cio.RequestTimeout,
	}

	// Make the request
	res, err := httpClient.Do(httpReq)
	if err != nil {
		return fmt.Errorf("Failed to make request: %s", err)
	}

	// Parse the response
	defer func() {
		if closeErr := res.Body.Close(); closeErr != nil {
			logBodyCloseError(cio.Log, closeErr)
		}
	}()

	resBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return fmt.Errorf("Could not read response: %s", err)
	}
	resBodyString := string(resBody)

	// Unmarshal result
	err = json.Unmarshal(resBody, &result)

	// Log the response
	logResponse(cio.Log, httpReq.Method, cioURL, res.StatusCode, resBodyString, err)

	// Return special error if Status Code >= 400
	if res.StatusCode >= 400 {
		return fmt.Errorf("%d Status Code with Payload %s", res.StatusCode, resBodyString)
	}

	// Return Unmarshal error (if any) if Status Code is < 400
	return err
}

// logRequest logs the request about to be made to CIO, redacting sensitive information in the body
func logRequest(log Logger, method string, cioURL string, bodyValues url.Values) {
	if log != nil {

		// Copy url.Values
		redactedValues := url.Values{}
		for k, v := range bodyValues {
			redactedValues[k] = v
		}

		// Redact sensitive information
		if val := redactedValues.Get("password"); len(val) > 0 {
			redactedValues.Set("password", "redacted")
		}
		if val := redactedValues.Get("provider_refresh_token"); len(val) > 0 {
			redactedValues.Set("provider_refresh_token", "redacted")
		}
		if val := redactedValues.Get("provider_consumer_key"); len(val) > 0 {
			redactedValues.Set("provider_consumer_key", "redacted")
		}
		if val := redactedValues.Get("provider_consumer_secret"); len(val) > 0 {
			redactedValues.Set("provider_consumer_secret", "redacted")
		}

		// Actually log
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithFields(logrus.Fields{"httpMethod": method, "url": cioURL, "payload": redactedValues.Encode()}).Debug("Creating new request to CIO")
		} else {
			// Else just log with Println
			log.Println("Creating new " + method + " request to: " + cioURL + " with payload: " + redactedValues.Encode())
		}
	}
}

// logBodyCloseError logs any error that happens when trying to close the *http.Response.Body
func logBodyCloseError(log Logger, closeError error) {
	if log != nil {
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithError(closeError).Warn("Unable to close response body from CIO")
		} else {
			// Else just log with Println
			log.Println("Unable to close response body from CIO, with error: " + closeError.Error())
		}
	}
}

// logResponse logs the response from CIO, if any logger is set
func logResponse(log Logger, method string, cioURL string, statusCode int, responseBody string, unmarshalError error) {
	if log != nil {

		// TODO: redact access_token and access_token_secret before logging (only occurs with 3-legged oauth [rare])

		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logEntry := logrusLogger.WithFields(logrus.Fields{
				"httpMethod": method,
				"url":        cioURL,
				"statusCode": fmt.Sprintf("%d", statusCode),
				"payload":    responseBody})
			if unmarshalError != nil || statusCode >= 400 {
				logEntry.Warn("Received response from CIO")
			} else {
				logEntry.Debug("Received response from CIO")
			}

		} else {
			// Else just log with Println
			log.Println("Received response from " + method + " to: " + cioURL +
				" with status code: " + fmt.Sprintf("%d", statusCode) +
				" and payload: " + responseBody)
		}
	}
}
<commit_msg>only log a snippet of the returned payload<commit_after>package cioutil

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/Sirupsen/logrus"
	"github.com/garyburd/go-oauth/oauth"
)

// ClientRequest defines information that can be used to make a request
type ClientRequest struct {
	Method      string
	Path        string
	FormValues  interface{}
	QueryValues interface{}
}

// DoFormRequest makes the actual request
func (cio Cio) DoFormRequest(request ClientRequest, result interface{}) error {

	// Construct the url
	cioURL := cio.Host + request.Path + QueryString(request.QueryValues)

	// Construct the body
	var bodyReader io.Reader
	bodyValues := FormValues(request.FormValues)
	bodyString := bodyValues.Encode()
	if len(bodyString) > 0 {
		bodyReader = bytes.NewReader([]byte(bodyString))
	}
	logRequest(cio.Log, request.Method, cioURL, bodyValues)

	// Construct the request
	httpReq, err := cio.createRequest(request, cioURL, bodyReader, bodyValues)
	if err != nil {
		return err
	}

	// Send the request
	return cio.sendRequest(httpReq, result, cioURL)
}

// createRequest creates the *http.Request object
func (cio Cio) createRequest(request ClientRequest, cioURL string, bodyReader io.Reader, bodyValues url.Values) (*http.Request, error) {
	// Construct the request
	httpReq, err := http.NewRequest(request.Method, cioURL, bodyReader)
	if err != nil {
		return httpReq, fmt.Errorf("Could not create request: %s", err)
	}

	// oAuth signature
	var client oauth.Client
	client.Credentials = oauth.Credentials{Token: cio.apiKey, Secret: cio.apiSecret}

	// Add headers
	httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	httpReq.Header.Set("Accept", "application/json")
	httpReq.Header.Set("Accept-Charset", "utf-8")
	httpReq.Header.Set("User-Agent", "Golang CIO Library")
	httpReq.Header.Set("Authorization", client.AuthorizationHeader(nil, request.Method, httpReq.URL, bodyValues))

	return httpReq, nil
}

// sendRequest sends the *http.Request
func (cio Cio) sendRequest(httpReq *http.Request, result interface{}, cioURL string) error {
	// Create the HTTP client
	httpClient := &http.Client{
		Transport: http.DefaultTransport,
		Timeout:   cio.RequestTimeout,
	}

	// Make the request
	res, err := httpClient.Do(httpReq)
	if err != nil {
		return fmt.Errorf("Failed to make request: %s", err)
	}

	// Parse the response
	defer func() {
		if closeErr := res.Body.Close(); closeErr != nil {
			logBodyCloseError(cio.Log, closeErr)
		}
	}()

	resBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return fmt.Errorf("Could not read response: %s", err)
	}
	resBodyString := string(resBody)

	// Unmarshal result
	err = json.Unmarshal(resBody, &result)

	// Log the response
	logResponse(cio.Log, httpReq.Method, cioURL, res.StatusCode, resBodyString, err)

	// Return special error if Status Code >= 400
	if res.StatusCode >= 400 {
		return fmt.Errorf("%d Status Code with Payload %s", res.StatusCode, resBodyString)
	}

	// Return Unmarshal error (if any) if Status Code is < 400
	return err
}

// logRequest logs the request about to be made to CIO, redacting sensitive information in the body
func logRequest(log Logger, method string, cioURL string, bodyValues url.Values) {
	if log != nil {

		// Copy url.Values
		redactedValues := url.Values{}
		for k, v := range bodyValues {
			redactedValues[k] = v
		}

		// Redact sensitive information
		if val := redactedValues.Get("password"); len(val) > 0 {
			redactedValues.Set("password", "redacted")
		}
		if val := redactedValues.Get("provider_refresh_token"); len(val) > 0 {
			redactedValues.Set("provider_refresh_token", "redacted")
		}
		if val := redactedValues.Get("provider_consumer_key"); len(val) > 0 {
			redactedValues.Set("provider_consumer_key", "redacted")
		}
		if val := redactedValues.Get("provider_consumer_secret"); len(val) > 0 {
			redactedValues.Set("provider_consumer_secret", "redacted")
		}

		// Actually log
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithFields(logrus.Fields{"httpMethod": method, "url": cioURL, "payload": redactedValues.Encode()}).Debug("Creating new request to CIO")
		} else {
			// Else just log with Println
			log.Println("Creating new " + method + " request to: " + cioURL + " with payload: " + redactedValues.Encode())
		}
	}
}

// logBodyCloseError logs any error that happens when trying to close the *http.Response.Body
func logBodyCloseError(log Logger, closeError error) {
	if log != nil {
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithError(closeError).Warn("Unable to close response body from CIO")
		} else {
			// Else just log with Println
			log.Println("Unable to close response body from CIO, with error: " + closeError.Error())
		}
	}
}

// logResponse logs the response from CIO, if any logger is set
func logResponse(log Logger, method string, cioURL string, statusCode int, responseBody string, unmarshalError error) {
	if log != nil {

		// TODO: redact access_token and access_token_secret before logging (only occurs with 3-legged oauth [rare])

		// Take only the first 2000 characters from the responseBody, which should be more than enough to debug anything
		if bodyLen := len(responseBody); bodyLen > 2000 {
			responseBody = responseBody[:2000]
		}

		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logEntry := logrusLogger.WithFields(logrus.Fields{
				"httpMethod":     method,
				"url":            cioURL,
				"statusCode":     fmt.Sprintf("%d", statusCode),
				"payloadSnippet": responseBody})
			if unmarshalError != nil || statusCode >= 400 {
				logEntry.Warn("Received response from CIO")
			} else {
				logEntry.Debug("Received response from CIO")
			}

		} else {
			// Else just log with Println
			log.Println("Received response from " + method + " to: " + cioURL +
				" with status code: " + fmt.Sprintf("%d", statusCode) +
				" and payload snippet: " + responseBody)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"

	"github.com/cilium/cilium/api/v1/models"
	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("K8sValidatedKafkaPolicyTest", func() {

	var microscopeErr error
	var microscopeCancel func() error
	var kubectl *helpers.Kubectl
	var ciliumPod string

	var (
		logger      = log.WithFields(logrus.Fields{"testName": "K8sValidatedKafkaPolicyTest"})
		l7Policy    = helpers.ManifestGet("kafka-sw-security-policy.yaml")
		demoPath    = helpers.ManifestGet("kafka-sw-app.yaml")
		kafkaApp    = "kafka"
		zookApp     = "zook"
		backupApp   = "empire-backup"
		empireHqApp = "empire-hq"
		outpostApp  = "empire-outpost"
		apps        = []string{kafkaApp, zookApp, backupApp, empireHqApp, outpostApp}
		appPods     = map[string]string{}

		prodHqAnnounce    = `-c "echo 'Happy 40th Birthday to General Tagge' | ./kafka-produce.sh --topic empire-announce"`
		conOutpostAnnoune = `-c "./kafka-consume.sh --topic empire-announce --from-beginning --max-messages 1"`
		prodHqDeathStar   = `-c "echo 'deathstar reactor design v3' | ./kafka-produce.sh --topic deathstar-plans"`
		conOutDeathStar   = `-c "./kafka-consume.sh --topic deathstar-plans --from-beginning --max-messages 1"`
		prodBackAnnounce  = `-c "echo 'Happy 40th Birthday to General Tagge' | ./kafka-produce.sh --topic empire-announce"`
		prodOutAnnounce   = `-c "echo 'Vader Booed at Empire Karaoke Party' | ./kafka-produce.sh --topic empire-announce"`
	)

	BeforeAll(func() {
		logger = log.WithFields(logrus.Fields{"testName": "K8sValidatedKafkaPolicyTest"})
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		err := kubectl.CiliumInstall(helpers.CiliumDSPath)
		Expect(err).To(BeNil(), "Cilium cannot be installed")
		ExpectCiliumReady(kubectl)
		ExpectKubeDNSReady(kubectl)

		kubectl.Apply(demoPath)
		err = kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=kafkaTestApp", 300)
		Expect(err).Should(BeNil(), "Kafka Pods are not ready after timeout")

		appPods = helpers.GetAppPods(apps, helpers.DefaultNamespace, kubectl, "app")

		ciliumPod, err = kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s2)
		Expect(err).To(BeNil(), "Cannot get cilium Pod")

	})

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.KubeSystemNamespace,
			"cilium service list",
			"cilium endpoint list")
	})

	JustBeforeEach(func() {
		microscopeErr, microscopeCancel = kubectl.MicroscopeStart()
		Expect(microscopeErr).To(BeNil(), "Microscope cannot be started")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
		Expect(microscopeCancel()).To(BeNil(), "cannot stop microscope")
	})

	AfterEach(func() {
		// On aftereach don't make assertions to delete all.
		_ = kubectl.Delete(demoPath)
		_ = kubectl.Delete(l7Policy)

		ExpectAllPodsTerminated(kubectl)

	})

	It("KafkaPolicies", func() {

		By("Testing basic Kafka Produce and Consume")
		// We need to produce first, since consumer script waits for
		// some messages to be already there by the producer.

		err := kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqDeathStar))
		Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic deathstar-plans")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutDeathStar))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic deathstar-plans")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[backupApp], fmt.Sprintf(prodBackAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to backup on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(prodOutAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to outpost on topic empire-announce")

		By("Apply L7 kafka policy and wait")

		_, err = kubectl.CiliumPolicyAction(
			helpers.KubeSystemNamespace, l7Policy,
			helpers.KubectlApply, helpers.HelperTimeout)
		Expect(err).To(BeNil(), "L7 policy cannot be imported correctly")

		ExpectCEPUpdates(kubectl)

		By("validate that the pods have the correct policy")

		desiredPolicyStatus := map[string]models.EndpointPolicyEnabled{
			backupApp:   models.EndpointPolicyEnabledNone,
			empireHqApp: models.EndpointPolicyEnabledNone,
			kafkaApp:    models.EndpointPolicyEnabledIngress,
			outpostApp:  models.EndpointPolicyEnabledNone,
			zookApp:     models.EndpointPolicyEnabledNone,
		}

		for app, policy := range desiredPolicyStatus {
			cep := kubectl.CepGet(helpers.DefaultNamespace, appPods[app])
			Expect(cep).ToNot(BeNil(), "cannot get cep for app %q and pod %s", app, appPods[app])
			Expect(cep.Status.Policy.Spec.PolicyEnabled).To(Equal(policy), "Policy for %q mismatch", app)
		}

		By("Validating Policy trace")
		trace := kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 9092",
			appPods[empireHqApp], appPods[kafkaApp]))
		trace.ExpectSuccess("Cilium policy trace failed")
		trace.ExpectContains("Final verdict: ALLOWED")

		trace = kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 9092",
			appPods[backupApp], appPods[kafkaApp]))
		trace.ExpectSuccess("Cilium policy trace failed")
		trace.ExpectContains("Final verdict: ALLOWED")

		trace = kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 80",
			appPods[empireHqApp], appPods[kafkaApp]))
		trace.ExpectSuccess("Failed cilium policy trace")
		trace.ExpectContains("Final verdict: DENIED")

		By("Testing Kafka L7 policy enforcement status")
		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqDeathStar))
		Expect(err).Should(BeNil(), "Failed to produce from empire-hq on topic deathstar-plans")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[backupApp], fmt.Sprintf(prodBackAnnounce))
		Expect(err).Should(HaveOccurred(), " Produce to backup on topic empire-announce should have been denied")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutDeathStar))
		Expect(err).Should(HaveOccurred(), " Consume from outpost on topic deathstar-plans should have been denied")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(prodOutAnnounce))
		Expect(err).Should(HaveOccurred(), "Produce to outpost on topic empire-announce should have been denied")
	})
})
<commit_msg>test/k8sT: wrap KafkaPolicies test within Context<commit_after>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"

	"github.com/cilium/cilium/api/v1/models"
	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("K8sValidatedKafkaPolicyTest", func() {

	var microscopeErr error
	var microscopeCancel func() error
	var kubectl *helpers.Kubectl
	var ciliumPod string

	var (
		logger      = log.WithFields(logrus.Fields{"testName": "K8sValidatedKafkaPolicyTest"})
		l7Policy    = helpers.ManifestGet("kafka-sw-security-policy.yaml")
		demoPath    = helpers.ManifestGet("kafka-sw-app.yaml")
		kafkaApp    = "kafka"
		zookApp     = "zook"
		backupApp   = "empire-backup"
		empireHqApp = "empire-hq"
		outpostApp  = "empire-outpost"
		apps        = []string{kafkaApp, zookApp, backupApp, empireHqApp, outpostApp}
		appPods     = map[string]string{}

		prodHqAnnounce    = `-c "echo 'Happy 40th Birthday to General Tagge' | ./kafka-produce.sh --topic empire-announce"`
		conOutpostAnnoune = `-c "./kafka-consume.sh --topic empire-announce --from-beginning --max-messages 1"`
		prodHqDeathStar   = `-c "echo 'deathstar reactor design v3' | ./kafka-produce.sh --topic deathstar-plans"`
		conOutDeathStar   = `-c "./kafka-consume.sh --topic deathstar-plans --from-beginning --max-messages 1"`
		prodBackAnnounce  = `-c "echo 'Happy 40th Birthday to General Tagge' | ./kafka-produce.sh --topic empire-announce"`
		prodOutAnnounce   = `-c "echo 'Vader Booed at Empire Karaoke Party' | ./kafka-produce.sh --topic empire-announce"`
	)

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.KubeSystemNamespace,
			"cilium service list",
			"cilium endpoint list")
	})

	// GH-4414: put test in a context so that if any failures occur in BeforeAll,
	// logs will be gathered by the above "AfterFailed".
	Context("Kafka Policy Tests", func() {
		JustBeforeEach(func() {
			microscopeErr, microscopeCancel = kubectl.MicroscopeStart()
			Expect(microscopeErr).To(BeNil(), "Microscope cannot be started")
		})

		JustAfterEach(func() {
			kubectl.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
			Expect(microscopeCancel()).To(BeNil(), "cannot stop microscope")
		})

		AfterEach(func() {
			// On aftereach don't make assertions to delete all.
			_ = kubectl.Delete(demoPath)
			_ = kubectl.Delete(l7Policy)

			ExpectAllPodsTerminated(kubectl)
		})

		BeforeAll(func() {
			logger = log.WithFields(logrus.Fields{"testName": "K8sValidatedKafkaPolicyTest"})
			kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

			err := kubectl.CiliumInstall(helpers.CiliumDSPath)
			Expect(err).To(BeNil(), "Cilium cannot be installed")
			ExpectCiliumReady(kubectl)
			ExpectKubeDNSReady(kubectl)

			kubectl.Apply(demoPath)
			err = kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=kafkaTestApp", 300)
			Expect(err).Should(BeNil(), "Kafka Pods are not ready after timeout")

			appPods = helpers.GetAppPods(apps, helpers.DefaultNamespace, kubectl, "app")

			ciliumPod, err = kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s2)
			Expect(err).To(BeNil(), "Cannot get cilium Pod")
		})

		It("KafkaPolicies", func() {

			By("Testing basic Kafka Produce and Consume")
			// We need to produce first, since consumer script waits for
			// some messages to be already there by the producer.

			err := kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqAnnounce))
			Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic empire-announce")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
			Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqDeathStar))
			Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic deathstar-plans")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutDeathStar))
			Expect(err).Should(BeNil(), "Failed to consume from outpost on topic deathstar-plans")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[backupApp], fmt.Sprintf(prodBackAnnounce))
			Expect(err).Should(BeNil(), "Failed to produce to backup on topic empire-announce")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(prodOutAnnounce))
			Expect(err).Should(BeNil(), "Failed to produce to outpost on topic empire-announce")

			By("Apply L7 kafka policy and wait")

			_, err = kubectl.CiliumPolicyAction(
				helpers.KubeSystemNamespace, l7Policy,
				helpers.KubectlApply, helpers.HelperTimeout)
			Expect(err).To(BeNil(), "L7 policy cannot be imported correctly")

			ExpectCEPUpdates(kubectl)

			By("validate that the pods have the correct policy")

			desiredPolicyStatus := map[string]models.EndpointPolicyEnabled{
				backupApp:   models.EndpointPolicyEnabledNone,
				empireHqApp: models.EndpointPolicyEnabledNone,
				kafkaApp:    models.EndpointPolicyEnabledIngress,
				outpostApp:  models.EndpointPolicyEnabledNone,
				zookApp:     models.EndpointPolicyEnabledNone,
			}

			for app, policy := range desiredPolicyStatus {
				cep := kubectl.CepGet(helpers.DefaultNamespace, appPods[app])
				Expect(cep).ToNot(BeNil(), "cannot get cep for app %q and pod %s", app, appPods[app])
				Expect(cep.Status.Policy.Spec.PolicyEnabled).To(Equal(policy), "Policy for %q mismatch", app)
			}

			By("Validating Policy trace")
			trace := kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
				"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 9092",
				appPods[empireHqApp], appPods[kafkaApp]))
			trace.ExpectSuccess("Cilium policy trace failed")
			trace.ExpectContains("Final verdict: ALLOWED")

			trace = kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
				"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 9092",
				appPods[backupApp], appPods[kafkaApp]))
			trace.ExpectSuccess("Cilium policy trace failed")
			trace.ExpectContains("Final verdict: ALLOWED")

			trace = kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
				"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 80",
				appPods[empireHqApp], appPods[kafkaApp]))
			trace.ExpectSuccess("Failed cilium policy trace")
			trace.ExpectContains("Final verdict: DENIED")

			By("Testing Kafka L7 policy enforcement status")
			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqAnnounce))
			Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic empire-announce")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
			Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqDeathStar))
			Expect(err).Should(BeNil(), "Failed to produce from empire-hq on topic deathstar-plans")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
			Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[backupApp], fmt.Sprintf(prodBackAnnounce))
			Expect(err).Should(HaveOccurred(), " Produce to backup on topic empire-announce should have been denied")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutDeathStar))
			Expect(err).Should(HaveOccurred(), " Consume from outpost on topic deathstar-plans should have been denied")

			err = kubectl.ExecKafkaPodCmd(
				helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(prodOutAnnounce))
			Expect(err).Should(HaveOccurred(), "Produce to outpost on topic empire-announce should have been denied")
		})
	})
})
<|endoftext|>


<commit_before>package model

import (
	"github.com/sparrowdb/compression"
	"github.com/sparrowdb/util"
	"github.com/sparrowdb/util/uuid"
)

const (
	// DataDefinitionActive active status
	DataDefinitionActive = iota

	// DataDefinitionRemoved removed status
	DataDefinitionRemoved
)

// DataDefinition holds the stored item
type DataDefinition struct {
	Key    string
	Size   uint32
	Token  string
	Ext    string
	Status uint16
	Buf    []byte
}

// DataDefinitionResult holds DataDefinition query result
type DataDefinitionResult struct {
	Key       string
	Size      uint32
	Token     string
	TImestamp string
	Ext       string
	Status    string
}

// QueryResult convert DataDefinition to DataDefinitionResult
func (df *DataDefinition) QueryResult() *DataDefinitionResult {
	dfr := DataDefinitionResult{
		Key:   df.Key,
		Size:  df.Size,
		Token: df.Token,
		Ext:   df.Ext,
	}

	switch df.Status {
	case 1:
		dfr.Status = "Active"
	case 2:
		dfr.Status = "Removed"
	}

	uuid, _ := uuid.ParseUUID(df.Token)
	dfr.TImestamp = uuid.Time().String()

	return &dfr
}

// ToByteStream convert DataDefinition to ByteStream
func (df *DataDefinition) ToByteStream() *util.ByteStream {
	byteStream := util.NewByteStream()
	byteStream.PutString(df.Key)
	byteStream.PutString(df.Token)
	byteStream.PutUInt32(df.Size)
	byteStream.PutString(df.Ext)
	byteStream.PutUInt16(df.Status)

	encoded := compression.Compress(df.Buf)
	byteStream.PutBytes(encoded)

	return byteStream
}

// NewDataDefinitionFromByteStream convert ByteStream to DataDefinition
func NewDataDefinitionFromByteStream(bs *util.ByteStream) *DataDefinition {
	df := DataDefinition{}
	df.Key = bs.GetString()
	df.Token = bs.GetString()
	df.Size = bs.GetUInt32()
	df.Ext = bs.GetString()
	df.Status = bs.GetUInt16()

	buf := bs.GetBytes()
	if decoded, err := compression.Decompress(buf); err == nil {
		df.Buf = decoded
	}

	return &df
}
<commit_msg>removed status from data definition query result<commit_after>package model

import (
	"github.com/sparrowdb/compression"
	"github.com/sparrowdb/util"
	"github.com/sparrowdb/util/uuid"
)

const (
	// DataDefinitionActive active status
	DataDefinitionActive = iota

	// DataDefinitionRemoved removed status
	DataDefinitionRemoved
)

// DataDefinition holds the stored item
type DataDefinition struct {
	Key    string
	Size   uint32
	Token  string
	Ext    string
	Status uint16
	Buf    []byte
}

// DataDefinitionResult holds DataDefinition query result
type DataDefinitionResult struct {
	Key       string
	Size      uint32
	Token     string
	TImestamp string
	Ext       string
}

// QueryResult convert DataDefinition to DataDefinitionResult
func (df *DataDefinition) QueryResult() *DataDefinitionResult {
	dfr := DataDefinitionResult{
		Key:   df.Key,
		Size:  df.Size,
		Token: df.Token,
		Ext:   df.Ext,
	}

	uuid, _ := uuid.ParseUUID(df.Token)
	dfr.TImestamp = uuid.Time().String()

	return &dfr
}

// ToByteStream convert DataDefinition to ByteStream
func (df *DataDefinition) ToByteStream() *util.ByteStream {
	byteStream := util.NewByteStream()
	byteStream.PutString(df.Key)
	byteStream.PutString(df.Token)
	byteStream.PutUInt32(df.Size)
	byteStream.PutString(df.Ext)
	byteStream.PutUInt16(df.Status)

	encoded := compression.Compress(df.Buf)
	byteStream.PutBytes(encoded)

	return byteStream
}

// NewDataDefinitionFromByteStream convert ByteStream to DataDefinition
func NewDataDefinitionFromByteStream(bs *util.ByteStream) *DataDefinition {
	df := DataDefinition{}
	df.Key = bs.GetString()
	df.Token = bs.GetString()
	df.Size = bs.GetUInt32()
	df.Ext = bs.GetString()
	df.Status = bs.GetUInt16()

	buf := bs.GetBytes()
	if decoded, err := compression.Decompress(buf); err == nil {
		df.Buf = decoded
	}

	return &df
}
<|endoftext|>


<commit_before>package models

import (
	"crypto/x509"
	"encoding/pem"
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestNewCollectorKeysAreSignedByAPIKey(t *testing.T) {
	assert := assert.New(t)

	collector_cert_data, _, err := newCollectorKeys()
	assert.Nil(err)
	block, _ := pem.Decode(collector_cert_data)
	ca, err := x509.ParseCertificate(APITLSCertificate().Certificate[0])
	assert.Nil(err)
	client, err := x509.ParseCertificate(block.Bytes)
	assert.Nil(err)

	err = client.CheckSignatureFrom(ca)
	assert.Nil(err)
}
<commit_msg>collector model tests<commit_after>package models

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"github.com/stretchr/testify/assert"
	"testing"
)

func TextCollectorTLSConfigCreatesCorrectConfig(t *testing.T) {
	assert := assert.New(t)

	collector1, err := CreateCollector("thing1")
	assert.Nil(err)
	collector2, err := CreateCollector("thing2")
	assert.Nil(err)

	config := CollectorTLSConfig()
	if assert.Equal(2, len(config.Certificates)) {
		assert.NotEqual(config.Certificates[0], config.Certificates[1])
	}

	block1, _ := pem.Decode(collector1.CaCert)
	client1, err := x509.ParseCertificate(block1.Bytes)
	assert.Nil(err)

	block2, _ := pem.Decode(collector2.CaCert)
	client2, err := x509.ParseCertificate(block2.Bytes)
	assert.Nil(err)

	for _, cert := range config.Certificates {
		assert.Equal(
			true,
			bytes.Equal(
				cert.Certificate[0],
				client1.Raw,
			) || bytes.Equal(
				cert.Certificate[0],
				client2.Raw,
			),
		)
	}
}

func TestNewCollectorKeysAreSignedByAPIKey(t *testing.T) {
	assert := assert.New(t)

	collector_cert_data, _, err := newCollectorKeys()
	assert.Nil(err)
	block, _ := pem.Decode(collector_cert_data)
	ca, err := x509.ParseCertificate(APITLSCertificate().Certificate[0])
	assert.Nil(err)
	client, err := x509.ParseCertificate(block.Bytes)
	assert.Nil(err)

	err = client.CheckSignatureFrom(ca)
	assert.Nil(err)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/urfave/cli"
	"tantalic.com/cistatus"
)

func main() {
	app := cli.App{
		Name:         filepath.Base(os.Args[0]),
		HelpName:     filepath.Base(os.Args[0]),
		Usage:        "updates the anybar status icon based on the status of the continuous integration server",
		UsageText:    "cistatusanybar [options] [ci-hostname]",
		Version:      cistatus.Version,
		BashComplete: cli.DefaultAppComplete,
		Writer:       os.Stdout,
	}

	var conf config
	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:        "status-port",
			EnvVar:      "STATUS_PORT",
			Usage:       "udp port the anybar app is listening on",
			Destination: &conf.StatusPort,
		},
		cli.StringFlag{
			Name:        "anybar-host",
			EnvVar:      "ANYBAR_HOST",
			Usage:       "host that is running the anybar app",
			Destination: &conf.AnyBarHost,
		},
		cli.IntFlag{
			Name:        "anybar-port",
			Value:       1739,
			EnvVar:      "ANYBAR_PORT",
			Usage:       "udp port the anybar app is listening on",
			Destination: &conf.AnyBarPort,
		},
		cli.BoolFlag{
			Name:        "start-anybar",
			EnvVar:      "START_ANYBAR",
			Usage:       "start (or restart) the anybar app",
			Destination: &conf.StartAnyBar,
		},
		cli.BoolFlag{
			Name:        "verbose",
			Usage:       "print status information",
			EnvVar:      "VERBOSE",
			Destination: &conf.Verbose,
		},
		cli.BoolFlag{
			Name:        "install-launch-agent",
			Usage:       "install macos launchd agent to launch on startup and keep running",
			Destination: &conf.InstallLaunchAgent,
		},
	}

	app.Action = func(c *cli.Context) error {
		if len(c.Args()) != 1 {
			err := cli.NewExitError("status server hostname argument must be provided", 1)
			printUsage(c)
			return err
		}

		conf.StatusHost = c.Args().Get(0)

		if conf.InstallLaunchAgent {
			return installLaunchAgent(conf)
		}

		return runloop(conf)
	}

	app.Run(os.Args)
}

func printUsage(c *cli.Context) {
	fmt.Fprintf(c.App.Writer, "usage: %s\n", c.App.UsageText)
}
<commit_msg>Fixes help text for the —status-port flag in cistatusanybar<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/urfave/cli"
	"tantalic.com/cistatus"
)

func main() {
	app := cli.App{
		Name:         filepath.Base(os.Args[0]),
		HelpName:     filepath.Base(os.Args[0]),
		Usage:        "updates the anybar status icon based on the status of the continuous integration server",
		UsageText:    "cistatusanybar [options] [ci-hostname]",
		Version:      cistatus.Version,
		BashComplete: cli.DefaultAppComplete,
		Writer:       os.Stdout,
	}

	var conf config
	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:        "status-port",
			EnvVar:      "STATUS_PORT",
			Usage:       "tcp (http) port to connect to the status server on",
			Destination: &conf.StatusPort,
		},
		cli.StringFlag{
			Name:        "anybar-host",
			EnvVar:      "ANYBAR_HOST",
			Usage:       "host that is running the anybar app",
			Destination: &conf.AnyBarHost,
		},
		cli.IntFlag{
			Name:        "anybar-port",
			Value:       1739,
			EnvVar:      "ANYBAR_PORT",
			Usage:       "udp port the anybar app is listening on",
			Destination: &conf.AnyBarPort,
		},
		cli.BoolFlag{
			Name:        "start-anybar",
			EnvVar:      "START_ANYBAR",
			Usage:       "start (or restart) the anybar app",
			Destination: &conf.StartAnyBar,
		},
		cli.BoolFlag{
			Name:        "verbose",
			Usage:       "print status information",
			EnvVar:      "VERBOSE",
			Destination: &conf.Verbose,
		},
		cli.BoolFlag{
			Name:        "install-launch-agent",
			Usage:       "install macos launchd agent to launch on startup and keep running",
			Destination: &conf.InstallLaunchAgent,
		},
	}

	app.Action = func(c *cli.Context) error {
		if len(c.Args()) != 1 {
			err := cli.NewExitError("status server hostname argument must be provided", 1)
			printUsage(c)
			return err
		}

		conf.StatusHost = c.Args().Get(0)

		if conf.InstallLaunchAgent {
			return installLaunchAgent(conf)
		}

		return runloop(conf)
	}

	app.Run(os.Args)
}

func printUsage(c *cli.Context) {
	fmt.Fprintf(c.App.Writer, "usage: %s\n", c.App.UsageText)
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"syscall"
	"time"

	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/lager/lagertest"

	"github.com/cloudfoundry-incubator/converger/converger_runner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

var _ = Describe("Converger", func() {
	var (
		etcdRunner *etcdstorerunner.ETCDClusterRunner
		bbs        *Bbs.BBS
		runner     *converger_runner.ConvergerRunner

		taskKickInterval = 1 * time.Second

		expireCompletedTaskDuration = 3 * time.Second

		etcdClient storeadapter.StoreAdapter

		convergeRepeatInterval = time.Second
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := gexec.Build("github.com/cloudfoundry-incubator/converger/cmd/converger", "-race")
		Ω(err).ShouldNot(HaveOccurred())
		return []byte(convergerBinPath)
	}, func(convergerBinPath []byte) {
		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)

		etcdClient = etcdRunner.Adapter()
		bbs = Bbs.NewBBS(etcdClient, timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))

		runner = converger_runner.New(string(convergerBinPath), etcdCluster, "info")
	})

	SynchronizedAfterSuite(func() {
		etcdRunner.Stop()
	}, func() {
		gexec.CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()

		cellPresence := models.CellPresence{
			CellID: "the-cell-id",
			Stack:  "the-stack",
		}

		value, err := models.ToJSON(cellPresence)
		Ω(err).ShouldNot(HaveOccurred())
		etcdClient.Create(storeadapter.StoreNode{
			Key:   shared.CellSchemaPath(cellPresence.CellID),
			Value: value,
		})
	})

	AfterEach(func() {
		runner.KillWithFire()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner.Start(convergeRepeatInterval, taskKickInterval, 30*time.Minute, expireCompletedTaskDuration, 30*time.Second, 300*time.Second)
		time.Sleep(convergeRepeatInterval)
	}

	createClaimedTaskWithDeadCell := func() {
		task := models.Task{
			Domain: "tests",

			TaskGuid: "task-guid",
			Stack:    "stack",
			Action: &models.RunAction{
				Path: "cat",
				Args: []string{"/tmp/file"},
			},
		}

		err := bbs.DesireTask(task)
		Ω(err).ShouldNot(HaveOccurred())

		err = bbs.ClaimTask(task.TaskGuid, "dead-cell")
		Ω(err).ShouldNot(HaveOccurred())
	}

	desireLRP := func() {
		err := bbs.DesireLRP(models.DesiredLRP{
			Domain: "tests",

			ProcessGuid: "the-guid",

			Stack: "some-stack",

			Instances: 3,
			MemoryMB:  128,
			DiskMB:    512,

			Action: &models.RunAction{
				Path: "the-start-command",
			},
		})
		Ω(err).ShouldNot(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when an LRP is desired", func() {
			JustBeforeEach(desireLRP)

			It("does not create start auctions for apps that are missing instances", func() {
				Consistently(bbs.LRPStartAuctions, 0.5).Should(BeEmpty())
			})
		})

		Context("when a claimed task with a dead cell is present", func() {
			JustBeforeEach(createClaimedTaskWithDeadCell)

			It("does not change the task", func() {
				Consistently(bbs.CompletedTasks, taskKickInterval*2).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		JustBeforeEach(startConverger)

		Describe("when an LRP is desired", func() {
			BeforeEach(desireLRP)

			Context("for an app that is not running at all", func() {
				It("desires N start auctions in the BBS", func() {
					Eventually(bbs.LRPStartAuctions, 0.5).Should(HaveLen(3))
				})
			})

			Context("for an app that is missing instances", func() {
				BeforeEach(func() {
					err := bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "a",
						Domain:       "the-domain",
						Index:        0,
					}, "the-cell-id")
					Ω(err).ShouldNot(HaveOccurred())
				})

				It("start auctions for the missing instances", func() {
					Eventually(bbs.LRPStartAuctions, 0.5).Should(HaveLen(2))
					auctions, err := bbs.LRPStartAuctions()
					Ω(err).ShouldNot(HaveOccurred())

					indices := []int{auctions[0].Index, auctions[1].Index}
					Ω(indices).Should(ContainElement(1))
					Ω(indices).Should(ContainElement(2))

					Consistently(bbs.LRPStartAuctions).Should(HaveLen(2))
				})
			})

			Context("for an app that has extra instances", func() {
				BeforeEach(func() {
					err := bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "a",
						Domain:       "the-domain",
						Index:        0,
					}, "the-cell-id")
					Ω(err).ShouldNot(HaveOccurred())

					err = bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "b",
						Domain:       "the-domain",
						Index:        1,
					}, "the-cell-id")
					Ω(err).ShouldNot(HaveOccurred())

					err = bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "c",
						Domain:       "the-domain",
						Index:        2,
					}, "the-cell-id")
					Ω(err).ShouldNot(HaveOccurred())

					err = bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "d-extra",
						Domain:       "the-domain",
						Index:        3,
					}, "the-cell-id")
					Ω(err).ShouldNot(HaveOccurred())
				})

				It("stops the extra instances", func() {
					Consistently(bbs.LRPStartAuctions, 0.5).Should(BeEmpty())
					Eventually(bbs.StopLRPInstances).Should(HaveLen(1))
					stopInstances, err := bbs.StopLRPInstances()
					Ω(err).ShouldNot(HaveOccurred())

					Ω(stopInstances[0].ProcessGuid).Should(Equal("the-guid"))
					Ω(stopInstances[0].Index).Should(Equal(3))
					Ω(stopInstances[0].InstanceGuid).Should(Equal("d-extra"))
				})
			})
		})

		Describe("when a claimed task with a dead cell is present", func() {
			JustBeforeEach(createClaimedTaskWithDeadCell)

			It("marks the task as failed after the kick interval", func() {
				Eventually(bbs.CompletedTasks, taskKickInterval*2).Should(HaveLen(1))
				tasks, err := bbs.Tasks()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(tasks).Should(HaveLen(1))
				Ω(tasks[0].State).Should(Equal(models.TaskStateCompleted))
				Ω(tasks[0].Failed).Should(BeTrue())
			})

			It("deletes the task after the 'expire completed task' interval", func() {
				Eventually(bbs.CompletedTasks, taskKickInterval*2).Should(HaveLen(1))
				tasks, err := bbs.Tasks()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(tasks).Should(HaveLen(1))
				Ω(tasks[0].State).Should(Equal(models.TaskStateCompleted))
				Ω(tasks[0].Failed).Should(BeTrue())

				guid := tasks[0].TaskGuid
				_, err = bbs.TaskByGuid(guid)
				Ω(err).ShouldNot(HaveOccurred())

				getTaskError := func() error {
					_, err := bbs.TaskByGuid(guid)
					return err
				}

				Consistently(getTaskError, expireCompletedTaskDuration-time.Second).ShouldNot(HaveOccurred())
				Eventually(getTaskError, expireCompletedTaskDuration+time.Second).Should(Equal(storeadapter.ErrorKeyNotFound))
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			err := etcdClient.Update(storeadapter.StoreNode{
				Key:   shared.LockSchemaPath("converge_lock"),
				Value: []byte("something-else"),
			})
			Ω(err).ShouldNot(HaveOccurred())

			time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
		})

		itIsInactive()

		It("exits with an error", func() {
			Eventually(runner.Session.ExitCode).Should(Equal(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		BeforeEach(func() {
			err := etcdClient.Create(storeadapter.StoreNode{
				Key:   shared.LockSchemaPath("converge_lock"),
				Value: []byte("something-else"),
			})
			Ω(err).ShouldNot(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				err := etcdClient.Delete(shared.LockSchemaPath("converge_lock"))
				Ω(err).ShouldNot(HaveOccurred())

				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when an LRP is desired", func() {
				JustBeforeEach(desireLRP)

				Context("for an app that is not running at all", func() {
					It("desires N start auctions in the BBS", func() {
						Eventually(bbs.LRPStartAuctions, 0.5).Should(HaveLen(3))
					})
				})
			})

			Describe("when a claimed task with a dead cell is present", func() {
				JustBeforeEach(createClaimedTaskWithDeadCell)

				It("eventually marks the task as failed", func() {
					Eventually(bbs.CompletedTasks, taskKickInterval*2).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGINT)
				Eventually(runner.Session, 4).Should(gexec.Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGTERM)
				Eventually(runner.Session, 4).Should(gexec.Exit(0))
			})
		})
	})
})
<commit_msg>Converger no longer needs to know about storeadapter errors. [#82149700]<commit_after>package main_test

import (
	"fmt"
	"syscall"
	"time"

	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/lager/lagertest"

	"github.com/cloudfoundry-incubator/converger/converger_runner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/bbserrors"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

var _ = Describe("Converger", func() {
	var (
		etcdRunner *etcdstorerunner.ETCDClusterRunner
		bbs        *Bbs.BBS
		runner     *converger_runner.ConvergerRunner

		taskKickInterval = 1 * time.Second

		expireCompletedTaskDuration = 3 * time.Second

		etcdClient storeadapter.StoreAdapter

		convergeRepeatInterval = time.Second
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := gexec.Build("github.com/cloudfoundry-incubator/converger/cmd/converger", "-race")
		Ω(err).ShouldNot(HaveOccurred())
		return []byte(convergerBinPath)
	}, func(convergerBinPath []byte) {
		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)

		etcdClient = etcdRunner.Adapter()
		bbs = Bbs.NewBBS(etcdClient, timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))

		runner = converger_runner.New(string(convergerBinPath), etcdCluster, "info")
	})

	SynchronizedAfterSuite(func() {
		etcdRunner.Stop()
	}, func() {
		gexec.CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()

		cellPresence := models.CellPresence{
			CellID: "the-cell-id",
			Stack:  "the-stack",
		}

		value, err := models.ToJSON(cellPresence)
		Ω(err).ShouldNot(HaveOccurred())
		etcdClient.Create(storeadapter.StoreNode{
			Key:   shared.CellSchemaPath(cellPresence.CellID),
			Value: value,
		})
	})

	AfterEach(func() {
		runner.KillWithFire()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner.Start(convergeRepeatInterval, taskKickInterval, 30*time.Minute, expireCompletedTaskDuration, 30*time.Second, 300*time.Second)
		time.Sleep(convergeRepeatInterval)
	}

	createClaimedTaskWithDeadCell := func() {
		task := models.Task{
			Domain: "tests",

			TaskGuid: "task-guid",
			Stack:    "stack",
			Action: &models.RunAction{
				Path: "cat",
				Args: []string{"/tmp/file"},
			},
		}

		err := bbs.DesireTask(task)
		Ω(err).ShouldNot(HaveOccurred())

		err = bbs.ClaimTask(task.TaskGuid, "dead-cell")
		Ω(err).ShouldNot(HaveOccurred())
	}

	desireLRP := func() {
		err := bbs.DesireLRP(models.DesiredLRP{
			Domain: "tests",

			ProcessGuid: "the-guid",

			Stack: "some-stack",

			Instances: 3,
			MemoryMB:  128,
			DiskMB:    512,

			Action: &models.RunAction{
				Path: "the-start-command",
			},
		})
		Ω(err).ShouldNot(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when an LRP is desired", func() {
			JustBeforeEach(desireLRP)

			It("does not create start auctions for apps that are missing instances", func() {
				Consistently(bbs.LRPStartAuctions, 0.5).Should(BeEmpty())
			})
		})

		Context("when a claimed task with a dead cell is present", func() {
			JustBeforeEach(createClaimedTaskWithDeadCell)

			It("does not change the task", func() {
				Consistently(bbs.CompletedTasks, taskKickInterval*2).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		JustBeforeEach(startConverger)

		Describe("when an LRP is desired", func() {
			BeforeEach(desireLRP)

			Context("for an app that is not running at all", func() {
				It("desires N start auctions in the BBS", func() {
					Eventually(bbs.LRPStartAuctions, 0.5).Should(HaveLen(3))
				})
			})

			Context("for an app that is missing instances", func() {
				BeforeEach(func() {
					err := bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "a",
						Domain:       "the-domain",
						Index:        0,
					}, "the-cell-id")
					Ω(err).ShouldNot(HaveOccurred())
				})

				It("start auctions for the missing instances", func() {
					Eventually(bbs.LRPStartAuctions, 0.5).Should(HaveLen(2))
					auctions, err := bbs.LRPStartAuctions()
					Ω(err).ShouldNot(HaveOccurred())

					indices := []int{auctions[0].Index, auctions[1].Index}
					Ω(indices).Should(ContainElement(1))
					Ω(indices).Should(ContainElement(2))

					Consistently(bbs.LRPStartAuctions).Should(HaveLen(2))
				})
			})

			Context("for an app that has extra instances", func() {
				BeforeEach(func() {
					err := bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "a",
						Domain:       "the-domain",
						Index:        0,
					}, "the-cell-id")
					Ω(err).ShouldNot(HaveOccurred())

					err = bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "b",
						Domain:       "the-domain",
						Index:        1,
					}, "the-cell-id")
					Ω(err).ShouldNot(HaveOccurred())

					err = bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "c",
						Domain:       "the-domain",
						Index:        2,
					}, "the-cell-id")
					Ω(err).ShouldNot(HaveOccurred())

					err = bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "d-extra",
						Domain:       "the-domain",
						Index:        3,
					}, "the-cell-id")
					Ω(err).ShouldNot(HaveOccurred())
				})

				It("stops the extra instances", func() {
					Consistently(bbs.LRPStartAuctions, 0.5).Should(BeEmpty())
					Eventually(bbs.StopLRPInstances).Should(HaveLen(1))
					stopInstances, err := bbs.StopLRPInstances()
					Ω(err).ShouldNot(HaveOccurred())

					Ω(stopInstances[0].ProcessGuid).Should(Equal("the-guid"))
					Ω(stopInstances[0].Index).Should(Equal(3))
					Ω(stopInstances[0].InstanceGuid).Should(Equal("d-extra"))
				})
			})
		})

		Describe("when a claimed task with a dead cell is present", func() {
			JustBeforeEach(createClaimedTaskWithDeadCell)

			It("marks the task as failed after the kick interval", func() {
				Eventually(bbs.CompletedTasks, taskKickInterval*2).Should(HaveLen(1))
				tasks, err := bbs.Tasks()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(tasks).Should(HaveLen(1))
				Ω(tasks[0].State).Should(Equal(models.TaskStateCompleted))
				Ω(tasks[0].Failed).Should(BeTrue())
			})

			It("deletes the task after the 'expire completed task' interval", func() {
				Eventually(bbs.CompletedTasks, taskKickInterval*2).Should(HaveLen(1))
				tasks, err := bbs.Tasks()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(tasks).Should(HaveLen(1))
				Ω(tasks[0].State).Should(Equal(models.TaskStateCompleted))
				Ω(tasks[0].Failed).Should(BeTrue())

				guid := tasks[0].TaskGuid
				_, err = bbs.TaskByGuid(guid)
				Ω(err).ShouldNot(HaveOccurred())

				getTaskError := func() error {
					_, err := bbs.TaskByGuid(guid)
					return err
				}

				Consistently(getTaskError, expireCompletedTaskDuration-time.Second).ShouldNot(HaveOccurred())
				Eventually(getTaskError, expireCompletedTaskDuration+time.Second).Should(Equal(bbserrors.ErrStoreResourceNotFound))
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			err := etcdClient.Update(storeadapter.StoreNode{
				Key:   shared.LockSchemaPath("converge_lock"),
				Value: []byte("something-else"),
			})
			Ω(err).ShouldNot(HaveOccurred())

			time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
		})

		itIsInactive()

		It("exits with an error", func() {
			Eventually(runner.Session.ExitCode).Should(Equal(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		BeforeEach(func() {
			err := etcdClient.Create(storeadapter.StoreNode{
				Key:   shared.LockSchemaPath("converge_lock"),
				Value: []byte("something-else"),
			})
			Ω(err).ShouldNot(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				err := etcdClient.Delete(shared.LockSchemaPath("converge_lock"))
				Ω(err).ShouldNot(HaveOccurred())

				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when an LRP is desired", func() {
				JustBeforeEach(desireLRP)

				Context("for an app that is not running at all", func() {
					It("desires N start auctions in the BBS", func() {
						Eventually(bbs.LRPStartAuctions, 0.5).Should(HaveLen(3))
					})
				})
			})

			Describe("when a claimed task with a dead cell is present", func() {
				JustBeforeEach(createClaimedTaskWithDeadCell)

				It("eventually marks the task as failed", func() {
					Eventually(bbs.CompletedTasks, taskKickInterval*2).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGINT)
				Eventually(runner.Session, 4).Should(gexec.Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGTERM)
				Eventually(runner.Session, 4).Should(gexec.Exit(0))
			})
		})
	})
})
<|endoftext|>