<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/mendersoftware/go-lib-micro/log"

	"github.com/mendersoftware/deployments/config"
	"github.com/spf13/viper"
)

func main() {

	var configPath string
	var printVersion bool
	flag.StringVar(&configPath, "config", "", "Configuration file path. Supports JSON, TOML, YAML and HCL formatted configs.")
	flag.BoolVar(&printVersion, "version", false, "Show version")

	flag.Parse()

	if printVersion {
		fmt.Println("Version:", CreateVersionString())
		fmt.Println("BuildNumber:", BuildNumber)
		os.Exit(0)
	}

	l := log.New(log.Ctx{})

	configuration, err := HandleConfigFile(configPath)
	if err != nil {
		l.Fatalf("error loading configuration: %s", err)
	}

	l.Fatal(RunServer(configuration))
}

func HandleConfigFile(filePath string) (config.ConfigReader, error) {

	c := viper.New()

	// Enable setting also other conig values by environment variables
	c.SetEnvPrefix("DEPLOYMENTS")
	c.AutomaticEnv()

	// Set default values for config
	SetDefaultConfigs(c)

	// Find and read the config file
	if filePath != "" {
		c.SetConfigFile(filePath)
		if err := c.ReadInConfig(); err != nil {
			return nil, err
		}
	}

	// Validate config
	if err := config.ValidateConfig(c,
		ValidateAwsAuth,
		ValidateHttps,
	); err != nil {
		return nil, err
	}

	return c, nil
}

func SetDefaultConfigs(config *viper.Viper) {
	config.SetDefault(SettingListen, SettingListenDefault)
	config.SetDefault(SettingAwsS3Region, SettingAwsS3RegionDefault)
	config.SetDefault(SettingAwsS3Bucket, SettingAwsS3BucketDefault)
	config.SetDefault(SettingMongo, SettingMongoDefault)
	config.SetDefault(SettingGateway, SettingGatewayDefault)
}
<commit_msg>main: set up replacer of config key names to match env variable names<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/mendersoftware/go-lib-micro/log"

	"github.com/mendersoftware/deployments/config"
	"github.com/spf13/viper"
)

func main() {

	var configPath string
	var printVersion bool
	flag.StringVar(&configPath, "config", "", "Configuration file path. Supports JSON, TOML, YAML and HCL formatted configs.")
	flag.BoolVar(&printVersion, "version", false, "Show version")

	flag.Parse()

	if printVersion {
		fmt.Println("Version:", CreateVersionString())
		fmt.Println("BuildNumber:", BuildNumber)
		os.Exit(0)
	}

	l := log.New(log.Ctx{})

	configuration, err := HandleConfigFile(configPath)
	if err != nil {
		l.Fatalf("error loading configuration: %s", err)
	}

	l.Fatal(RunServer(configuration))
}

func HandleConfigFile(filePath string) (config.ConfigReader, error) {

	c := viper.New()

	// map settings such as foo.bar and foo-bar to FOO_BAR environment keys
	c.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))

	// Enable setting also other conig values by environment variables
	c.SetEnvPrefix("DEPLOYMENTS")
	c.AutomaticEnv()

	// Set default values for config
	SetDefaultConfigs(c)

	// Find and read the config file
	if filePath != "" {
		c.SetConfigFile(filePath)
		if err := c.ReadInConfig(); err != nil {
			return nil, err
		}
	}

	// Validate config
	if err := config.ValidateConfig(c,
		ValidateAwsAuth,
		ValidateHttps,
	); err != nil {
		return nil, err
	}

	return c, nil
}

func SetDefaultConfigs(config *viper.Viper) {
	config.SetDefault(SettingListen, SettingListenDefault)
	config.SetDefault(SettingAwsS3Region, SettingAwsS3RegionDefault)
	config.SetDefault(SettingAwsS3Bucket, SettingAwsS3BucketDefault)
	config.SetDefault(SettingMongo, SettingMongoDefault)
	config.SetDefault(SettingGateway, SettingGatewayDefault)
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	htmltemplate "html/template"
	"text/template"

	"github.com/Masterminds/sprig"
)

var (
	flagInput     = flag.String("f", "-", "Input source")
	flagOutput    = flag.String("w", "-", "Output destination")
	flagHTML      = flag.Bool("html", false, "If true, use html/template instead of text/template")
	flagRecursive = flag.String("r", "", "If provided, traverse the argument as a directory")
	flagStripN    = flag.Int("stripn", 0, "If provided, strips this many directories from the output (only valid if -r and -w are provided)")
)

func main() {
	flag.Parse()
	if err := run(*flagInput, *flagOutput, *flagRecursive, *flagHTML); err != nil {
		fmt.Fprintln(os.Stderr, "tmpl error:", err)
		os.Exit(1)
	}
}

func run(input, output string, recurseDir string, htmlMode bool) error {
	in, err := getInput(*flagInput)
	if err != nil {
		return err
	}
	if recurseDir != "" {
		return runDir(recurseDir, htmlMode, *flagOutput, *flagStripN, envMap())
	}
	out, err := getOutput(*flagOutput)
	if err != nil {
		return err
	}
	return tmpl(in, *flagHTML, out, envMap())
}

func getInput(path string) (io.Reader, error) {
	if path == "-" {
		return os.Stdin, nil
	}
	return os.Open(path)
}

func getOutput(path string) (io.Writer, error) {
	if path == "-" {
		return os.Stdout, nil
	}
	return os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
}

func envMap() map[string]string {
	result := map[string]string{}
	for _, envvar := range os.Environ() {
		parts := strings.SplitN(envvar, "=", 2)
		result[parts[0]] = parts[1]
	}
	return result
}

func tmpl(in io.Reader, htmlMode bool, out io.Writer, ctx interface{}) error {
	i, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	if htmlMode {
		tmpl, err := htmltemplate.New("format string").Funcs(sprig.HtmlFuncMap()).Parse(string(i))
		if err != nil {
			return err
		}
		return tmpl.Execute(out, ctx)
	}
	tmpl, err := template.New("format string").Funcs(sprig.TxtFuncMap()).Parse(string(i))
	if err != nil {
		return err
	}
	return tmpl.Execute(out, ctx)
}

func tmplToString(in io.Reader, htmlMode bool, ctx interface{}) (string, error) {
	o := bytes.NewBuffer([]byte{})
	err := tmpl(in, htmlMode, o, ctx)
	return o.String(), err
}

func tmplStr(in string, ctx interface{}) string {
	o := bytes.NewBuffer([]byte{})
	err := tmpl(strings.NewReader(in), false, o, ctx)
	if err != nil {
		fmt.Fprintln(os.Stderr, "issue rendering string:", err)
	}
	return o.String()
}

func runDir(dir string, htmlMode bool, outPath string, stripN int, ctx interface{}) error {
	buf := new(bytes.Buffer)
	tw := tar.NewWriter(buf)
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.Mode().IsRegular() {
			return nil
		}
		f, err := os.Open(path)
		if err != nil {
			return err
		}
		contents, err := tmplToString(f, htmlMode, ctx)
		if err != nil {
			return err
		}
		hdr := &tar.Header{
			Name: tmplStr(path, ctx),
			Mode: int64(info.Mode()),
			Size: int64(len(contents)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		if _, err := tw.Write([]byte(contents)); err != nil {
			return err
		}
		return tw.Flush()
	})
	if err != nil {
		return fmt.Errorf("issue recursing: %w", err)
	}
	if outPath == "-" {
		_, err = io.Copy(os.Stdout, buf)
		return err
	}
	return extractTar(buf, outPath, stripN)
}

func extractTar(buf io.Reader, outPath string, stripN int) error {
	tarReader := tar.NewReader(buf)
	for true {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("Next() failed: %w", err)
		}
		path := header.Name
		parts := strings.Split(path, string(filepath.Separator))
		toStrip := stripN
		if toStrip >= len(parts) {
			toStrip = len(parts) - 1
		}
		if len(parts) > toStrip {
			path = strings.Join(parts[toStrip:], string(filepath.Separator))
		}
		fmt.Println("resulting path:", path)
		fullPath := filepath.Join(outPath, path)
		if err := ensureEnclosingDir(fullPath); err != nil {
			return fmt.Errorf("issue ensuring directory exists: %w", err)
		}
		switch header.Typeflag {
		case tar.TypeDir:
			if err := os.Mkdir(fullPath, 0755); err != nil {
				return fmt.Errorf("Mkdir() failed: %w", err)
			}
		case tar.TypeReg:
			outFile, err := os.Create(fullPath)
			if err != nil {
				return fmt.Errorf("Create() failed: %w", err)
			}
			if _, err := io.Copy(outFile, tarReader); err != nil {
				return fmt.Errorf("io.Copy() failed: %w", err)
			}
			outFile.Close()

		default:
			return fmt.Errorf("extractTar: uknown type: %v in %v", header.Typeflag, header.Name)
		}
	}
	return nil
}

func ensureEnclosingDir(path string) error {
	return os.MkdirAll(filepath.Dir(path), 0755)
}
<commit_msg>tmpl: Remove extraneous output<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	htmltemplate "html/template"
	"text/template"

	"github.com/Masterminds/sprig"
)

var (
	flagInput     = flag.String("f", "-", "Input source")
	flagOutput    = flag.String("w", "-", "Output destination")
	flagHTML      = flag.Bool("html", false, "If true, use html/template instead of text/template")
	flagRecursive = flag.String("r", "", "If provided, traverse the argument as a directory")
	flagStripN    = flag.Int("stripn", 0, "If provided, strips this many directories from the output (only valid if -r and -w are provided)")
)

func main() {
	flag.Parse()
	if err := run(*flagInput, *flagOutput, *flagRecursive, *flagHTML); err != nil {
		fmt.Fprintln(os.Stderr, "tmpl error:", err)
		os.Exit(1)
	}
}

func run(input, output string, recurseDir string, htmlMode bool) error {
	in, err := getInput(*flagInput)
	if err != nil {
		return err
	}
	if recurseDir != "" {
		return runDir(recurseDir, htmlMode, *flagOutput, *flagStripN, envMap())
	}
	out, err := getOutput(*flagOutput)
	if err != nil {
		return err
	}
	return tmpl(in, *flagHTML, out, envMap())
}

func getInput(path string) (io.Reader, error) {
	if path == "-" {
		return os.Stdin, nil
	}
	return os.Open(path)
}

func getOutput(path string) (io.Writer, error) {
	if path == "-" {
		return os.Stdout, nil
	}
	return os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
}

func envMap() map[string]string {
	result := map[string]string{}
	for _, envvar := range os.Environ() {
		parts := strings.SplitN(envvar, "=", 2)
		result[parts[0]] = parts[1]
	}
	return result
}

func tmpl(in io.Reader, htmlMode bool, out io.Writer, ctx interface{}) error {
	i, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	if htmlMode {
		tmpl, err := htmltemplate.New("format string").Funcs(sprig.HtmlFuncMap()).Parse(string(i))
		if err != nil {
			return err
		}
		return tmpl.Execute(out, ctx)
	}
	tmpl, err := template.New("format string").Funcs(sprig.TxtFuncMap()).Parse(string(i))
	if err != nil {
		return err
	}
	return tmpl.Execute(out, ctx)
}

func tmplToString(in io.Reader, htmlMode bool, ctx interface{}) (string, error) {
	o := bytes.NewBuffer([]byte{})
	err := tmpl(in, htmlMode, o, ctx)
	return o.String(), err
}

func tmplStr(in string, ctx interface{}) string {
	o := bytes.NewBuffer([]byte{})
	err := tmpl(strings.NewReader(in), false, o, ctx)
	if err != nil {
		fmt.Fprintln(os.Stderr, "issue rendering string:", err)
	}
	return o.String()
}

func runDir(dir string, htmlMode bool, outPath string, stripN int, ctx interface{}) error {
	buf := new(bytes.Buffer)
	tw := tar.NewWriter(buf)
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.Mode().IsRegular() {
			return nil
		}
		f, err := os.Open(path)
		if err != nil {
			return err
		}
		contents, err := tmplToString(f, htmlMode, ctx)
		if err != nil {
			return err
		}
		hdr := &tar.Header{
			Name: tmplStr(path, ctx),
			Mode: int64(info.Mode()),
			Size: int64(len(contents)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		if _, err := tw.Write([]byte(contents)); err != nil {
			return err
		}
		return tw.Flush()
	})
	if err != nil {
		return fmt.Errorf("issue recursing: %w", err)
	}
	if outPath == "-" {
		_, err = io.Copy(os.Stdout, buf)
		return err
	}
	return extractTar(buf, outPath, stripN)
}

func extractTar(buf io.Reader, outPath string, stripN int) error {
	tarReader := tar.NewReader(buf)
	for true {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("Next() failed: %w", err)
		}
		path := header.Name
		parts := strings.Split(path, string(filepath.Separator))
		toStrip := stripN
		if toStrip >= len(parts) {
			toStrip = len(parts) - 1
		}
		if len(parts) > toStrip {
			path = strings.Join(parts[toStrip:], string(filepath.Separator))
		}
		fullPath := filepath.Join(outPath, path)
		if err := ensureEnclosingDir(fullPath); err != nil {
			return fmt.Errorf("issue ensuring directory exists: %w", err)
		}
		switch header.Typeflag {
		case tar.TypeDir:
			if err := os.Mkdir(fullPath, 0755); err != nil {
				return fmt.Errorf("Mkdir() failed: %w", err)
			}
		case tar.TypeReg:
			outFile, err := os.Create(fullPath)
			if err != nil {
				return fmt.Errorf("Create() failed: %w", err)
			}
			if _, err := io.Copy(outFile, tarReader); err != nil {
				return fmt.Errorf("io.Copy() failed: %w", err)
			}
			outFile.Close()

		default:
			return fmt.Errorf("extractTar: uknown type: %v in %v", header.Typeflag, header.Name)
		}
	}
	return nil
}

func ensureEnclosingDir(path string) error {
	return os.MkdirAll(filepath.Dir(path), 0755)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/gorilla/websocket"
	"github.com/mediocregopher/lever"
)

var (
	listenAddr      string
	etcdAPIs        []string
	dnsRoot         string
	defaultCategory string
	timeout         time.Duration
	etcdClient      *etcd.Client
)

func main() {
	l := lever.New("skyapi", nil)
	l.Add(lever.Param{
		Name:        "--listen-addr",
		Description: "Address to listen for websocket connections on",
		Default:     ":8053",
	})
	l.Add(lever.Param{
		Name:         "--etcd-api",
		Description:  "scheme://address an etcd node in the cluster can be found on. Can be specified multiple times",
		DefaultMulti: []string{"http://127.0.0.1:4001"},
	})
	l.Add(lever.Param{
		Name:        "--dns-root",
		Description: "Root domain that SkyDNS is serving on",
		Default:     "skydns.local.",
	})
	l.Add(lever.Param{
		Name:        "--timeout",
		Description: "The TTL for entries in SkyDNS, in seconds. The server will ping at half this value, the client should also",
		Default:     "30",
	})
	l.Add(lever.Param{
		Name:        "--default-category",
		Description: "The default category to file incoming connections over, if they don't specify",
		Default:     "services",
	})
	l.Parse()

	listenAddr, _ = l.ParamStr("--listen-addr")
	etcdAPIs, _ = l.ParamStrs("--etcd-api")
	dnsRoot, _ = l.ParamStr("--dns-root")
	timeoutSecs, _ := l.ParamInt("--timeout")
	timeout = time.Duration(timeoutSecs) * time.Second
	defaultCategory, _ = l.ParamStr("--default-category")

	etcdClient = etcd.NewClient(etcdAPIs)

	http.Handle("/provide", http.HandlerFunc(handler))
	log.Printf("listening on %s", listenAddr)
	log.Fatal(http.ListenAndServe(listenAddr, nil))
}

func randID() string {
	b := make([]byte, 10)
	_, err := rand.Read(b)
	if err != nil {
		panic(err)
	}
	return hex.EncodeToString(b)
}

var upgrader = websocket.Upgrader{
	// Buffer sizes are 0 because nothing should ever be read or written
	ReadBufferSize:  0,
	WriteBufferSize: 0,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

type connData struct {
	id                string
	service, category string
	Host              string `json:"host"`
	Port              int    `json:"port,omitempty"`
	Priority          int    `json:"priority,omitempty"`
	Weight            int    `json:"weight,omitempty"`
}

func (cd connData) sprintf(s string, args ...interface{}) string {
	realArgs := make([]interface{}, 0, len(args)+5)
	realArgs = append(realArgs, cd.id, cd.service, cd.category, cd.Host, cd.Port)
	realArgs = append(realArgs, args...)
	return fmt.Sprintf("[%s - %s.%s] %s:%d - "+s, realArgs...)
}

func (cd connData) toPath() (string, string) {
	parts := strings.Split(dnsRoot, ".")
	partsR := append(make([]string, 0, len(parts)+2), "/skydns")
	for i := len(parts) - 1; i >= 0; i-- {
		partsR = append(partsR, parts[i])
	}
	partsR = append(partsR, cd.category, cd.service)
	dir := path.Join(partsR...)
	file := path.Join(dir, cd.id)
	return dir, file
}

func errorMessagef(conn *websocket.Conn, s string, args ...interface{}) {
	msg := fmt.Sprintf("ERROR: "+s, args...)
	conn.WriteMessage(websocket.TextMessage, []byte(msg))
}

func handler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Printf("error upgrading: %s", err)
		return
	}
	closeCh := make(chan struct{})
	go readDiscard(conn, closeCh)
	defer conn.Close()

	cd, err := parseConnData(r)
	if err != nil {
		log.Printf("parseConnData: %s", err)
		errorMessagef(conn, "parseConnData: %s", err)
		return
	}

	defer log.Print(cd.sprintf("closed"))
	defer etcdDelete(cd)
	log.Print(cd.sprintf("connected"))

	tick := time.Tick(timeout / 2)
	if !doTick(conn, cd) {
		return
	}
	for {
		select {
		case <-tick:
			if !doTick(conn, cd) {
				return
			}
		case <-closeCh:
			return
		}
	}
}

func doTick(conn *websocket.Conn, cd connData) bool {
	deadline := time.Now().Add(timeout / 2)
	err := conn.WriteControl(websocket.PingMessage, nil, deadline)
	if err != nil {
		log.Print(cd.sprintf("timedout"))
		return false
	}

	if err = etcdStore(cd); err != nil {
		log.Print(cd.sprintf("storing etcd data: %s", err))
		errorMessagef(conn, "storing etcd data: %s", err)
		return false
	}
	return true
}

func readDiscard(conn *websocket.Conn, closeCh chan struct{}) {
	for {
		if _, _, err := conn.NextReader(); err != nil {
			close(closeCh)
			return
		}
	}
}

func parseConnData(r *http.Request) (connData, error) {
	service := r.FormValue("service")
	category := r.FormValue("category")
	host := r.FormValue("host")
	portStr := r.FormValue("port")
	var port int
	var err error

	if service == "" {
		err = fmt.Errorf("service and port are required parameters")
		return connData{}, err
	}

	if category == "" {
		category = defaultCategory
	}

	if host == "" {
		host = r.RemoteAddr[:strings.Index(r.RemoteAddr, ":")]
	}

	if portStr != "" {
		port, err = strconv.Atoi(portStr)
		if err != nil {
			return connData{}, err
		}
	}

	var priority int
	priorityStr, weightStr := r.FormValue("priority"), r.FormValue("weight")
	if priorityStr != "" {
		if priority, err = strconv.Atoi(priorityStr); err != nil {
			return connData{}, err
		}
	}

	var weight int
	if weightStr != "" {
		if weight, err = strconv.Atoi(weightStr); err != nil {
			return connData{}, err
		}
	}

	return connData{
		id:       randID(),
		service:  service,
		category: category,
		Host:     host,
		Port:     port,
		Priority: priority,
		Weight:   weight,
	}, nil
}

// Creates the given dir (and all of its parent directories if they don't
// already exist). Will not return an error if the given directory already
// exists
func MkDirP(ec *etcd.Client, dir string) error {
	parts := make([]string, 0, 4)
	for {
		parts = append(parts, dir)
		dir = path.Dir(dir)
		if dir == "/" {
			break
		}
	}

	for i := range parts {
		ai := len(parts) - i - 1
		_, err := ec.CreateDir(parts[ai], 0)
		if err != nil && err.(*etcd.EtcdError).ErrorCode != 105 {
			return err
		}
	}
	return nil
}

func etcdStore(cd connData) error {
	dir, file := cd.toPath()
	if err := MkDirP(etcdClient, dir); err != nil {
		return err
	}

	j, err := json.Marshal(cd)
	if err != nil {
		return err
	}

	_, err = etcdClient.Set(file, string(j), uint64(timeout.Seconds()))
	if err != nil {
		return err
	}

	return nil
}

func etcdDelete(cd connData) error {
	_, file := cd.toPath()
	_, err := etcdClient.Delete(file, false)
	return err
}
<commit_msg>make the ids stored in etcd be based on hashes of the connection's information, so they are consistent for the same service<commit_after>package main

import (
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/gorilla/websocket"
	"github.com/mediocregopher/lever"
)

var (
	listenAddr      string
	etcdAPIs        []string
	dnsRoot         string
	defaultCategory string
	timeout         time.Duration
	etcdClient      *etcd.Client
)

func main() {
	l := lever.New("skyapi", nil)
	l.Add(lever.Param{
		Name:        "--listen-addr",
		Description: "Address to listen for websocket connections on",
		Default:     ":8053",
	})
	l.Add(lever.Param{
		Name:         "--etcd-api",
		Description:  "scheme://address an etcd node in the cluster can be found on. Can be specified multiple times",
		DefaultMulti: []string{"http://127.0.0.1:4001"},
	})
	l.Add(lever.Param{
		Name:        "--dns-root",
		Description: "Root domain that SkyDNS is serving on",
		Default:     "skydns.local.",
	})
	l.Add(lever.Param{
		Name:        "--timeout",
		Description: "The TTL for entries in SkyDNS, in seconds. The server will ping at half this value, the client should also",
		Default:     "30",
	})
	l.Add(lever.Param{
		Name:        "--default-category",
		Description: "The default category to file incoming connections over, if they don't specify",
		Default:     "services",
	})
	l.Parse()

	listenAddr, _ = l.ParamStr("--listen-addr")
	etcdAPIs, _ = l.ParamStrs("--etcd-api")
	dnsRoot, _ = l.ParamStr("--dns-root")
	timeoutSecs, _ := l.ParamInt("--timeout")
	timeout = time.Duration(timeoutSecs) * time.Second
	defaultCategory, _ = l.ParamStr("--default-category")

	etcdClient = etcd.NewClient(etcdAPIs)

	http.Handle("/provide", http.HandlerFunc(handler))
	log.Printf("listening on %s", listenAddr)
	log.Fatal(http.ListenAndServe(listenAddr, nil))
}

var upgrader = websocket.Upgrader{
	// Buffer sizes are 0 because nothing should ever be read or written
	ReadBufferSize:  0,
	WriteBufferSize: 0,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

type connData struct {
	id                string
	service, category string
	Host              string `json:"host"`
	Port              int    `json:"port,omitempty"`
	Priority          int    `json:"priority,omitempty"`
	Weight            int    `json:"weight,omitempty"`
}

func (cd connData) sprintf(s string, args ...interface{}) string {
	realArgs := make([]interface{}, 0, len(args)+5)
	realArgs = append(realArgs, cd.id, cd.service, cd.category, cd.Host, cd.Port)
	realArgs = append(realArgs, args...)
	return fmt.Sprintf("[%s - %s.%s] %s:%d - "+s, realArgs...)
}

func (cd connData) toPath() (string, string) {
	parts := strings.Split(dnsRoot, ".")
	partsR := append(make([]string, 0, len(parts)+2), "/skydns")
	for i := len(parts) - 1; i >= 0; i-- {
		partsR = append(partsR, parts[i])
	}
	partsR = append(partsR, cd.category, cd.service)
	dir := path.Join(partsR...)
	file := path.Join(dir, cd.id)
	return dir, file
}

func errorMessagef(conn *websocket.Conn, s string, args ...interface{}) {
	msg := fmt.Sprintf("ERROR: "+s, args...)
	conn.WriteMessage(websocket.TextMessage, []byte(msg))
}

func handler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Printf("error upgrading: %s", err)
		return
	}
	closeCh := make(chan struct{})
	go readDiscard(conn, closeCh)
	defer conn.Close()

	cd, err := parseConnData(r)
	if err != nil {
		log.Printf("parseConnData: %s", err)
		errorMessagef(conn, "parseConnData: %s", err)
		return
	}

	defer log.Print(cd.sprintf("closed"))
	defer etcdDelete(cd)
	log.Print(cd.sprintf("connected"))

	tick := time.Tick(timeout / 2)
	if !doTick(conn, cd) {
		return
	}
	for {
		select {
		case <-tick:
			if !doTick(conn, cd) {
				return
			}
		case <-closeCh:
			return
		}
	}
}

func doTick(conn *websocket.Conn, cd connData) bool {
	deadline := time.Now().Add(timeout / 2)
	err := conn.WriteControl(websocket.PingMessage, nil, deadline)
	if err != nil {
		log.Print(cd.sprintf("timedout"))
		return false
	}

	if err = etcdStore(cd); err != nil {
		log.Print(cd.sprintf("storing etcd data: %s", err))
		errorMessagef(conn, "storing etcd data: %s", err)
		return false
	}
	return true
}

func readDiscard(conn *websocket.Conn, closeCh chan struct{}) {
	for {
		if _, _, err := conn.NextReader(); err != nil {
			close(closeCh)
			return
		}
	}
}

func parseConnData(r *http.Request) (connData, error) {
	service := r.FormValue("service")
	category := r.FormValue("category")
	host := r.FormValue("host")
	portStr := r.FormValue("port")
	var port int
	var err error

	if service == "" {
		err = fmt.Errorf("service and port are required parameters")
		return connData{}, err
	}

	if category == "" {
		category = defaultCategory
	}

	if host == "" {
		host = r.RemoteAddr[:strings.Index(r.RemoteAddr, ":")]
	}

	if portStr != "" {
		port, err = strconv.Atoi(portStr)
		if err != nil {
			return connData{}, err
		}
	}

	var priority int
	priorityStr, weightStr := r.FormValue("priority"), r.FormValue("weight")
	if priorityStr != "" {
		if priority, err = strconv.Atoi(priorityStr); err != nil {
			return connData{}, err
		}
	}

	var weight int
	if weightStr != "" {
		if weight, err = strconv.Atoi(weightStr); err != nil {
			return connData{}, err
		}
	}

	sha := sha1.New()
	fmt.Fprint(sha, service)
	fmt.Fprint(sha, category)
	fmt.Fprint(sha, host)
	fmt.Fprint(sha, port)
	id := hex.EncodeToString(sha.Sum(nil))

	return connData{
		id:       id,
		service:  service,
		category: category,
		Host:     host,
		Port:     port,
		Priority: priority,
		Weight:   weight,
	}, nil
}

// Creates the given dir (and all of its parent directories if they don't
// already exist). Will not return an error if the given directory already
// exists
func MkDirP(ec *etcd.Client, dir string) error {
	parts := make([]string, 0, 4)
	for {
		parts = append(parts, dir)
		dir = path.Dir(dir)
		if dir == "/" {
			break
		}
	}

	for i := range parts {
		ai := len(parts) - i - 1
		_, err := ec.CreateDir(parts[ai], 0)
		if err != nil && err.(*etcd.EtcdError).ErrorCode != 105 {
			return err
		}
	}
	return nil
}

func etcdStore(cd connData) error {
	dir, file := cd.toPath()
	if err := MkDirP(etcdClient, dir); err != nil {
		return err
	}

	j, err := json.Marshal(cd)
	if err != nil {
		return err
	}

	_, err = etcdClient.Set(file, string(j), uint64(timeout.Seconds()))
	if err != nil {
		return err
	}

	return nil
}

func etcdDelete(cd connData) error {
	_, file := cd.toPath()
	_, err := etcdClient.Delete(file, false)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"github.com/golang/glog"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
)

const (
	// declarations
	a string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	base int = 62	// character's known in 'a'
	domain_name = "foxley.co:22222" // remove port when kube manifest are ready
	prefix = "http://"
	sep = "/"
	configFile = ".env/foxley_mock.json"
)

//	default handler
func (env *Env) notFound(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("route not found... sorry! :("))
}

func (env *Env) createEntry(res http.ResponseWriter, req *http.Request) {

	glog.Info("Inbound Traffic: ")
	// declare content type 'json'
	res.Header().Set("Content-Type", "application/json; charset=UTF-8")

	// capture and decode the payload from stream
	url := new(Url)
	decoder := json.NewDecoder(req.Body)
	error := decoder.Decode(&url)
	checkErr(error)
	glog.Info(url)

	// prepare insert statement
	query, _ := env.db.Prepare("INSERT IGNORE INTO nin SET url=?")

	// execute
	execute, err := query.Exec(url.Url)
	checkErr(err)

	// grab the id, encode, and suffix it to domain_name
	id, err2 := execute.LastInsertId()
	checkErr(err2)
	shorterUrl := domain_name + sep + "p" + sep + encode(int(id), base, a)

	// return the response
	glog.Info(shorterUrl)
	res.WriteHeader(http.StatusCreated)
	if err3 := json.NewEncoder(res).Encode(&Url{
		shorterUrl,
	}); err3 != nil {
		log.Fatal(err)
	}
}

func (env *Env) retrieveEntry(res http.ResponseWriter, req *http.Request) {

	glog.Info("Outbound Traffic: ")
	// declare content type 'json'
	res.Header().Set("Content-Type", "application/json; charset=UTF-8")

	vars := mux.Vars(req)
	shorterUrl := vars["encoded_value"]
	id := decode(shorterUrl, a)

	// query statement
	var decodedUri string
	err := env.db.QueryRow("SELECT URL FROM nin WHERE ID=?",
		strconv.Itoa(id)).Scan(&decodedUri)

	if len(decodedUri) == 0 || decodedUri == "" {
		decodedUri = domain_name + sep + "404" + sep
	}
	fmt.Print(err)
	//checkErr(err)

	glog.Info("Redirecting: ", prefix + decodedUri)
	http.Redirect(res, req, prefix + decodedUri, 301)
}

func main() {

	glog.Info("Service is warming up")
	// load config from file (remove this in the future, should use kub secret)
	config := loadConfig(configFile)

	dsn := config.DB_USER + ":" + config.DB_PASS + "@" + config.DB_HOST + "/" + config.DB_NAME
	db, err := NewDB(dsn)
	checkErr(err)
	defer db.Close()

	env := &Env{db : db}

	startService(env, db)
}

func startService(env *Env, db *sql.DB) {
	glog.Info("Current config: ", configFile)
	var version string
	db.QueryRow("SELECT VERSION()").Scan(&version)
	glog.Info("Connected to: ", version)

	// CORS; re-review later
	headersOk := handlers.AllowedHeaders([]string{"Content-Type", "Authorization", "Content-Length", "X-Requested-With"})
	originsOk := handlers.AllowedOrigins([]string{"*"})
	methodsOk := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "OPTIONS"})

	r := mux.NewRouter()
	r.HandleFunc("/p/{encoded_value}", env.retrieveEntry)
	r.HandleFunc("/create", env.createEntry).Methods("POST")
	r.NotFoundHandler = http.Handler(http.StripPrefix("/404", http.FileServer(http.Dir("./static/404/"))))

	glog.Info("Service is ready")
	glog.Fatal(http.ListenAndServe(":22222", handlers.CORS(originsOk, headersOk, methodsOk)(r)))
}
//
//curl -X POST -H "Content-Type: application/json" -d '{"url":"foxley.co/okay"}' http://127.0.0.1:8080/create -v
<commit_msg>added hardcored validation for http, https, www, stripping<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"
	s "strings"
	"github.com/golang/glog"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
)

const (
	// declarations
	a string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	base int = 62	// character's known in 'a'
	domain_name = "foxley.co:22222" // remove port when kube manifest are ready
	prefix = "http://"
	sep = "/"
	configFile = ".env/foxley_mock.json"
)

//	default handler
func (env *Env) notFound(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("route not found... sorry! :("))
}

func (env *Env) createEntry(res http.ResponseWriter, req *http.Request) {

	glog.Info("Inbound Traffic: ")
	// declare content type 'json'
	res.Header().Set("Content-Type", "application/json; charset=UTF-8")

	// capture and decode the payload from stream
	urlPayload := new(Url)
	decoder := json.NewDecoder(req.Body)
	error := decoder.Decode(&urlPayload)
	checkErr(error)

	// check for http, https, and www, fail other cases, should subsitiute as regex later
	updatedUrl := urlPayload.Url
	updatedUrl = s.Replace(updatedUrl, "http://www", "", 1)
	updatedUrl = s.Replace(updatedUrl, "https://www", "", 1)
	updatedUrl = s.Replace(updatedUrl, "https://", "", 1)
	updatedUrl = s.Replace(updatedUrl, "http://", "", 1)

	// check if it really exists?
	//_, err := url.ParseRequestURI(updatedUrl)
	//if err != nil {
	//	glog.Fatal(err)
	//}

	// prepare insert statement
	query, _ := env.db.Prepare("INSERT IGNORE INTO nin SET url=?")

	// execute
	execute, err := query.Exec(updatedUrl)
	checkErr(err)

	// grab the id, encode, and suffix it to domain_name
	id, err2 := execute.LastInsertId()
	checkErr(err2)
	shorterUrl := domain_name + sep + "p" + sep + encode(int(id), base, a)

	// return the response
	glog.Info(shorterUrl)
	res.WriteHeader(http.StatusCreated)
	if err3 := json.NewEncoder(res).Encode(&Url{
		shorterUrl,
	}); err3 != nil {
		log.Fatal(err)
	}
}

func (env *Env) retrieveEntry(res http.ResponseWriter, req *http.Request) {

	glog.Info("Outbound Traffic: ")
	// declare content type 'json'
	res.Header().Set("Content-Type", "application/json; charset=UTF-8")

	vars := mux.Vars(req)
	shorterUrl := vars["encoded_value"]
	id := decode(shorterUrl, a)

	// query statement
	var decodedUri string
	err := env.db.QueryRow("SELECT URL FROM nin WHERE ID=?",
		strconv.Itoa(id)).Scan(&decodedUri)

	if len(decodedUri) == 0 || decodedUri == "" {
		decodedUri = domain_name + sep + "404" + sep
	}
	fmt.Print(err)
	//checkErr(err)

	glog.Info("Redirecting: ", prefix + decodedUri)
	http.Redirect(res, req, prefix + decodedUri, 301)
}

func main() {

	glog.Info("Service is warming up")
	// load config from file (remove this in the future, should use kub secret)
	config := loadConfig(configFile)

	dsn := config.DB_USER + ":" + config.DB_PASS + "@" + config.DB_HOST + "/" + config.DB_NAME
	db, err := NewDB(dsn)
	checkErr(err)
	defer db.Close()

	env := &Env{db : db}

	startService(env, db)
}

func startService(env *Env, db *sql.DB) {
	glog.Info("Current config: ", configFile)
	var version string
	db.QueryRow("SELECT VERSION()").Scan(&version)
	glog.Info("Connected to: ", version)

	// CORS; re-review later
	headersOk := handlers.AllowedHeaders([]string{"Content-Type", "Authorization", "Content-Length", "X-Requested-With"})
	originsOk := handlers.AllowedOrigins([]string{"*"})
	methodsOk := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "OPTIONS"})

	r := mux.NewRouter()
	r.HandleFunc("/p/{encoded_value}", env.retrieveEntry)
	r.HandleFunc("/create", env.createEntry).Methods("POST")
	r.NotFoundHandler = http.Handler(http.StripPrefix("/404", http.FileServer(http.Dir("./static/404/"))))

	glog.Info("Service is ready")
	glog.Fatal(http.ListenAndServe(":22222", handlers.CORS(originsOk, headersOk, methodsOk)(r)))
}
//
//curl -X POST -H "Content-Type: application/json" -d '{"url":"foxley.co/okay"}' http://127.0.0.1:8080/create -v
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/gogoprotobuf/proto"
	pb "github.com/dgryski/carbonzipper/carbonzipperpb"

	"github.com/bradfitz/gomemcache/memcache"
	"github.com/peterbourgon/g2g"
)

type zipper string

var Zipper zipper

var Metrics = struct {
	Requests         *expvar.Int
	RequestCacheHits *expvar.Int

	FindRequests  *expvar.Int
	FindCacheHits *expvar.Int

	RenderRequests *expvar.Int
}{
	Requests:         expvar.NewInt("requests"),
	RequestCacheHits: expvar.NewInt("request_cache_hits"),

	FindRequests:  expvar.NewInt("find_requests"),
	FindCacheHits: expvar.NewInt("find_cache_hits"),

	RenderRequests: expvar.NewInt("render_requests"),
}

var queryCache bytesCache
var findCache bytesCache

var timeFormats = []string{"15:04 20060102", "20060102", "01/02/06"}

// dateParamToEpoch turns a passed string parameter into an epoch we can send to the Zipper
func dateParamToEpoch(s string, d int64) int32 {

	if s == "" {
		// return the default if nothing was passed
		return int32(d)
	}

	// relative timestamp
	if s[0] == '-' {

		offset, err := intervalString(s[1:])
		if err != nil {
			return int32(d)
		}

		return int32(timeNow().Add(-time.Duration(offset) * time.Second).Unix())
	}

	if s == "now" {
		return int32(timeNow().Unix())
	}

	sint, err := strconv.Atoi(s)
	if err == nil && len(s) > 8 {
		return int32(sint) // We got a timestamp so returning it
	}

	if strings.Contains(s, "_") {
		s = strings.Replace(s, "_", " ", 1) // Go can't parse _ in date strings
	}

	for _, format := range timeFormats {
		t, err := time.Parse(format, s)
		if err == nil {
			return int32(t.Unix())
		}
	}
	return int32(d)
}

func intervalString(s string) (int32, error) {

	// TODO(dgryski): add defaultSign param for use if there is no +/- sign provided by string

	var j int

	for j < len(s) && s[j] >= '0' && s[j] <= '9' {
		j++
	}
	offsetStr, unitStr := s[:j], s[j:]

	var units int
	switch unitStr {
	case "s", "sec", "secs", "second", "seconds":
		units = 1
	case "min", "minute", "minutes":
		units = 60
	case "h", "hour", "hours":
		units = 60 * 60
	case "d", "day", "days":
		units = 24 * 60 * 60
	case "w", "week", "weeks":
		units = 7 * 24 * 60 * 60
	case "mon", "month", "months":
		units = 30 * 24 * 60 * 60
	case "y", "year", "years":
		units = 365 * 24 * 60 * 60
	default:
		return 0, errors.New("unknown time units")
	}

	offset, err := strconv.Atoi(offsetStr)
	if err != nil {
		return 0, err
	}

	return int32(offset * units), nil
}

// FIXME(dgryski): extract the http.Get + unproto code into its own function

func (z zipper) Find(metric string) (pb.GlobResponse, error) {

	u, _ := url.Parse(string(z) + "/metrics/find/")

	u.RawQuery = url.Values{
		"query":  []string{metric},
		"format": []string{"protobuf"},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Find: http.Get: %+v\n", err)
		return pb.GlobResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Find: ioutil.ReadAll: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	var pbresp pb.GlobResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Find: proto.Unmarshal: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	return pbresp, nil
}

func (z zipper) Render(metric string, from, until int32) (pb.FetchResponse, error) {

	u, _ := url.Parse(string(z) + "/render/")

	u.RawQuery = url.Values{
		"target": []string{metric},
		"format": []string{"protobuf"},
		"from":   []string{strconv.Itoa(int(from))},
		"until":  []string{strconv.Itoa(int(until))},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Render: http.Get: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Render: ioutil.ReadAll: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	var pbresp pb.FetchResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Render: proto.Unmarshal: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	return pbresp, nil
}

type limiter chan struct{}

func (l limiter) enter() { l <- struct{}{} }
func (l limiter) leave() { <-l }

var Limiter limiter

// for testing
var timeNow = time.Now

type graphitePoint struct {
	value float64
	t     int32
}

func (g graphitePoint) MarshalJSON() ([]byte, error) {
	// TODO(dgryski): fmt.Sprintf() is slow, use strconv.Append{Float,Int}
	// TODO(dgryski): MarshalJSON call should be on jsonResponse to reduce overhead
	if math.IsNaN(g.value) {
		return []byte(fmt.Sprintf("[null,%d]", g.t)), nil
	}
	return []byte(fmt.Sprintf("[%g,%d]", g.value, g.t)), nil
}

type jsonResponse struct {
	Target     string          `json:"target"`
	Datapoints []graphitePoint `json:"datapoints"`
}

func renderHandler(w http.ResponseWriter, r *http.Request) {

	Metrics.Requests.Add(1)

	err := r.ParseForm()
	if err != nil {
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}

	targets := r.Form["target"]
	from := r.FormValue("from")
	until := r.FormValue("until")
	useCache := r.FormValue("noCache") == ""

	// make sure the cache key doesn't say noCache, because it will never hit
	r.Form.Del("noCache")

	cacheKey := r.Form.Encode()

	if response, ok := queryCache.get(cacheKey); useCache && ok {
		Metrics.RequestCacheHits.Add(1)
		w.Header().Set("Content-Type", "application/json")
		w.Write(response)
		return
	}

	// normalize from and until values
	// BUG(dgryski): doesn't handle timezones the same as graphite-web
	from32 := dateParamToEpoch(from, timeNow().Add(-24*time.Hour).Unix())
	until32 := dateParamToEpoch(until, timeNow().Unix())

	var results []*pb.FetchResponse
	metricMap := make(map[metricRequest][]*pb.FetchResponse)

	for _, target := range targets {

		exp, e, err := parseExpr(target)
		if err != nil || e != "" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}

		for _, m := range exp.metrics() {

			mfetch := m
			mfetch.from += from32
			mfetch.until += until32

			if _, ok := metricMap[mfetch]; ok {
				// already fetched this metric for this request
				continue
			}

			var glob pb.GlobResponse
			var haveCacheData bool

			if response, ok := findCache.get(m.metric); useCache && ok {
				Metrics.FindCacheHits.Add(1)
				err := proto.Unmarshal(response, &glob)
				haveCacheData = err == nil
			}

			if !haveCacheData {
				var err error
				Metrics.FindRequests.Add(1)
				glob, err = Zipper.Find(m.metric)
				if err != nil {
					continue
				}
				b, err := proto.Marshal(&glob)
				if err == nil {
					findCache.set(m.metric, b)
				}
			}

			// For each metric returned in the Find response, query Render
			// This is a conscious decision to *not* cache render data
			rch := make(chan *pb.FetchResponse, len(glob.GetMatches()))
			leaves := 0
			for _, m := range glob.GetMatches() {
				if !m.GetIsLeaf() {
					continue
				}
				Metrics.RenderRequests.Add(1)
				leaves++
				Limiter.enter()
				go func(m *pb.GlobMatch, from, until int32) {
					var rptr *pb.FetchResponse
					r, err := Zipper.Render(m.GetPath(), from, until)
					if err == nil {
						rptr = &r
					}
					rch <- rptr
					Limiter.leave()
				}(m, mfetch.from, mfetch.until)
			}

			for i := 0; i < leaves; i++ {
				r := <-rch
				if r != nil {
					metricMap[mfetch] = append(metricMap[mfetch], r)
				}
			}
		}

		exprs := evalExpr(exp, from32, until32, metricMap)
		results = append(results, exprs...)
	}

	var jresults []jsonResponse

	for _, r := range results {
		if r == nil {
			log.Println("skipping nil result")
		}
		datapoints := make([]graphitePoint, 0, len(r.Values))
		t := *r.StartTime
		for i, v := range r.Values {
			if r.IsAbsent[i] {
				v = math.NaN()
			}
			datapoints = append(datapoints, graphitePoint{value: v, t: t})
			t += *r.StepTime
		}
		jresults = append(jresults, jsonResponse{Target: r.GetName(), Datapoints: datapoints})
	}

	jout, err := json.Marshal(jresults)
	if err != nil {
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	queryCache.set(cacheKey, jout)

	w.Header().Set("Content-Type", "application/json")
	w.Write(jout)
}

func lbcheckHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Ok\n")
}

func main() {

	z := flag.String("z", "", "zipper")
	port := flag.Int("p", 8080, "port")
	l := flag.Int("l", 20, "concurrency limit")
	mc := flag.String("mc", "", "comma separated memcached server list")
	cpus := flag.Int("cpus", 0, "number of CPUs to use")

	flag.Parse()

	if p := os.Getenv("PORT"); p != "" {
		*port, _ = strconv.Atoi(p)
	}

	Limiter = make(chan struct{}, *l)

	if *z == "" {
		log.Fatal("no zipper provided")
	}

	if _, err := url.Parse(*z); err != nil {
		log.Fatal("unable to parze zipper:", err)
	}

	log.Println("using zipper", *z)

	if *mc != "" {
		servers := strings.Split(*mc, ",")
		log.Println("using memcache servers:", servers)
		queryCache = &memcachedCache{client: memcache.New(servers...)}
		findCache = &memcachedCache{client: memcache.New(servers...)}
	} else {
		queryCache = &expireCache{cache: make(map[string]cacheElement)}
		go queryCache.(*expireCache).cleaner()

		findCache = &expireCache{cache: make(map[string]cacheElement)}
		go findCache.(*expireCache).cleaner()
	}

	Zipper = zipper(*z)

	if *cpus != 0 {
		log.Println("using GOMAXPROCS", *cpus)
		runtime.GOMAXPROCS(*cpus)
	}

	if host := os.Getenv("GRAPHITEHOST") + ":" + os.Getenv("GRAPHITEPORT"); host != ":" {

		log.Println("Using graphite host", host)

		// register our metrics with graphite
		graphite, err := g2g.NewGraphite(host, 60*time.Second, 10*time.Second)
		if err != nil {
			log.Fatal("unable to connect to to graphite: ", host, ":", err)
		}

		hostname, _ := os.Hostname()
		hostname = strings.Replace(hostname, ".", "_", -1)

		graphite.Register(fmt.Sprintf("carbon.api.%s.requests", hostname), Metrics.Requests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.request_cache_hits", hostname), Metrics.RequestCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.find_requests", hostname), Metrics.FindRequests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.find_cache_hits", hostname), Metrics.FindCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.render_requests", hostname), Metrics.RenderRequests)
	}

	http.HandleFunc("/render/", renderHandler)
	http.HandleFunc("/lbcheck", lbcheckHandler)

	log.Println("listening on port", *port)
	log.Fatalln(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
<commit_msg>fixme -> todo, since it's not a bug, just a 'would-be-nice'<commit_after>package main

import (
	"encoding/json"
	"errors"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/gogoprotobuf/proto"
	pb "github.com/dgryski/carbonzipper/carbonzipperpb"

	"github.com/bradfitz/gomemcache/memcache"
	"github.com/peterbourgon/g2g"
)

type zipper string

var Zipper zipper

var Metrics = struct {
	Requests         *expvar.Int
	RequestCacheHits *expvar.Int

	FindRequests  *expvar.Int
	FindCacheHits *expvar.Int

	RenderRequests *expvar.Int
}{
	Requests:         expvar.NewInt("requests"),
	RequestCacheHits: expvar.NewInt("request_cache_hits"),

	FindRequests:  expvar.NewInt("find_requests"),
	FindCacheHits: expvar.NewInt("find_cache_hits"),

	RenderRequests: expvar.NewInt("render_requests"),
}

var queryCache bytesCache
var findCache bytesCache

var timeFormats = []string{"15:04 20060102", "20060102", "01/02/06"}

// dateParamToEpoch turns a passed string parameter into an epoch we can send to the Zipper
func dateParamToEpoch(s string, d int64) int32 {

	if s == "" {
		// return the default if nothing was passed
		return int32(d)
	}

	// relative timestamp
	if s[0] == '-' {

		offset, err := intervalString(s[1:])
		if err != nil {
			return int32(d)
		}

		return int32(timeNow().Add(-time.Duration(offset) * time.Second).Unix())
	}

	if s == "now" {
		return int32(timeNow().Unix())
	}

	sint, err := strconv.Atoi(s)
	if err == nil && len(s) > 8 {
		return int32(sint) // We got a timestamp so returning it
	}

	if strings.Contains(s, "_") {
		s = strings.Replace(s, "_", " ", 1) // Go can't parse _ in date strings
	}

	for _, format := range timeFormats {
		t, err := time.Parse(format, s)
		if err == nil {
			return int32(t.Unix())
		}
	}
	return int32(d)
}

func intervalString(s string) (int32, error) {

	// TODO(dgryski): add defaultSign param for use if there is no +/- sign provided by string

	var j int

	for j < len(s) && s[j] >= '0' && s[j] <= '9' {
		j++
	}
	offsetStr, unitStr := s[:j], s[j:]

	var units int
	switch unitStr {
	case "s", "sec", "secs", "second", "seconds":
		units = 1
	case "min", "minute", "minutes":
		units = 60
	case "h", "hour", "hours":
		units = 60 * 60
	case "d", "day", "days":
		units = 24 * 60 * 60
	case "w", "week", "weeks":
		units = 7 * 24 * 60 * 60
	case "mon", "month", "months":
		units = 30 * 24 * 60 * 60
	case "y", "year", "years":
		units = 365 * 24 * 60 * 60
	default:
		return 0, errors.New("unknown time units")
	}

	offset, err := strconv.Atoi(offsetStr)
	if err != nil {
		return 0, err
	}

	return int32(offset * units), nil
}

// TODO(dgryski): extract the http.Get + unproto code into its own function

func (z zipper) Find(metric string) (pb.GlobResponse, error) {

	u, _ := url.Parse(string(z) + "/metrics/find/")

	u.RawQuery = url.Values{
		"query":  []string{metric},
		"format": []string{"protobuf"},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Find: http.Get: %+v\n", err)
		return pb.GlobResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Find: ioutil.ReadAll: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	var pbresp pb.GlobResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Find: proto.Unmarshal: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	return pbresp, nil
}

func (z zipper) Render(metric string, from, until int32) (pb.FetchResponse, error) {

	u, _ := url.Parse(string(z) + "/render/")

	u.RawQuery = url.Values{
		"target": []string{metric},
		"format": []string{"protobuf"},
		"from":   []string{strconv.Itoa(int(from))},
		"until":  []string{strconv.Itoa(int(until))},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Render: http.Get: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Render: ioutil.ReadAll: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	var pbresp pb.FetchResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Render: proto.Unmarshal: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	return pbresp, nil
}

type limiter chan struct{}

func (l limiter) enter() { l <- struct{}{} }
func (l limiter) leave() { <-l }

var Limiter limiter

// for testing
var timeNow = time.Now

type graphitePoint struct {
	value float64
	t     int32
}

func (g graphitePoint) MarshalJSON() ([]byte, error) {
	// TODO(dgryski): fmt.Sprintf() is slow, use strconv.Append{Float,Int}
	// TODO(dgryski): MarshalJSON call should be on jsonResponse to reduce overhead
	if math.IsNaN(g.value) {
		return []byte(fmt.Sprintf("[null,%d]", g.t)), nil
	}
	return []byte(fmt.Sprintf("[%g,%d]", g.value, g.t)), nil
}

type jsonResponse struct {
	Target     string          `json:"target"`
	Datapoints []graphitePoint `json:"datapoints"`
}

func renderHandler(w http.ResponseWriter, r *http.Request) {

	Metrics.Requests.Add(1)

	err := r.ParseForm()
	if err != nil {
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}

	targets := r.Form["target"]
	from := r.FormValue("from")
	until := r.FormValue("until")
	useCache := r.FormValue("noCache") == ""

	// make sure the cache key doesn't say noCache, because it will never hit
	r.Form.Del("noCache")

	cacheKey := r.Form.Encode()

	if response, ok := queryCache.get(cacheKey); useCache && ok {
		Metrics.RequestCacheHits.Add(1)
		w.Header().Set("Content-Type", "application/json")
		w.Write(response)
		return
	}

	// normalize from and until values
	// BUG(dgryski): doesn't handle timezones the same as graphite-web
	from32 := dateParamToEpoch(from, timeNow().Add(-24*time.Hour).Unix())
	until32 := dateParamToEpoch(until, timeNow().Unix())

	var results []*pb.FetchResponse
	metricMap := make(map[metricRequest][]*pb.FetchResponse)

	for _, target := range targets {

		exp, e, err := parseExpr(target)
		if err != nil || e != "" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}

		for _, m := range exp.metrics() {

			mfetch := m
			mfetch.from += from32
			mfetch.until += until32

			if _, ok := metricMap[mfetch]; ok {
				// already fetched this metric for this request
				continue
			}

			var glob pb.GlobResponse
			var haveCacheData bool

			if response, ok := findCache.get(m.metric); useCache && ok {
				Metrics.FindCacheHits.Add(1)
				err := proto.Unmarshal(response, &glob)
				haveCacheData = err == nil
			}

			if !haveCacheData {
				var err error
				Metrics.FindRequests.Add(1)
				glob, err = Zipper.Find(m.metric)
				if err != nil {
					continue
				}
				b, err := proto.Marshal(&glob)
				if err == nil {
					findCache.set(m.metric, b)
				}
			}

			// For each metric returned in the Find response, query Render
			// This is a conscious decision to *not* cache render data
			rch := make(chan *pb.FetchResponse, len(glob.GetMatches()))
			leaves := 0
			for _, m := range glob.GetMatches() {
				if !m.GetIsLeaf() {
					continue
				}
				Metrics.RenderRequests.Add(1)
				leaves++
				Limiter.enter()
				go func(m *pb.GlobMatch, from, until int32) {
					var rptr *pb.FetchResponse
					r, err := Zipper.Render(m.GetPath(), from, until)
					if err == nil {
						rptr = &r
					}
					rch <- rptr
					Limiter.leave()
				}(m, mfetch.from, mfetch.until)
			}

			for i := 0; i < leaves; i++ {
				r := <-rch
				if r != nil {
					metricMap[mfetch] = append(metricMap[mfetch], r)
				}
			}
		}

		exprs := evalExpr(exp, from32, until32, metricMap)
		results = append(results, exprs...)
	}

	var jresults []jsonResponse

	for _, r := range results {
		if r == nil {
			log.Println("skipping nil result")
		}
		datapoints := make([]graphitePoint, 0, len(r.Values))
		t := *r.StartTime
		for i, v := range r.Values {
			if r.IsAbsent[i] {
				v = math.NaN()
			}
			datapoints = append(datapoints, graphitePoint{value: v, t: t})
			t += *r.StepTime
		}
		jresults = append(jresults, jsonResponse{Target: r.GetName(), Datapoints: datapoints})
	}

	jout, err := json.Marshal(jresults)
	if err != nil {
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	queryCache.set(cacheKey, jout)

	w.Header().Set("Content-Type", "application/json")
	w.Write(jout)
}

func lbcheckHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Ok\n")
}

func main() {

	z := flag.String("z", "", "zipper")
	port := flag.Int("p", 8080, "port")
	l := flag.Int("l", 20, "concurrency limit")
	mc := flag.String("mc", "", "comma separated memcached server list")
	cpus := flag.Int("cpus", 0, "number of CPUs to use")

	flag.Parse()

	if p := os.Getenv("PORT"); p != "" {
		*port, _ = strconv.Atoi(p)
	}

	Limiter = make(chan struct{}, *l)

	if *z == "" {
		log.Fatal("no zipper provided")
	}

	if _, err := url.Parse(*z); err != nil {
		log.Fatal("unable to parze zipper:", err)
	}

	log.Println("using zipper", *z)

	if *mc != "" {
		servers := strings.Split(*mc, ",")
		log.Println("using memcache servers:", servers)
		queryCache = &memcachedCache{client: memcache.New(servers...)}
		findCache = &memcachedCache{client: memcache.New(servers...)}
	} else {
		queryCache = &expireCache{cache: make(map[string]cacheElement)}
		go queryCache.(*expireCache).cleaner()

		findCache = &expireCache{cache: make(map[string]cacheElement)}
		go findCache.(*expireCache).cleaner()
	}

	Zipper = zipper(*z)

	if *cpus != 0 {
		log.Println("using GOMAXPROCS", *cpus)
		runtime.GOMAXPROCS(*cpus)
	}

	if host := os.Getenv("GRAPHITEHOST") + ":" + os.Getenv("GRAPHITEPORT"); host != ":" {

		log.Println("Using graphite host", host)

		// register our metrics with graphite
		graphite, err := g2g.NewGraphite(host, 60*time.Second, 10*time.Second)
		if err != nil {
			log.Fatal("unable to connect to to graphite: ", host, ":", err)
		}

		hostname, _ := os.Hostname()
		hostname = strings.Replace(hostname, ".", "_", -1)

		graphite.Register(fmt.Sprintf("carbon.api.%s.requests", hostname), Metrics.Requests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.request_cache_hits", hostname), Metrics.RequestCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.find_requests", hostname), Metrics.FindRequests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.find_cache_hits", hostname), Metrics.FindCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.render_requests", hostname), Metrics.RenderRequests)
	}

	http.HandleFunc("/render/", renderHandler)
	http.HandleFunc("/lbcheck", lbcheckHandler)

	log.Println("listening on port", *port)
	log.Fatalln(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"reflect"
	"time"

	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"

	"io/ioutil"
	"os"
	"strings"

	"github.com/fiunchinho/dmz-controller/repository"
	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/tools/clientcmd"
)

// DMZConfigMapName is an internal annotation used to store addreses whitelisted by this controller
const DMZConfigMapName = "dmz-controller"

var (
	namespace string
	// queue is a queue of resources to be processed.
	// It performs exponential backoff rate limiting, with a minimum retry period of 5 seconds and a maximum of 1 minute.
	queue = workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(time.Second*15, time.Minute))

	// stopCh can be used to stop all the informer, as well as control loops within the application.
	stopCh = make(chan struct{})

	// sharedFactory is a shared informer factory that is used as a cache for
	// items in the API server. It saves each informer listing and watching the
	// same resources independently of each other, thus providing more up to
	// date results with less 'effort'
	sharedFactory informers.SharedInformerFactory

	// client is a Kubernetes API client for our custom resource definition type
	client kubernetes.Interface
)

func getNamespace() string {
	if ns := os.Getenv("TILLER_NAMESPACE"); ns != "" {
		return ns
	}

	// Fall back to the namespace associated with the service account token, if available
	if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}

	return ""
}

func main() {
	// When running as a pod in-cluster, a kubeconfig is not needed. Instead this will make use of the service account injected into the pod.
	// However, allow the use of a local kubeconfig as this can make local development & testing easier.
	kubeconfig := flag.String("kubeconfig", "", "Path to a kubeconfig file")

	// We log to stderr because glog will default to logging to a file.
	// By setting this debugging is easier via `kubectl logs`
	flag.Set("logtostderr", "true")
	flag.Parse()

	namespace = getNamespace()
	if namespace == "" {
		glog.Fatalf("The NAMESPACE environment variable is not set, and the file /var/run/secrets/kubernetes.io/serviceaccount/namespace can't be read")
	}

	// Build the client config - optionally using a provided kubeconfig file.
	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		glog.Fatalf("Failed to load client config: %v", err)
	}

	// Construct the Kubernetes client
	client, err = kubernetes.NewForConfig(config)
	if err != nil {
		log.Fatalf("Error creating kubernetes client: %s", err.Error())
	}

	log.Printf("Created Kubernetes client.")

	// we use a shared informer from the informer factory, to save calls to the API as we grow our application
	// and so state is consistent between our control loops.
	// We set a resync period of 30 seconds, in case any create/replace/update/delete operations are missed when watching
	sharedFactory = informers.NewSharedInformerFactory(client, time.Second*30)
	informer := sharedFactory.Extensions().V1beta1().Ingresses().Informer()
	cmInformer := sharedFactory.Core().V1().ConfigMaps().Informer()

	// we add a new event handler, watching for changes to API resources.
	informer.AddEventHandler(
		cache.ResourceEventHandlerFuncs{
			AddFunc: enqueue,
			UpdateFunc: func(old, cur interface{}) {
				if !reflect.DeepEqual(old, cur) {
					enqueue(cur)
				}
			},
		},
	)
	cmInformer.AddEventHandler(
		cache.ResourceEventHandlerFuncs{
			UpdateFunc: func(old, cur interface{}) {
				if cur.(*v1.ConfigMap).Name == DMZConfigMapName {
					if !reflect.DeepEqual(old, cur) {
						ingresses, err := sharedFactory.Extensions().V1beta1().Ingresses().Lister().Ingresses("default").List(labels.Everything())
						if err != nil {
							log.Fatalf("error listing ingresses to notify change on configmap: %s", err.Error())
						}
						for _, ingress := range ingresses {
							log.Printf("Queuing ingress %s object so it gets notified of the ConfigMap change", ingress.Name)
							enqueue(ingress)
						}
					}
				}
			},
		},
	)

	// start the informer. This will cause it to begin receiving updates from the configured API server and firing event handlers in response.
	sharedFactory.Start(stopCh)
	log.Printf("Started informer factory.")

	// wait for the informer cache to finish performing it's initial applyWhiteList of resources
	if !cache.WaitForCacheSync(stopCh, cmInformer.HasSynced, informer.HasSynced) {
		log.Fatalf("error waiting for informer cache to applyWhiteList: %s", err.Error())
	}

	ingressWhitelister := IngressWhitelister{
		namespace:           namespace,
		ingressRepository:   repository.NewIngressRepository(client, sharedFactory),
		configMapRepository: repository.NewConfigMapRepository(client, sharedFactory),
	}

	log.Printf("Finished populating shared informers cache.")

	// here we start reading objects off the queue
	for {
		// we read a message off the queue
		key, shutdown := queue.Get()

		// if the queue has been shut down, we should exit the work queue here
		if shutdown {
			stopCh <- struct{}{}
			return
		}

		// convert the queue item into a string. If it's not a string, we'll simply discard it as invalid data and log a message.
		var strKey string
		var ok bool
		if strKey, ok = key.(string); !ok {
			runtime.HandleError(fmt.Errorf("key in queue should be of type string but got %T. discarding", key))
			return
		}

		// we define a function here to process a queue item, so that we can use 'defer' to make sure the message is marked as Done on the queue
		// Done marks item as done processing, and if it has been marked as dirty again while it was being processed,
		// it will be re-added to the queue for re-processing.
		func(key string) {
			defer queue.Done(key)

			// attempt to split the 'key' into namespace and object name
			_, name, err := cache.SplitMetaNamespaceKey(strKey)
			if err != nil {
				runtime.HandleError(fmt.Errorf("error splitting meta namespace key into parts: %s", err.Error()))
				return
			}

			log.Printf("Read key '%s/%s' off workqueue. Fetching from cache...", namespace, name)

			err = ingressWhitelister.Whitelist(name)

			if err != nil {
				runtime.HandleError(fmt.Errorf("error getting object '%s/%s' from api: %s", namespace, name, err.Error()))
				return
			}

			// as we managed to process this successfully, we can forget it
			// from the work queue altogether.
			// Forget indicates that an item is finished being retried.  Doesn't matter whether its for perm failing
			// or for success, we'll stop the rate limiter from tracking it.  This only clears the `rateLimiter`, you
			// still have to call `Done` on the queue.
			log.Printf("Finished processing '%s/%s' successfully! Removing from queue.", namespace, name)
			queue.Forget(key)
		}(strKey)
	}
}

// enqueue will add an object 'obj' into the workqueue. The object being added
// must be of type metav1.Object, metav1.ObjectAccessor or cache.ExplicitKey.
func enqueue(obj interface{}) {
	// DeletionHandlingMetaNamespaceKeyFunc will convert an object into a
	// 'namespace/name' string. We do this because our item may be processed
	// much later than now, and so we want to ensure it gets a fresh copy of
	// the resource when it starts. Also, this allows us to keep adding the
	// same item into the work queue without duplicates building up.
	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
	if err != nil {
		runtime.HandleError(fmt.Errorf("error obtaining key for object being enqueue: %s", err.Error()))
		return
	}
	// add the item to the queue
	queue.Add(key)
}
<commit_msg>Fix environment variable name<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"reflect"
	"time"

	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"

	"io/ioutil"
	"os"
	"strings"

	"github.com/fiunchinho/dmz-controller/repository"
	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/tools/clientcmd"
)

// DMZConfigMapName is an internal annotation used to store addreses whitelisted by this controller
const DMZConfigMapName = "dmz-controller"

var (
	namespace string
	// queue is a queue of resources to be processed.
	// It performs exponential backoff rate limiting, with a minimum retry period of 5 seconds and a maximum of 1 minute.
	queue = workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(time.Second*15, time.Minute))

	// stopCh can be used to stop all the informer, as well as control loops within the application.
	stopCh = make(chan struct{})

	// sharedFactory is a shared informer factory that is used as a cache for
	// items in the API server. It saves each informer listing and watching the
	// same resources independently of each other, thus providing more up to
	// date results with less 'effort'
	sharedFactory informers.SharedInformerFactory

	// client is a Kubernetes API client for our custom resource definition type
	client kubernetes.Interface
)

func getNamespace() string {
	if ns := os.Getenv("NAMESPACE"); ns != "" {
		return ns
	}

	// Fall back to the namespace associated with the service account token, if available
	if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}

	return ""
}

func main() {
	// When running as a pod in-cluster, a kubeconfig is not needed. Instead this will make use of the service account injected into the pod.
	// However, allow the use of a local kubeconfig as this can make local development & testing easier.
	kubeconfig := flag.String("kubeconfig", "", "Path to a kubeconfig file")

	// We log to stderr because glog will default to logging to a file.
	// By setting this debugging is easier via `kubectl logs`
	flag.Set("logtostderr", "true")
	flag.Parse()

	namespace = getNamespace()
	if namespace == "" {
		glog.Fatalf("The NAMESPACE environment variable is not set, and the file /var/run/secrets/kubernetes.io/serviceaccount/namespace can't be read")
	}

	// Build the client config - optionally using a provided kubeconfig file.
	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		glog.Fatalf("Failed to load client config: %v", err)
	}

	// Construct the Kubernetes client
	client, err = kubernetes.NewForConfig(config)
	if err != nil {
		log.Fatalf("Error creating kubernetes client: %s", err.Error())
	}

	log.Printf("Created Kubernetes client.")

	// we use a shared informer from the informer factory, to save calls to the API as we grow our application
	// and so state is consistent between our control loops.
	// We set a resync period of 30 seconds, in case any create/replace/update/delete operations are missed when watching
	sharedFactory = informers.NewSharedInformerFactory(client, time.Second*30)
	informer := sharedFactory.Extensions().V1beta1().Ingresses().Informer()
	cmInformer := sharedFactory.Core().V1().ConfigMaps().Informer()

	// we add a new event handler, watching for changes to API resources.
	informer.AddEventHandler(
		cache.ResourceEventHandlerFuncs{
			AddFunc: enqueue,
			UpdateFunc: func(old, cur interface{}) {
				if !reflect.DeepEqual(old, cur) {
					enqueue(cur)
				}
			},
		},
	)
	cmInformer.AddEventHandler(
		cache.ResourceEventHandlerFuncs{
			UpdateFunc: func(old, cur interface{}) {
				if cur.(*v1.ConfigMap).Name == DMZConfigMapName {
					if !reflect.DeepEqual(old, cur) {
						ingresses, err := sharedFactory.Extensions().V1beta1().Ingresses().Lister().Ingresses("default").List(labels.Everything())
						if err != nil {
							log.Fatalf("error listing ingresses to notify change on configmap: %s", err.Error())
						}
						for _, ingress := range ingresses {
							log.Printf("Queuing ingress %s object so it gets notified of the ConfigMap change", ingress.Name)
							enqueue(ingress)
						}
					}
				}
			},
		},
	)

	// start the informer. This will cause it to begin receiving updates from the configured API server and firing event handlers in response.
	sharedFactory.Start(stopCh)
	log.Printf("Started informer factory.")

	// wait for the informer cache to finish performing it's initial applyWhiteList of resources
	if !cache.WaitForCacheSync(stopCh, cmInformer.HasSynced, informer.HasSynced) {
		log.Fatalf("error waiting for informer cache to applyWhiteList: %s", err.Error())
	}

	ingressWhitelister := IngressWhitelister{
		namespace:           namespace,
		ingressRepository:   repository.NewIngressRepository(client, sharedFactory),
		configMapRepository: repository.NewConfigMapRepository(client, sharedFactory),
	}

	log.Printf("Finished populating shared informers cache.")

	// here we start reading objects off the queue
	for {
		// we read a message off the queue
		key, shutdown := queue.Get()

		// if the queue has been shut down, we should exit the work queue here
		if shutdown {
			stopCh <- struct{}{}
			return
		}

		// convert the queue item into a string. If it's not a string, we'll simply discard it as invalid data and log a message.
		var strKey string
		var ok bool
		if strKey, ok = key.(string); !ok {
			runtime.HandleError(fmt.Errorf("key in queue should be of type string but got %T. discarding", key))
			return
		}

		// we define a function here to process a queue item, so that we can use 'defer' to make sure the message is marked as Done on the queue
		// Done marks item as done processing, and if it has been marked as dirty again while it was being processed,
		// it will be re-added to the queue for re-processing.
		func(key string) {
			defer queue.Done(key)

			// attempt to split the 'key' into namespace and object name
			_, name, err := cache.SplitMetaNamespaceKey(strKey)
			if err != nil {
				runtime.HandleError(fmt.Errorf("error splitting meta namespace key into parts: %s", err.Error()))
				return
			}

			log.Printf("Read key '%s/%s' off workqueue. Fetching from cache...", namespace, name)

			err = ingressWhitelister.Whitelist(name)

			if err != nil {
				runtime.HandleError(fmt.Errorf("error getting object '%s/%s' from api: %s", namespace, name, err.Error()))
				return
			}

			// as we managed to process this successfully, we can forget it
			// from the work queue altogether.
			// Forget indicates that an item is finished being retried.  Doesn't matter whether its for perm failing
			// or for success, we'll stop the rate limiter from tracking it.  This only clears the `rateLimiter`, you
			// still have to call `Done` on the queue.
			log.Printf("Finished processing '%s/%s' successfully! Removing from queue.", namespace, name)
			queue.Forget(key)
		}(strKey)
	}
}

// enqueue will add an object 'obj' into the workqueue. The object being added
// must be of type metav1.Object, metav1.ObjectAccessor or cache.ExplicitKey.
func enqueue(obj interface{}) {
	// DeletionHandlingMetaNamespaceKeyFunc will convert an object into a
	// 'namespace/name' string. We do this because our item may be processed
	// much later than now, and so we want to ensure it gets a fresh copy of
	// the resource when it starts. Also, this allows us to keep adding the
	// same item into the work queue without duplicates building up.
	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
	if err != nil {
		runtime.HandleError(fmt.Errorf("error obtaining key for object being enqueue: %s", err.Error()))
		return
	}
	// add the item to the queue
	queue.Add(key)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/auth0/go-jwt-middleware"
	"github.com/codegangsta/negroni"
	"github.com/dgrijalva/jwt-go"
	"github.com/gorilla/mux"
	"github.com/joho/godotenv"
	"log"
	"net/http"
	"os"
)

func main() {

	err := godotenv.Load()
	if err != nil {
		log.Fatal("Error loading .env file")
	}

	StartServer()

}

func StartServer() {
	r := mux.NewRouter()

        jwtMiddleware := jwtmiddleware.New(jwtmiddleware.Options{
          ValidationKeyGetter: func(token *jwt.Token) (interface{}, error) {
            secret := os.Getenv("AUTH0_CLIENT_SECRET")
            if secret == "" {
              log.Fatal("AUTH0_CLIENT_SECRET is not set")
            }
            return secret, nil
          },
        })

	r.HandleFunc("/ping", PingHandler)
	r.Handle("/secured/ping", negroni.New(
		negroni.HandlerFunc(jwtMiddleware.HandlerWithNext),
		negroni.Wrap(http.HandlerFunc(SecuredPingHandler)),
	))
	http.Handle("/", r)
	http.ListenAndServe(":3001", nil)
}

type Response struct {
	Text string `json:"text"`
}

func respondJson(text string, w http.ResponseWriter) {
	response := Response{text}

	jsonResponse, err := json.Marshal(response)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(jsonResponse)
}

func PingHandler(w http.ResponseWriter, r *http.Request) {
	respondJson("All good. You don't need to be authenticated to call this", w)
}

func SecuredPingHandler(w http.ResponseWriter, r *http.Request) {
	respondJson("All good. You only get this message if you're authenticated", w)
}
<commit_msg>Fix secret<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"os"

	jwtmiddleware "github.com/auth0/go-jwt-middleware"
	"github.com/codegangsta/negroni"
	jwt "github.com/dgrijalva/jwt-go"
	"github.com/gorilla/mux"
	"github.com/joho/godotenv"
)

func main() {

	err := godotenv.Load()
	if err != nil {
		log.Fatal("Error loading .env file")
	}

	StartServer()

}

func StartServer() {
	r := mux.NewRouter()

	jwtMiddleware := jwtmiddleware.New(jwtmiddleware.Options{
		ValidationKeyGetter: func(token *jwt.Token) (interface{}, error) {
			secret := []byte(os.Getenv("AUTH0_CLIENT_SECRET"))

			if len(secret) == 0 {
				log.Fatal("AUTH0_CLIENT_SECRET is not set")
			}

			return secret, nil
		},
	})

	r.HandleFunc("/ping", PingHandler)
	r.Handle("/secured/ping", negroni.New(
		negroni.HandlerFunc(jwtMiddleware.HandlerWithNext),
		negroni.Wrap(http.HandlerFunc(SecuredPingHandler)),
	))
	http.Handle("/", r)
	http.ListenAndServe(":3001", nil)
}

type Response struct {
	Text string `json:"text"`
}

func respondJson(text string, w http.ResponseWriter) {
	response := Response{text}

	jsonResponse, err := json.Marshal(response)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(jsonResponse)
}

func PingHandler(w http.ResponseWriter, r *http.Request) {
	respondJson("All good. You don't need to be authenticated to call this", w)
}

func SecuredPingHandler(w http.ResponseWriter, r *http.Request) {
	respondJson("All good. You only get this message if you're authenticated", w)
}
<|endoftext|>


<commit_before>package ec2

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/coreos/coreos-cloudinit/datasource/metadata"
	"github.com/coreos/coreos-cloudinit/pkg"
)

const (
	DefaultAddress = "http://169.254.169.254/"
	apiVersion     = "2009-04-04"
	userdataPath   = apiVersion + "/user-data"
	metadataPath   = apiVersion + "/meta-data"
)

type metadataService struct {
	metadata.MetadataService
}

func NewDatasource(root string) *metadataService {
	return &metadataService{metadata.NewDatasource(root, apiVersion, userdataPath, metadataPath)}
}

func (ms metadataService) FetchMetadata() ([]byte, error) {
	attrs := make(map[string]interface{})
	if keynames, err := ms.fetchAttributes(fmt.Sprintf("%s/public-keys", ms.MetadataUrl())); err == nil {
		keyIDs := make(map[string]string)
		for _, keyname := range keynames {
			tokens := strings.SplitN(keyname, "=", 2)
			if len(tokens) != 2 {
				return nil, fmt.Errorf("malformed public key: %q", keyname)
			}
			keyIDs[tokens[1]] = tokens[0]
		}

		keys := make(map[string]string)
		for name, id := range keyIDs {
			sshkey, err := ms.fetchAttribute(fmt.Sprintf("%s/public-keys/%s/openssh-key", ms.MetadataUrl(), id))
			if err != nil {
				return nil, err
			}
			keys[name] = sshkey
			fmt.Printf("Found SSH key for %q\n", name)
		}
		attrs["public_keys"] = keys
	} else if _, ok := err.(pkg.ErrNotFound); !ok {
		return nil, err
	}

	if hostname, err := ms.fetchAttribute(fmt.Sprintf("%s/hostname", ms.MetadataUrl())); err == nil {
		attrs["hostname"] = hostname
	} else if _, ok := err.(pkg.ErrNotFound); !ok {
		return nil, err
	}

	if localAddr, err := ms.fetchAttribute(fmt.Sprintf("%s/local-ipv4", ms.MetadataUrl())); err == nil {
		attrs["local-ipv4"] = localAddr
	} else if _, ok := err.(pkg.ErrNotFound); !ok {
		return nil, err
	}

	if publicAddr, err := ms.fetchAttribute(fmt.Sprintf("%s/public-ipv4", ms.MetadataUrl())); err == nil {
		attrs["public-ipv4"] = publicAddr
	} else if _, ok := err.(pkg.ErrNotFound); !ok {
		return nil, err
	}

	if content_path, err := ms.fetchAttribute(fmt.Sprintf("%s/network_config/content_path", ms.MetadataUrl())); err == nil {
		attrs["network_config"] = map[string]string{
			"content_path": content_path,
		}
	} else if _, ok := err.(pkg.ErrNotFound); !ok {
		return nil, err
	}

	return json.Marshal(attrs)
}

func (ms metadataService) Type() string {
	return "ec2-metadata-service"
}

func (ms metadataService) fetchAttributes(url string) ([]string, error) {
	resp, err := ms.FetchData(url)
	if err != nil {
		return nil, err
	}
	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
	data := make([]string, 0)
	for scanner.Scan() {
		data = append(data, scanner.Text())
	}
	return data, scanner.Err()
}

func (ms metadataService) fetchAttribute(url string) (string, error) {
	if attrs, err := ms.fetchAttributes(url); err == nil && len(attrs) > 0 {
		return attrs[0], nil
	} else {
		return "", err
	}
}
<commit_msg>AWS: Eucalyptus 4.x compatibility fix<commit_after>package ec2

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/coreos/coreos-cloudinit/datasource/metadata"
	"github.com/coreos/coreos-cloudinit/pkg"
)

const (
	DefaultAddress = "http://169.254.169.254/"
	apiVersion     = "2009-04-04/"
	userdataPath   = apiVersion + "user-data"
	metadataPath   = apiVersion + "meta-data"
)

type metadataService struct {
	metadata.MetadataService
}

func NewDatasource(root string) *metadataService {
	return &metadataService{metadata.NewDatasource(root, apiVersion, userdataPath, metadataPath)}
}

func (ms metadataService) FetchMetadata() ([]byte, error) {
	attrs := make(map[string]interface{})
	if keynames, err := ms.fetchAttributes(fmt.Sprintf("%s/public-keys", ms.MetadataUrl())); err == nil {
		keyIDs := make(map[string]string)
		for _, keyname := range keynames {
			tokens := strings.SplitN(keyname, "=", 2)
			if len(tokens) != 2 {
				return nil, fmt.Errorf("malformed public key: %q", keyname)
			}
			keyIDs[tokens[1]] = tokens[0]
		}

		keys := make(map[string]string)
		for name, id := range keyIDs {
			sshkey, err := ms.fetchAttribute(fmt.Sprintf("%s/public-keys/%s/openssh-key", ms.MetadataUrl(), id))
			if err != nil {
				return nil, err
			}
			keys[name] = sshkey
			fmt.Printf("Found SSH key for %q\n", name)
		}
		attrs["public_keys"] = keys
	} else if _, ok := err.(pkg.ErrNotFound); !ok {
		return nil, err
	}

	if hostname, err := ms.fetchAttribute(fmt.Sprintf("%s/hostname", ms.MetadataUrl())); err == nil {
		attrs["hostname"] = hostname
	} else if _, ok := err.(pkg.ErrNotFound); !ok {
		return nil, err
	}

	if localAddr, err := ms.fetchAttribute(fmt.Sprintf("%s/local-ipv4", ms.MetadataUrl())); err == nil {
		attrs["local-ipv4"] = localAddr
	} else if _, ok := err.(pkg.ErrNotFound); !ok {
		return nil, err
	}

	if publicAddr, err := ms.fetchAttribute(fmt.Sprintf("%s/public-ipv4", ms.MetadataUrl())); err == nil {
		attrs["public-ipv4"] = publicAddr
	} else if _, ok := err.(pkg.ErrNotFound); !ok {
		return nil, err
	}

	if content_path, err := ms.fetchAttribute(fmt.Sprintf("%s/network_config/content_path", ms.MetadataUrl())); err == nil {
		attrs["network_config"] = map[string]string{
			"content_path": content_path,
		}
	} else if _, ok := err.(pkg.ErrNotFound); !ok {
		return nil, err
	}

	return json.Marshal(attrs)
}

func (ms metadataService) Type() string {
	return "ec2-metadata-service"
}

func (ms metadataService) fetchAttributes(url string) ([]string, error) {
	resp, err := ms.FetchData(url)
	if err != nil {
		return nil, err
	}
	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
	data := make([]string, 0)
	for scanner.Scan() {
		data = append(data, scanner.Text())
	}
	return data, scanner.Err()
}

func (ms metadataService) fetchAttribute(url string) (string, error) {
	if attrs, err := ms.fetchAttributes(url); err == nil && len(attrs) > 0 {
		return attrs[0], nil
	} else {
		return "", err
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	docker "github.com/fsouza/go-dockerclient"
	"net"
	"time"
)

func LaunchContainer(name string, service Service, container chan<- *docker.Container, quit <-chan bool, finished chan<- bool) {
	dcli, err := docker.NewClient("tcp://127.0.0.1:2375")
	if err != nil {
		container <- nil
		return
	}

	c, err := dcli.CreateContainer(docker.CreateContainerOptions{
		"",
		&docker.Config{
			Image:     service.Image,
			PortSpecs: []string{service.Port + "/tcp"},
		},
		nil,
	})
	if err != nil {
		fmt.Printf("[%s] %s\n", name, err.Error())
		container <- nil
		return
	}

	if err := dcli.StartContainer(c.ID, &docker.HostConfig{}); err != nil {
		fmt.Printf("[%s] %s\n", name, err.Error())
		container <- nil
		return
	}

	c, err = dcli.InspectContainer(c.ID)
	if err != nil {
		fmt.Printf("[%s] %s\n", name, err.Error())
		container <- nil
		return
	}

	if cmds, ok := service.Hooks["pre-run"]; ok {
		for range cmds {
			// TODO: exec pre-run hook
		}
	}

	// Wait for port listen
	for x := 0; ; x++ {
		addr := fmt.Sprintf("%s:%s", c.NetworkSettings.IPAddress, firstPort(c.NetworkSettings.Ports))
		if conn, err := net.Dial("tcp", addr); err == nil {
			conn.Close()
			break
		}
		time.Sleep(1 * time.Second)

		if x == 10 {
			fmt.Printf("[%s] max connection retries exceeded\n", name)
			container <- nil
			return
		}
	}

	container <- c

	// Waiting for quit signal
	<-quit

	if cmds, ok := service.Hooks["post-run"]; ok {
		for range cmds {
			// TODO: exec post-run hook
		}
	}

	dcli.StopContainer(c.ID, 10)
	dcli.RemoveContainer(docker.RemoveContainerOptions{ID: c.ID})

	finished <- true
}

func Exec(cID string, cfg docker.ExecProcessConfig) error {
	return nil
}

func firstPort(ports map[docker.Port][]docker.PortBinding) string {
	for p := range ports {
		return p.Port()
	}
	return ""
}
<commit_msg>docker: implemented pre-run and post-run dependency hooks<commit_after>package main

import (
	"fmt"
	"net"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/flynn/go-shlex"
)

func LaunchContainer(name string, service Service, container chan<- *docker.Container, quit <-chan bool, finished chan<- bool) {
	dcli, err := docker.NewClient("tcp://127.0.0.1:2375")
	if err != nil {
		container <- nil
		return
	}

	c, err := dcli.CreateContainer(docker.CreateContainerOptions{
		"",
		&docker.Config{
			Image:     service.Image,
			PortSpecs: []string{service.Port + "/tcp"},
		},
		nil,
	})
	if err != nil {
		fmt.Printf("[%s] %s\n", name, err.Error())
		container <- nil
		return
	}

	if err := dcli.StartContainer(c.ID, &docker.HostConfig{}); err != nil {
		fmt.Printf("[%s] %s\n", name, err.Error())
		container <- nil
		return
	}

	c, err = dcli.InspectContainer(c.ID)
	if err != nil {
		fmt.Printf("[%s] %s\n", name, err.Error())
		container <- nil
		return
	}

	if cmds, ok := service.Hooks["pre-run"]; ok {
		for _, cmd := range cmds {
			if err := Exec(dcli, c.ID, cmd); err != nil {
				fmt.Printf("[%s] pre-run: %s\n", name, err.Error())
				container <- nil
				return
			}
		}
	}

	// Wait for port listen
	for x := 0; ; x++ {
		addr := fmt.Sprintf("%s:%s", c.NetworkSettings.IPAddress, firstPort(c.NetworkSettings.Ports))
		if conn, err := net.Dial("tcp", addr); err == nil {
			conn.Close()
			break
		}
		time.Sleep(1 * time.Second)

		if x == 10 {
			fmt.Printf("[%s] max connection retries exceeded\n", name)
			container <- nil
			return
		}
	}

	container <- c

	// Waiting for quit signal
	<-quit

	if cmds, ok := service.Hooks["post-run"]; ok {
		for _, cmd := range cmds {
			if err := Exec(dcli, c.ID, cmd); err != nil {
				fmt.Printf("[%s] post-run: %s\n", name, err.Error())
				container <- nil
				return
			}
		}
	}

	dcli.StopContainer(c.ID, 10)
	dcli.RemoveContainer(docker.RemoveContainerOptions{ID: c.ID})

	finished <- true
}

func Exec(dcli *docker.Client, cID, cmd string) error {
	cmdsplt, err := shlex.Split(cmd)
	if err != nil {
		return err
	}
	exec, err := dcli.CreateExec(docker.CreateExecOptions{
		Cmd:       cmdsplt,
		Container: cID,
	})
	if err != nil {
		return err
	}
	return dcli.StartExec(exec.ID, docker.StartExecOptions{})
}

func firstPort(ports map[docker.Port][]docker.PortBinding) string {
	for p := range ports {
		return p.Port()
	}
	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/fsouza/go-dockerclient"
	"log"
	"time"
)

const (
	Endpoint = "unix:///var/run/docker.sock"
)

type DockerClient struct {
	client *docker.Client
}

func NewDocker() DockerClient {
	client, _ := docker.NewClient(Endpoint)
	return DockerClient{client: client}
}

type ContainerInfo struct {
	ID      string
	Name    string
	Image   string
	Created time.Time
	Config  *docker.Config
}

func (d *DockerClient) buildContainerInfo(container *docker.Container) ContainerInfo {
	return ContainerInfo{
		ID:      container.ID,
		Name:    container.Name,
		Image:   container.Image,
		Created: container.Created,
		Config:  container.Config,
	}
}

func main() {
	log.Print("Starting Pencil ... \n\n")
	client := NewDocker()
	s := client.getRunningContainers()
	log.Print(s)
}

// getRunningContainers finds running containers and returns specific details.
func (d *DockerClient) getRunningContainers() []ContainerInfo {
	running_containers_ids := d.getContainersIDs()
	a := d.getContainersDetails(running_containers_ids)
	return a
}

// getContainersIDs retruns a list of running docker contianers.
func (d *DockerClient) getContainersIDs() []docker.APIContainers {
	options := docker.ListContainersOptions{}
	containers, _ := d.client.ListContainers(options)
	return containers
}

// getContainersDetails iterate over a list of containers and returns a list of ContainerInfo struct.
func (d *DockerClient) getContainersDetails(containers []docker.APIContainers) []ContainerInfo {
	list := []ContainerInfo{}
	for _, c := range containers {
		list = append(list, d.inspectContainer(c.ID))
	}
	return list
}

// inspectContainer extract container info for a continer ID.
func (d *DockerClient) inspectContainer(cid string) ContainerInfo {
	data, _ := d.client.InspectContainer(cid)
	return d.buildContainerInfo(data)
}
<commit_msg>better var name<commit_after>package main

import (
	"github.com/fsouza/go-dockerclient"
	"log"
	"time"
)

const (
	Endpoint = "unix:///var/run/docker.sock"
)

type DockerClient struct {
	client *docker.Client
}

func NewDocker() DockerClient {
	client, _ := docker.NewClient(Endpoint)
	return DockerClient{client: client}
}

type ContainerInfo struct {
	ID      string
	Name    string
	Image   string
	Created time.Time
	Config  *docker.Config
}

func (d *DockerClient) buildContainerInfo(container *docker.Container) ContainerInfo {
	return ContainerInfo{
		ID:      container.ID,
		Name:    container.Name,
		Image:   container.Image,
		Created: container.Created,
		Config:  container.Config,
	}
}

func main() {
	log.Print("Starting Pencil ... \n\n")
	client := NewDocker()
	s := client.getRunningContainers()
	log.Print(s)
}

// getRunningContainers finds running containers and returns specific details.
func (d *DockerClient) getRunningContainers() []ContainerInfo {
	containersIDs := d.getContainersIDs()
	containersDetails := d.getContainersDetails(containersIDs)
	log.Print("Running containers count: ", len(containersDetails), "\n\n")
	return containersDetails
}

// getContainersIDs retruns a list of running docker contianers.
func (d *DockerClient) getContainersIDs() []docker.APIContainers {
	options := docker.ListContainersOptions{}
	containers, _ := d.client.ListContainers(options)
	return containers
}

// getContainersDetails iterate over a list of containers and returns a list of ContainerInfo struct.
func (d *DockerClient) getContainersDetails(containers []docker.APIContainers) []ContainerInfo {
	list := []ContainerInfo{}
	for _, c := range containers {
		list = append(list, d.inspectContainer(c.ID))
	}
	return list
}

// inspectContainer extract container info for a continer ID.
func (d *DockerClient) inspectContainer(cid string) ContainerInfo {
	data, _ := d.client.InspectContainer(cid)
	return d.buildContainerInfo(data)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
)

type Book struct {
	command     string
	environment string
	layout      map[string]Layout
	status      int
}

type Recipe struct {
	attributes map[string]interface{}
	source     string
}

func (r *Recipe) render() string {
	return r.source
}

type Layout struct {
	Attributes map[string]interface{} `json:"attributes"`
	Servers    map[string][]string    `json:"servers"`
}

func (b *Book) ParseLayout() {
	input, err := ioutil.ReadFile(path.Join(".", "layout.json"))

	if err != nil {
		fmt.Println(err)
	}

	json.Unmarshal(input, &b.layout)
}

func (b *Book) exec(server, recipe string) (string, int) {
	template_path := path.Join(".", "recipe", recipe, b.command)

	if _, err := os.Stat(template_path); os.IsNotExist(err) {
		fmt.Print("\033[01;33mMISSING\033[00m ")
		return "unable to locate: " + template_path, 1
	}

	/*
		source, err := ioutil.ReadFile(template_path)

		if err != nil {
			return "unable to read file: " + template_path, 1
		}
	*/
	// template := &Recipe{source: string(source), attributes: attributes}

	// out, status := ssh(server, template.render())
	out, status := ssh(server, template_path)

	return out, status
}

func ssh(server, script string) (string, int) {
	cmd := exec.Command(script)
	output, err := cmd.CombinedOutput()

	if err != nil {
		fmt.Print(err)
		return string(output), 1
	}

	// out, status = Open3.capture2e("ssh -T -F #{path("ssh_config")} #{server}", :stdin_data => script)
	return string(output), 0
}

func (b *Book) run() {
	b.ParseLayout()

	for server := range b.layout[b.environment].Servers {
		recipes := b.layout[b.environment].Servers[server]

		fmt.Println(server)

		for _, recipe := range recipes {
			fmt.Printf("  %s: ", recipe)

			stdout, status := b.exec(server, recipe)

			if status != 0 && len(stdout) > 0 {
				fmt.Fprintf(os.Stderr, " %s\n", stdout)
			}

			if status != 0 {
				fmt.Print("\033[01;31mERROR\033[00m\n")
				b.status = 1
				break
			}

			fmt.Print("\033[01;32mOK\033[00m\n")
		}
	}
}

func main() {
	book := &Book{command: os.Args[1], environment: os.Args[2], status: 0}

	book.run()

	os.Exit(book.status)
}
<commit_msg>errors on bad json<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
)

type Book struct {
	command     string
	environment string
	layout      map[string]Layout
	status      int
}

type Recipe struct {
	attributes map[string]interface{}
	source     string
}

func (r *Recipe) render() string {
	return r.source
}

type Layout struct {
	Attributes map[string]interface{} `json:"attributes"`
	Servers    map[string][]string    `json:"servers"`
}

func (b *Book) ParseLayout() {
	input, err := ioutil.ReadFile(path.Join(".", "layout.json"))

	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(input, &b.layout)

	if err != nil {
		fmt.Println(err)
	}
}

func (b *Book) exec(server, recipe string) (string, int) {
	template_path := path.Join(".", "recipe", recipe, b.command)

	if _, err := os.Stat(template_path); os.IsNotExist(err) {
		fmt.Print("\033[01;33mMISSING\033[00m ")
		return "unable to locate: " + template_path, 1
	}

	/*
		source, err := ioutil.ReadFile(template_path)

		if err != nil {
			return "unable to read file: " + template_path, 1
		}
	*/
	// template := &Recipe{source: string(source), attributes: attributes}

	// out, status := ssh(server, template.render())
	out, status := ssh(server, template_path)

	return out, status
}

func ssh(server, script string) (string, int) {
	cmd := exec.Command(script)
	output, err := cmd.CombinedOutput()

	if err != nil {
		fmt.Print(err)
		return string(output), 1
	}

	// out, status = Open3.capture2e("ssh -T -F #{path("ssh_config")} #{server}", :stdin_data => script)
	return string(output), 0
}

func (b *Book) run() {
	b.ParseLayout()

	for server := range b.layout[b.environment].Servers {
		recipes := b.layout[b.environment].Servers[server]

		fmt.Println(server)

		for _, recipe := range recipes {
			fmt.Printf("  %s: ", recipe)

			stdout, status := b.exec(server, recipe)

			if status != 0 && len(stdout) > 0 {
				fmt.Fprintf(os.Stderr, " %s\n", stdout)
			}

			if status != 0 {
				fmt.Print("\033[01;31mERROR\033[00m\n")
				b.status = 1
				break
			}

			fmt.Print("\033[01;32mOK\033[00m\n")
		}
	}
}

func main() {
	book := &Book{command: os.Args[1], environment: os.Args[2], status: 0}

	book.run()

	os.Exit(book.status)
}
<|endoftext|>