<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.
package parse

import (
	// "strings"
	// "unicode"
	"fmt"
	"github.com/demizer/go-elog"
	"unicode/utf8"
)

type itemElement int

const (
	itemEOF itemElement = iota
	itemError
	itemTitle
	itemSectionAdornment
	itemParagraph
)

var sectionAdornments = []rune{'!', '"', '#', '$', '\'', '%', '&', '(', ')', '*',
	'+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\',
	']', '^', '_', '`', '{', '|', '}', '~'}

func isSectionAdornment(r rune) bool {
	for _, a := range sectionAdornments {
		if a == r {
			return true
		}
	}
	return false
}

const eof = -1

type stateFn func(*lexer) stateFn

type item struct {
	typ itemElement
	pos Pos
	val string
}

func (i item) String() string {
	switch {
	case i.typ == itemEOF:
		return "EOF"
	case i.typ == itemError:
		return i.val
	}
	return fmt.Sprintf("%q", i.val)
}

type lexer struct {
	name    string
	input   string
	state   stateFn
	pos     Pos
	start   Pos
	width   Pos
	lastPos Pos
	items   chan item
}

func lex(name, input string) *lexer {
	l := &lexer{
		name:  name,
		input: input,
		items: make(chan item),
	}
	go l.run()
	return l
}

// emit passes an item back to the client.
func (l *lexer) emit(t itemElement) {
	l.items <- item{t, l.start, l.input[l.start:l.pos]}
	l.start = l.pos

}

func (l *lexer) backup() {
	l.pos -= l.width
}

func (l *lexer) current() rune {
	r, _ := utf8.DecodeRuneInString(l.input[l.pos:])
	return r
}

func (l *lexer) peek() rune {
	r := l.next()
	l.backup()
	return r
}

func (l *lexer) ignore() {
	l.pos += 1
	l.start = l.pos
}

// next returns the next rune in the input.
func (l *lexer) next() rune {
	if int(l.pos) >= len(l.input) {
		l.width = 0
		return eof
	}
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	l.width = Pos(w)
	l.pos += l.width
	return r
}

// nextItem returns the next item from the input.
func (l *lexer) nextItem() item {
	item := <-l.items
	l.lastPos = item.pos
	return item

}

func (l *lexer) run() {
	for l.state = lexStart; l.state != nil; {
		l.state = l.state(l)
	}
}

// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
	return r == ' ' || r == '\t'
}

// isEndOfLine reports whether r is an end-of-line character.
func isEndOfLine(r rune) bool {
	return r == '\r' || r == '\n'
}

func lexStart(l *lexer) stateFn {
	for {
		if len(l.input) == 0 {
			log.Debugln("\tEmit EOF!")
			l.emit(itemEOF)
			return nil
		}

		log.Debugf("\tlexParagraph: %q, Start: %d, Pos: %d\n",
			l.input[l.start:l.pos], l.start, l.pos)

		switch r := l.current(); {
		case isSectionAdornment(r) && isSectionAdornment(l.peek()) && l.pos == 1:
			log.Debugln("Transition lexSection...")
			return lexSection
		case isEndOfLine(r):
			log.Debugln("\tFound newline!")
			if isSectionAdornment(rune(l.input[l.pos+2])) {
				log.Debugln("Transition lexSection...")
				return lexSection
			}
			l.ignore()
		}

		if l.next() == eof {
			break
		}
	}

	// Correctly reached EOF.
	if l.pos > l.start {
		log.Debugln("\tEmit Paragraph!")
		l.emit(itemParagraph)
	}

	l.emit(itemEOF)
	return nil
}

func lexSection(l *lexer) stateFn {
	if len(l.input) > 0 {
		log.Debugf("\tlexSection: %q, Pos: %d\n",
			l.input[l.start:l.pos], l.pos)
	}

	if isEndOfLine(l.peek()) {
		log.Debugln("\tEmit itemTitle")
		l.emit(itemTitle)
		l.ignore()
	}

Loop:
	for {
		switch r := l.next(); {
		case isSectionAdornment(r):
			if len(l.input) > 0 {
				log.Debugf("\tlexSection: %q, Pos: %d\n",
					l.input[l.start:l.pos], l.pos)
			}
		case isEndOfLine(r):
			l.backup()
			log.Debugf("\tEmit itemSectionAdornment: %d-%d\n",
				l.start, l.pos)
			l.emit(itemSectionAdornment)
			l.ignore()
			break Loop
		}
	}
	log.Debugln("Transition lexStart...")
	return lexStart
}
<commit_msg>lex.go: Add String() method to itemElement<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.
package parse

import (
	// "strings"
	// "unicode"
	"fmt"
	"github.com/demizer/go-elog"
	"unicode/utf8"
)

type itemElement int

const (
	itemEOF itemElement = iota
	itemError
	itemTitle
	itemSectionAdornment
	itemParagraph
)

var elements = [...]string{
	"itemEOF",
	"itemError",
	"itemTitle",
	"itemSectionAdornment",
	"itemParagraph",
}

func (t itemElement) String() string { return elements[t] }

var sectionAdornments = []rune{'!', '"', '#', '$', '\'', '%', '&', '(', ')', '*',
	'+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\',
	']', '^', '_', '`', '{', '|', '}', '~'}

func isSectionAdornment(r rune) bool {
	for _, a := range sectionAdornments {
		if a == r {
			return true
		}
	}
	return false
}

const eof = -1

type stateFn func(*lexer) stateFn

type item struct {
	typ itemElement
	pos Pos
	val string
}

func (i item) String() string {
	switch {
	case i.typ == itemEOF:
		return "EOF"
	case i.typ == itemError:
		return i.val
	}
	return fmt.Sprintf("%q", i.val)
}

type lexer struct {
	name    string
	input   string
	state   stateFn
	pos     Pos
	start   Pos
	width   Pos
	lastPos Pos
	items   chan item
}

func lex(name, input string) *lexer {
	l := &lexer{
		name:  name,
		input: input,
		items: make(chan item),
	}
	go l.run()
	return l
}

// emit passes an item back to the client.
func (l *lexer) emit(t itemElement) {
	log.Debugf("\tEmit %s!\n", t)
	l.items <- item{t, l.start, l.input[l.start:l.pos]}
	l.start = l.pos

}

func (l *lexer) backup() {
	l.pos -= l.width
}

func (l *lexer) current() rune {
	r, _ := utf8.DecodeRuneInString(l.input[l.pos:])
	return r
}

func (l *lexer) peek() rune {
	r := l.next()
	l.backup()
	return r
}

func (l *lexer) ignore() {
	l.pos += 1
	l.start = l.pos
}

// next returns the next rune in the input.
func (l *lexer) next() rune {
	if int(l.pos) >= len(l.input) {
		l.width = 0
		return eof
	}
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	l.width = Pos(w)
	l.pos += l.width
	return r
}

// nextItem returns the next item from the input.
func (l *lexer) nextItem() item {
	item := <-l.items
	l.lastPos = item.pos
	return item

}

func (l *lexer) run() {
	for l.state = lexStart; l.state != nil; {
		l.state = l.state(l)
	}
}

// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
	return r == ' ' || r == '\t'
}

// isEndOfLine reports whether r is an end-of-line character.
func isEndOfLine(r rune) bool {
	return r == '\r' || r == '\n'
}

func lexStart(l *lexer) stateFn {
	log.Debugln("Transition lexStart...")
	for {
		if len(l.input) == 0 {
			log.Debugln("\tEmit EOF!")
			l.emit(itemEOF)
			return nil
		}

		log.Debugf("\tlexStart: %q, Start: %d, Pos: %d\n",
			l.input[l.start:l.pos], l.start, l.pos)

		switch r := l.current(); {
		case isSectionAdornment(r) && isSectionAdornment(l.peek()) && l.pos == 1:
			log.Debugln("Transition lexSection...")
			return lexSection
		case isEndOfLine(r):
			log.Debugln("\tFound newline!")
			if isSectionAdornment(rune(l.input[l.pos+2])) {
				log.Debugln("Transition lexSection...")
				return lexSection
			}
			l.ignore()
		}

		if l.next() == eof {
			break
		}
	}

	// Correctly reached EOF.
	if l.pos > l.start {
		l.emit(itemParagraph)
	}

	l.emit(itemEOF)
	return nil
}

func lexSection(l *lexer) stateFn {
	if len(l.input) > 0 {
		log.Debugf("\tlexSection: %q, Pos: %d\n",
			l.input[l.start:l.pos], l.pos)
	}

	if isEndOfLine(l.peek()) {
		l.emit(itemTitle)
		l.ignore()
	}

Loop:
	for {
		switch r := l.next(); {
		case isSectionAdornment(r):
			if len(l.input) > 0 {
				log.Debugf("\tlexSection: %q, Pos: %d\n",
					l.input[l.start:l.pos], l.pos)
			}
		case isEndOfLine(r):
			l.backup()
			l.emit(itemSectionAdornment)
			l.ignore()
			break Loop
		}
	}
	return lexStart
}
<|endoftext|>


<commit_before>// Copyright 2015 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package sa

import (
	"database/sql"
	"fmt"
	"net/url"
	"strings"

	_ "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/go-sql-driver/mysql"
	gorp "github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1"
	"github.com/letsencrypt/boulder/core"
	blog "github.com/letsencrypt/boulder/log"
)

// NewDbMap creates the root gorp mapping object. Create one of these for each
// database schema you wish to map. Each DbMap contains a list of mapped tables.
// It automatically maps the tables for the primary parts of Boulder around the
// Storage Authority. This may require some further work when we use a disjoint
// schema, like that for `certificate-authority-data.go`.
func NewDbMap(dbConnect string) (*gorp.DbMap, error) {
	logger := blog.GetAuditLogger()

	var err error
	dbConnect, err = recombineURLForDB(dbConnect)
	if err != nil {
		return nil, err
	}

	logger.Debug("Connecting to database")

	db, err := sql.Open("mysql", dbConnect)
	if err != nil {
		return nil, err
	}
	if err = db.Ping(); err != nil {
		return nil, err
	}

	dialect := gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}
	dbmap := &gorp.DbMap{Db: db, Dialect: dialect, TypeConverter: BoulderTypeConverter{}}

	initTables(dbmap)

	logger.Debug("Connected to database")

	return dbmap, err
}

// recombineURLForDB transforms a database URL to a URL-like string
// that the mysql driver can use. The mysql driver needs the Host data
// to be wrapped in "tcp()" but url.Parse will escape the parentheses
// and the mysql driver doesn't understand them. So, we can't have
// "tcp()" in the configs, but can't leave it out before passing it to
// the mysql driver. Similarly, the driver needs the password and
// username unescaped. Compromise by doing the leg work if the config
// says the database URL's scheme is a fake one called
// "mysqltcp://". See
// https://github.com/go-sql-driver/mysql/issues/362 for why we have
// to futz around and avoid URL.String.
func recombineURLForDB(dbConnect string) (string, error) {
	dbConnect = strings.TrimSpace(dbConnect)
	dbURL, err := url.Parse(dbConnect)
	if err != nil {
		return "", err
	}

	if dbURL.Scheme != "mysql+tcp" {
		format := "given database connection string was not a mysql+tcp:// URL, was %#v"
		return "", fmt.Errorf(format, dbURL.Scheme)
	}

	dsnVals, err := url.ParseQuery(dbURL.RawQuery)
	if err != nil {
		return "", err
	}

	dsnVals.Set("parseTime", "true")

	// Required to make UPDATE return the number of rows matched,
	// instead of the number of rows changed by the UPDATE.
	dsnVals.Set("clientFoundRows", "true")

	user := dbURL.User.Username()
	passwd, hasPass := dbURL.User.Password()
	dbConn := ""
	if user != "" {
		dbConn = url.QueryEscape(user)
	}
	if hasPass {
		dbConn += ":" + passwd
	}
	dbConn += "@tcp(" + dbURL.Host + ")"
	// TODO(jmhodges): should be dbURL.EscapedPath() but Travis doesn't have 1.5
	return dbConn + dbURL.Path + "?" + dsnVals.Encode(), nil
}

// SetSQLDebug enables/disables GORP SQL-level Debugging
func SetSQLDebug(dbMap *gorp.DbMap, state bool) {
	dbMap.TraceOff()

	if state {
		// Enable logging
		dbMap.TraceOn("SQL: ", &SQLLogger{blog.GetAuditLogger()})
	}
}

// SQLLogger adapts the AuditLogger to a format GORP can use.
type SQLLogger struct {
	log *blog.AuditLogger
}

// Printf adapts the AuditLogger to GORP's interface
func (log *SQLLogger) Printf(format string, v ...interface{}) {
	log.log.Debug(fmt.Sprintf(format, v))
}

// initTables constructs the table map for the ORM. If you want to also create
// the tables, call CreateTablesIfNotExists on the DbMap.
func initTables(dbMap *gorp.DbMap) {
	regTable := dbMap.AddTableWithName(regModel{}, "registrations").SetKeys(true, "ID")
	regTable.SetVersionCol("LockCol")
	regTable.ColMap("Key").SetNotNull(true)
	regTable.ColMap("KeySHA256").SetNotNull(true).SetUnique(true)
	pendingAuthzTable := dbMap.AddTableWithName(pendingauthzModel{}, "pending_authz").SetKeys(false, "ID")
	pendingAuthzTable.SetVersionCol("LockCol")
	pendingAuthzTable.ColMap("Challenges").SetMaxSize(1536)

	authzTable := dbMap.AddTableWithName(authzModel{}, "authz").SetKeys(false, "ID")
	authzTable.ColMap("Challenges").SetMaxSize(1536)

	dbMap.AddTableWithName(core.Certificate{}, "certificates").SetKeys(false, "Serial")
	dbMap.AddTableWithName(core.CertificateStatus{}, "certificateStatus").SetKeys(false, "Serial").SetVersionCol("LockCol")
	dbMap.AddTableWithName(core.OCSPResponse{}, "ocspResponses").SetKeys(true, "ID")
	dbMap.AddTableWithName(core.CRL{}, "crls").SetKeys(false, "Serial")
	dbMap.AddTableWithName(core.DeniedCSR{}, "deniedCSRs").SetKeys(true, "ID")
}
<commit_msg>use URL.EscapedPath now that we are on Go 1.5<commit_after>// Copyright 2015 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package sa

import (
	"database/sql"
	"fmt"
	"net/url"
	"strings"

	_ "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/go-sql-driver/mysql"
	gorp "github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1"
	"github.com/letsencrypt/boulder/core"
	blog "github.com/letsencrypt/boulder/log"
)

// NewDbMap creates the root gorp mapping object. Create one of these for each
// database schema you wish to map. Each DbMap contains a list of mapped tables.
// It automatically maps the tables for the primary parts of Boulder around the
// Storage Authority. This may require some further work when we use a disjoint
// schema, like that for `certificate-authority-data.go`.
func NewDbMap(dbConnect string) (*gorp.DbMap, error) {
	logger := blog.GetAuditLogger()

	var err error
	dbConnect, err = recombineURLForDB(dbConnect)
	if err != nil {
		return nil, err
	}

	logger.Debug("Connecting to database")

	db, err := sql.Open("mysql", dbConnect)
	if err != nil {
		return nil, err
	}
	if err = db.Ping(); err != nil {
		return nil, err
	}

	dialect := gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}
	dbmap := &gorp.DbMap{Db: db, Dialect: dialect, TypeConverter: BoulderTypeConverter{}}

	initTables(dbmap)

	logger.Debug("Connected to database")

	return dbmap, err
}

// recombineURLForDB transforms a database URL to a URL-like string
// that the mysql driver can use. The mysql driver needs the Host data
// to be wrapped in "tcp()" but url.Parse will escape the parentheses
// and the mysql driver doesn't understand them. So, we can't have
// "tcp()" in the configs, but can't leave it out before passing it to
// the mysql driver. Similarly, the driver needs the password and
// username unescaped. Compromise by doing the leg work if the config
// says the database URL's scheme is a fake one called
// "mysql+tcp://". See
// https://github.com/go-sql-driver/mysql/issues/362 for why we have
// to futz around and avoid URL.String.
func recombineURLForDB(dbConnect string) (string, error) {
	dbConnect = strings.TrimSpace(dbConnect)
	dbURL, err := url.Parse(dbConnect)
	if err != nil {
		return "", err
	}

	if dbURL.Scheme != "mysql+tcp" {
		format := "given database connection string was not a mysql+tcp:// URL, was %#v"
		return "", fmt.Errorf(format, dbURL.Scheme)
	}

	dsnVals, err := url.ParseQuery(dbURL.RawQuery)
	if err != nil {
		return "", err
	}

	dsnVals.Set("parseTime", "true")

	// Required to make UPDATE return the number of rows matched,
	// instead of the number of rows changed by the UPDATE.
	dsnVals.Set("clientFoundRows", "true")

	user := dbURL.User.Username()
	passwd, hasPass := dbURL.User.Password()
	dbConn := ""
	if user != "" {
		dbConn = url.QueryEscape(user)
	}
	if hasPass {
		dbConn += ":" + passwd
	}
	dbConn += "@tcp(" + dbURL.Host + ")"
	return dbConn + dbURL.EscapedPath() + "?" + dsnVals.Encode(), nil
}

// SetSQLDebug enables/disables GORP SQL-level Debugging
func SetSQLDebug(dbMap *gorp.DbMap, state bool) {
	dbMap.TraceOff()

	if state {
		// Enable logging
		dbMap.TraceOn("SQL: ", &SQLLogger{blog.GetAuditLogger()})
	}
}

// SQLLogger adapts the AuditLogger to a format GORP can use.
type SQLLogger struct {
	log *blog.AuditLogger
}

// Printf adapts the AuditLogger to GORP's interface
func (log *SQLLogger) Printf(format string, v ...interface{}) {
	log.log.Debug(fmt.Sprintf(format, v))
}

// initTables constructs the table map for the ORM. If you want to also create
// the tables, call CreateTablesIfNotExists on the DbMap.
func initTables(dbMap *gorp.DbMap) {
	regTable := dbMap.AddTableWithName(regModel{}, "registrations").SetKeys(true, "ID")
	regTable.SetVersionCol("LockCol")
	regTable.ColMap("Key").SetNotNull(true)
	regTable.ColMap("KeySHA256").SetNotNull(true).SetUnique(true)
	pendingAuthzTable := dbMap.AddTableWithName(pendingauthzModel{}, "pending_authz").SetKeys(false, "ID")
	pendingAuthzTable.SetVersionCol("LockCol")
	pendingAuthzTable.ColMap("Challenges").SetMaxSize(1536)

	authzTable := dbMap.AddTableWithName(authzModel{}, "authz").SetKeys(false, "ID")
	authzTable.ColMap("Challenges").SetMaxSize(1536)

	dbMap.AddTableWithName(core.Certificate{}, "certificates").SetKeys(false, "Serial")
	dbMap.AddTableWithName(core.CertificateStatus{}, "certificateStatus").SetKeys(false, "Serial").SetVersionCol("LockCol")
	dbMap.AddTableWithName(core.OCSPResponse{}, "ocspResponses").SetKeys(true, "ID")
	dbMap.AddTableWithName(core.CRL{}, "crls").SetKeys(false, "Serial")
	dbMap.AddTableWithName(core.DeniedCSR{}, "deniedCSRs").SetKeys(true, "ID")
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package network

import (
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/test/e2e/framework"
	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
	"k8s.io/kubernetes/test/e2e/upgrades"

	"github.com/onsi/ginkgo"
)

// ServiceUpgradeTest tests that a service is available before and
// after a cluster upgrade. During a master-only upgrade, it will test
// that a service remains available during the upgrade.
type ServiceUpgradeTest struct {
	jig          *e2eservice.TestJig
	tcpService   *v1.Service
	tcpIngressIP string
	svcPort      int
}

// Name returns the tracking name of the test.
func (ServiceUpgradeTest) Name() string { return "service-upgrade" }

func shouldTestPDBs() bool { return framework.ProviderIs("gce", "gke") }

// Setup creates a service with a load balancer and makes sure it's reachable.
func (t *ServiceUpgradeTest) Setup(f *framework.Framework) {
	serviceName := "service-test"
	jig := e2eservice.NewTestJig(f.ClientSet, f.Namespace.Name, serviceName)

	ns := f.Namespace
	cs := f.ClientSet

	ginkgo.By("creating a TCP service " + serviceName + " with type=LoadBalancer in namespace " + ns.Name)
	_, err := jig.CreateTCPService(func(s *v1.Service) {
		s.Spec.Type = v1.ServiceTypeLoadBalancer
	})
	framework.ExpectNoError(err)
	tcpService, err := jig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cs))
	framework.ExpectNoError(err)

	// Get info to hit it with
	tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])
	svcPort := int(tcpService.Spec.Ports[0].Port)

	ginkgo.By("creating pod to be part of service " + serviceName)
	rc, err := jig.Run(jig.AddRCAntiAffinity)
	framework.ExpectNoError(err)

	if shouldTestPDBs() {
		ginkgo.By("creating a PodDisruptionBudget to cover the ReplicationController")
		_, err = jig.CreatePDB(rc)
		framework.ExpectNoError(err)
	}

	// Hit it once before considering ourselves ready
	ginkgo.By("hitting the pod through the service's LoadBalancer")
	timeout := e2eservice.LoadBalancerLagTimeoutDefault
	if framework.ProviderIs("aws") {
		timeout = e2eservice.LoadBalancerLagTimeoutAWS
	}
	e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, timeout)

	t.jig = jig
	t.tcpService = tcpService
	t.tcpIngressIP = tcpIngressIP
	t.svcPort = svcPort
}

// Test runs a connectivity check to the service.
func (t *ServiceUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
	switch upgrade {
	case upgrades.MasterUpgrade, upgrades.ClusterUpgrade:
		t.test(f, done, true, true)
	case upgrades.NodeUpgrade:
		// Node upgrades should test during disruption only on GCE/GKE for now.
		t.test(f, done, shouldTestPDBs(), false)
	default:
		t.test(f, done, false, false)
	}
}

// Teardown cleans up any remaining resources.
func (t *ServiceUpgradeTest) Teardown(f *framework.Framework) {
	// rely on the namespace deletion to clean up everything
}

func (t *ServiceUpgradeTest) test(f *framework.Framework, done <-chan struct{}, testDuringDisruption, testFinalizer bool) {
	if testDuringDisruption {
		// Continuous validation
		ginkgo.By("continuously hitting the pod through the service's LoadBalancer")
		wait.Until(func() {
			e2eservice.TestReachableHTTP(t.tcpIngressIP, t.svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
		}, framework.Poll, done)
	} else {
		// Block until upgrade is done
		ginkgo.By("waiting for upgrade to finish without checking if service remains up")
		<-done
	}

	// Hit it once more
	ginkgo.By("hitting the pod through the service's LoadBalancer")
	e2eservice.TestReachableHTTP(t.tcpIngressIP, t.svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
	if testFinalizer {
		defer func() {
			ginkgo.By("Check that service can be deleted with finalizer")
			e2eservice.WaitForServiceDeletedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name)
		}()
		ginkgo.By("Check that finalizer is present on loadBalancer type service")
		e2eservice.WaitForServiceUpdatedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name, true)
	}
}
<commit_msg>UPSTREAM: <carry>: Always test PDB's during service upgrade test<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package network

import (
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/test/e2e/framework"
	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
	"k8s.io/kubernetes/test/e2e/upgrades"

	"github.com/onsi/ginkgo"
)

// ServiceUpgradeTest tests that a service is available before and
// after a cluster upgrade. During a master-only upgrade, it will test
// that a service remains available during the upgrade.
type ServiceUpgradeTest struct {
	jig          *e2eservice.TestJig
	tcpService   *v1.Service
	tcpIngressIP string
	svcPort      int
}

// Name returns the tracking name of the test.
func (ServiceUpgradeTest) Name() string { return "service-upgrade" }

func shouldTestPDBs() bool { return true }

// Setup creates a service with a load balancer and makes sure it's reachable.
func (t *ServiceUpgradeTest) Setup(f *framework.Framework) {
	serviceName := "service-test"
	jig := e2eservice.NewTestJig(f.ClientSet, f.Namespace.Name, serviceName)

	ns := f.Namespace
	cs := f.ClientSet

	ginkgo.By("creating a TCP service " + serviceName + " with type=LoadBalancer in namespace " + ns.Name)
	_, err := jig.CreateTCPService(func(s *v1.Service) {
		s.Spec.Type = v1.ServiceTypeLoadBalancer
	})
	framework.ExpectNoError(err)
	tcpService, err := jig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cs))
	framework.ExpectNoError(err)

	// Get info to hit it with
	tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])
	svcPort := int(tcpService.Spec.Ports[0].Port)

	ginkgo.By("creating pod to be part of service " + serviceName)
	rc, err := jig.Run(jig.AddRCAntiAffinity)
	framework.ExpectNoError(err)

	if shouldTestPDBs() {
		ginkgo.By("creating a PodDisruptionBudget to cover the ReplicationController")
		_, err = jig.CreatePDB(rc)
		framework.ExpectNoError(err)
	}

	// Hit it once before considering ourselves ready
	ginkgo.By("hitting the pod through the service's LoadBalancer")
	timeout := e2eservice.LoadBalancerLagTimeoutDefault
	if framework.ProviderIs("aws") {
		timeout = e2eservice.LoadBalancerLagTimeoutAWS
	}
	e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, timeout)

	t.jig = jig
	t.tcpService = tcpService
	t.tcpIngressIP = tcpIngressIP
	t.svcPort = svcPort
}

// Test runs a connectivity check to the service.
func (t *ServiceUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
	switch upgrade {
	case upgrades.MasterUpgrade, upgrades.ClusterUpgrade:
		t.test(f, done, true, true)
	case upgrades.NodeUpgrade:
		// Node upgrades should test during disruption only on GCE/GKE for now.
		t.test(f, done, shouldTestPDBs(), false)
	default:
		t.test(f, done, false, false)
	}
}

// Teardown cleans up any remaining resources.
func (t *ServiceUpgradeTest) Teardown(f *framework.Framework) {
	// rely on the namespace deletion to clean up everything
}

func (t *ServiceUpgradeTest) test(f *framework.Framework, done <-chan struct{}, testDuringDisruption, testFinalizer bool) {
	if testDuringDisruption {
		// Continuous validation
		ginkgo.By("continuously hitting the pod through the service's LoadBalancer")
		wait.Until(func() {
			e2eservice.TestReachableHTTP(t.tcpIngressIP, t.svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
		}, framework.Poll, done)
	} else {
		// Block until upgrade is done
		ginkgo.By("waiting for upgrade to finish without checking if service remains up")
		<-done
	}

	// Hit it once more
	ginkgo.By("hitting the pod through the service's LoadBalancer")
	e2eservice.TestReachableHTTP(t.tcpIngressIP, t.svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
	if testFinalizer {
		defer func() {
			ginkgo.By("Check that service can be deleted with finalizer")
			e2eservice.WaitForServiceDeletedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name)
		}()
		ginkgo.By("Check that finalizer is present on loadBalancer type service")
		e2eservice.WaitForServiceUpdatedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name, true)
	}
}
<|endoftext|>


<commit_before>package analytics

//
// dependencies
//

import "github.com/jehiah/go-strftime"
import "github.com/nu7hatch/gouuid"
import . "encoding/json"
import "io/ioutil"
import "net/http"
import "bytes"
import "time"
import "log"

//
// Library version
//

const Version = "0.0.1"

//
// Default API end-point
//

const api = "https://api.segment.io"

//
// Segment.io client
//

type Client struct {
	Debug         bool
	BufferSize    int
	FlushInterval time.Duration
	Endpoint      string
	Key           string
	buffer        []*interface{}
}

//
// Message context library
//

type contextLibrary struct {
	Name    string `json:"name"`
	Version string `json:"version"`
}

//
// Message context
//

type context struct {
	Library contextLibrary `json:"library"`
}

//
// Identify message
//

type identify struct {
	Action    string      `json:"action"`
	Traits    interface{} `json:"trailts"`
	Timestamp string      `json:"timestamp"`
}

//
// Alias message
//

type alias struct {
	Action     string `json:"action"`
	PreviousId string `json:"previousId"`
	Timestamp  string `json:"timestamp"`
}

//
// Track message
//

type track struct {
	Action     string      `json:"action"`
	Event      string      `json:"event"`
	Properties interface{} `json:"properties"`
	Timestamp  string      `json:"timestamp"`
}

//
// Group message
//

type group struct {
	Action    string      `json:"action"`
	GroupId   string      `json:"groupId"`
	Traits    interface{} `json:"trailts"`
	Timestamp string      `json:"timestamp"`
}

//
// Page message
//

type page struct {
	Action     string      `json:"action"`
	Category   string      `json:"category"`
	Name       string      `json:"name"`
	Properties interface{} `json:"properties"`
	Timestamp  string      `json:"timestamp"`
}

//
// Batch message
//

type batch struct {
	Context   context        `json:"context"`
	RequestId string         `json:"requestId"`
	Messages  []*interface{} `json:"batch"`
}

//
// Return a new Segment.io client
// with the given write key.
//

func New(key string) (c *Client) {
	defer func() {
		go func() {
			for {
				time.Sleep(c.FlushInterval)
				c.log("interval %v reached", c.FlushInterval)
				go c.flush()
			}
		}()
	}()

	return &Client{
		Debug:         false,
		BufferSize:    500,
		FlushInterval: 30 * time.Second,
		Key:           key,
		Endpoint:      api,
		buffer:        make([]*interface{}, 0),
	}
}

//
// Buffer an alias message
//

func (c *Client) Alias(previousId string) {
	c.bufferMessage(&alias{"Alias", previousId, timestamp()})
}

//
// Buffer a page message
//

func (c *Client) Page(name string, category string, properties interface{}) {
	c.bufferMessage(&page{"Page", name, category, properties, timestamp()})
}

//
// Buffer a screen message
//

func (c *Client) Screen(name string, category string, properties interface{}) {
	c.bufferMessage(&page{"Screen", name, category, properties, timestamp()})
}

//
// Buffer a group message
//

func (c *Client) Group(id string, traits interface{}) {
	c.bufferMessage(&group{"Group", id, traits, timestamp()})
}

//
// Buffer an identify message
//

func (c *Client) Identify(traits interface{}) {
	c.bufferMessage(&identify{"Identify", traits, timestamp()})
}

//
// Buffer a track message
//

func (c *Client) Track(event string, properties interface{}) {
	c.bufferMessage(&track{"Track", event, properties, timestamp()})
}

//
// Return formatted timestamp.
//

func timestamp() string {
	return strftime.Format("%Y-%m-%dT%H:%M:%S%z", time.Now())
}

//
// Log in debug mode.
//

func (c *Client) log(format string, v ...interface{}) {
	if c.Debug {
		log.Printf(format, v...)
	}
}

//
// Buffer the given message and flush
// when the buffer exceeds .BufferSize.
//

func (c *Client) bufferMessage(msg interface{}) {
	c.buffer = append(c.buffer, &msg)

	c.log("buffer (%d/%d) %v", len(c.buffer), c.BufferSize, msg)

	if len(c.buffer) >= c.BufferSize {
		go c.flush()
	}
}

// Return a batch message primed
// with context properties
//

func batchMessage(msgs []*interface{}) (*batch, error) {
	uid, err := uuid.NewV4()

	if err != nil {
		return nil, err
	}

	batch := &batch{
		RequestId: uid.String(),
		Messages:  msgs,
		Context: context{
			Library: contextLibrary{
				Name:    "analytics-go",
				Version: Version,
			},
		},
	}

	return batch, nil
}

//
// Flush the buffered messages.
//

func (c *Client) flush() error {
	if len(c.buffer) == 0 {
		c.log("no messages to flush")
		return nil
	}

	c.log("flushing %d messages", len(c.buffer))
	batch, err := batchMessage(c.buffer)

	if err != nil {
		return err
	}

	json, err := Marshal(batch)

	if err != nil {
		return err
	}

	c.buffer = nil

	client := &http.Client{}
	url := c.Endpoint + "/v1/import"
	c.log("POST %s with %d bytes", url, len(json))
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(json))

	if err != nil {
		return err
	}

	req.Header.Add("User-Agent", "analytics-go (version: "+Version+")")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Content-Length", string(len(json)))
	req.SetBasicAuth(c.Key, "")

	res, err := client.Do(req)
	c.log("%d response", res.StatusCode)

	if res.StatusCode >= 400 {
		body, _ := ioutil.ReadAll(res.Body)
		c.log("error: %s", string(body))
	}

	return err
}
<commit_msg>change Action to Type and lowercase<commit_after>package analytics

//
// dependencies
//

import "github.com/jehiah/go-strftime"
import "github.com/nu7hatch/gouuid"
import . "encoding/json"
import "io/ioutil"
import "net/http"
import "bytes"
import "time"
import "log"

//
// Library version
//

const Version = "0.0.1"

//
// Default API end-point
//

const api = "https://api.segment.io"

//
// Segment.io client
//

type Client struct {
	Debug         bool
	BufferSize    int
	FlushInterval time.Duration
	Endpoint      string
	Key           string
	buffer        []*interface{}
}

//
// Message context library
//

type contextLibrary struct {
	Name    string `json:"name"`
	Version string `json:"version"`
}

//
// Message context
//

type context struct {
	Library contextLibrary `json:"library"`
}

//
// Identify message
//

type identify struct {
	Type      string      `json:"type"`
	Traits    interface{} `json:"trailts"`
	Timestamp string      `json:"timestamp"`
}

//
// Alias message
//

type alias struct {
	Type       string `json:"type"`
	PreviousId string `json:"previousId"`
	Timestamp  string `json:"timestamp"`
}

//
// Track message
//

type track struct {
	Type       string      `json:"type"`
	Event      string      `json:"event"`
	Properties interface{} `json:"properties"`
	Timestamp  string      `json:"timestamp"`
}

//
// Group message
//

type group struct {
	Type      string      `json:"type"`
	GroupId   string      `json:"groupId"`
	Traits    interface{} `json:"trailts"`
	Timestamp string      `json:"timestamp"`
}

//
// Page message
//

type page struct {
	Type       string      `json:"type"`
	Category   string      `json:"category"`
	Name       string      `json:"name"`
	Properties interface{} `json:"properties"`
	Timestamp  string      `json:"timestamp"`
}

//
// Batch message
//

type batch struct {
	Context   context        `json:"context"`
	RequestId string         `json:"requestId"`
	Messages  []*interface{} `json:"batch"`
}

//
// Return a new Segment.io client
// with the given write key.
//

func New(key string) (c *Client) {
	defer func() {
		go func() {
			for {
				time.Sleep(c.FlushInterval)
				c.log("interval %v reached", c.FlushInterval)
				go c.flush()
			}
		}()
	}()

	return &Client{
		Debug:         false,
		BufferSize:    500,
		FlushInterval: 30 * time.Second,
		Key:           key,
		Endpoint:      api,
		buffer:        make([]*interface{}, 0),
	}
}

//
// Buffer an alias message
//

func (c *Client) Alias(previousId string) {
	c.bufferMessage(&alias{"alias", previousId, timestamp()})
}

//
// Buffer a page message
//

func (c *Client) Page(name string, category string, properties interface{}) {
	c.bufferMessage(&page{"page", name, category, properties, timestamp()})
}

//
// Buffer a screen message
//

func (c *Client) Screen(name string, category string, properties interface{}) {
	c.bufferMessage(&page{"screen", name, category, properties, timestamp()})
}

//
// Buffer a group message
//

func (c *Client) Group(id string, traits interface{}) {
	c.bufferMessage(&group{"group", id, traits, timestamp()})
}

//
// Buffer an identify message
//

func (c *Client) Identify(traits interface{}) {
	c.bufferMessage(&identify{"identify", traits, timestamp()})
}

//
// Buffer a track message
//

func (c *Client) Track(event string, properties interface{}) {
	c.bufferMessage(&track{"track", event, properties, timestamp()})
}

//
// Return formatted timestamp.
//

func timestamp() string {
	return strftime.Format("%Y-%m-%dT%H:%M:%S%z", time.Now())
}

//
// Log in debug mode.
//

func (c *Client) log(format string, v ...interface{}) {
	if c.Debug {
		log.Printf(format, v...)
	}
}

//
// Buffer the given message and flush
// when the buffer exceeds .BufferSize.
//

func (c *Client) bufferMessage(msg interface{}) {
	c.buffer = append(c.buffer, &msg)

	c.log("buffer (%d/%d) %v", len(c.buffer), c.BufferSize, msg)

	if len(c.buffer) >= c.BufferSize {
		go c.flush()
	}
}

// Return a batch message primed
// with context properties
//

func batchMessage(msgs []*interface{}) (*batch, error) {
	uid, err := uuid.NewV4()

	if err != nil {
		return nil, err
	}

	batch := &batch{
		RequestId: uid.String(),
		Messages:  msgs,
		Context: context{
			Library: contextLibrary{
				Name:    "analytics-go",
				Version: Version,
			},
		},
	}

	return batch, nil
}

//
// Flush the buffered messages.
//

func (c *Client) flush() error {
	if len(c.buffer) == 0 {
		c.log("no messages to flush")
		return nil
	}

	c.log("flushing %d messages", len(c.buffer))
	batch, err := batchMessage(c.buffer)

	if err != nil {
		return err
	}

	json, err := Marshal(batch)

	if err != nil {
		return err
	}

	c.buffer = nil

	client := &http.Client{}
	url := c.Endpoint + "/v1/import"
	c.log("POST %s with %d bytes", url, len(json))
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(json))

	if err != nil {
		return err
	}

	req.Header.Add("User-Agent", "analytics-go (version: "+Version+")")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Content-Length", string(len(json)))
	req.SetBasicAuth(c.Key, "")

	res, err := client.Do(req)
	c.log("%d response", res.StatusCode)

	if res.StatusCode >= 400 {
		body, _ := ioutil.ReadAll(res.Body)
		c.log("error: %s", string(body))
	}

	return err
}
<|endoftext|>


<commit_before>package daemon

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/docker/docker/graph"
	"github.com/docker/docker/image"
	"github.com/docker/docker/pkg/parsers"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/runconfig"
	"github.com/docker/libcontainer/label"
)

func (daemon *Daemon) ContainerCreate(name string, config *runconfig.Config, hostConfig *runconfig.HostConfig) (string, []string, error) {
	warnings, err := daemon.verifyHostConfig(hostConfig)
	if err != nil {
		return "", warnings, err
	}

	// The check for a valid workdir path is made on the server rather than in the
	// client. This is because we don't know the type of path (Linux or Windows)
	// to validate on the client.
	if config.WorkingDir != "" && !filepath.IsAbs(config.WorkingDir) {
		return "", warnings, fmt.Errorf("The working directory '%s' is invalid. It needs to be an absolute path.", config.WorkingDir)
	}

	container, buildWarnings, err := daemon.Create(config, hostConfig, name)
	if err != nil {
		if daemon.Graph().IsNotExist(err, config.Image) {
			_, tag := parsers.ParseRepositoryTag(config.Image)
			if tag == "" {
				tag = graph.DEFAULTTAG
			}
			return "", warnings, fmt.Errorf("No such image: %s (tag: %s)", config.Image, tag)
		}
		return "", warnings, err
	}

	container.LogEvent("create")
	warnings = append(warnings, buildWarnings...)

	return container.ID, warnings, nil
}

// Create creates a new container from the given configuration with a given name.
func (daemon *Daemon) Create(config *runconfig.Config, hostConfig *runconfig.HostConfig, name string) (*Container, []string, error) {
	var (
		container *Container
		warnings  []string
		img       *image.Image
		imgID     string
		err       error
	)

	if config.Image != "" {
		img, err = daemon.repositories.LookupImage(config.Image)
		if err != nil {
			return nil, nil, err
		}
		if err = img.CheckDepth(); err != nil {
			return nil, nil, err
		}
		imgID = img.ID
	}

	if warnings, err = daemon.mergeAndVerifyConfig(config, img); err != nil {
		return nil, nil, err
	}
	if !config.NetworkDisabled && daemon.SystemConfig().IPv4ForwardingDisabled {
		warnings = append(warnings, "IPv4 forwarding is disabled.\n")
	}
	if hostConfig == nil {
		hostConfig = &runconfig.HostConfig{}
	}
	if hostConfig.SecurityOpt == nil {
		hostConfig.SecurityOpt, err = daemon.GenerateSecurityOpt(hostConfig.IpcMode, hostConfig.PidMode)
		if err != nil {
			return nil, nil, err
		}
	}
	if container, err = daemon.newContainer(name, config, imgID); err != nil {
		return nil, nil, err
	}
	if err := daemon.Register(container); err != nil {
		return nil, nil, err
	}
	if err := daemon.createRootfs(container); err != nil {
		return nil, nil, err
	}
	if err := daemon.setHostConfig(container, hostConfig); err != nil {
		return nil, nil, err
	}
	if err := container.Mount(); err != nil {
		return nil, nil, err
	}
	defer container.Unmount()

	for spec := range config.Volumes {
		var (
			name, destination string
			parts             = strings.Split(spec, ":")
		)
		switch len(parts) {
		case 2:
			name, destination = parts[0], filepath.Clean(parts[1])
		default:
			name = stringid.GenerateRandomID()
			destination = filepath.Clean(parts[0])
		}
		// Skip volumes for which we already have something mounted on that
		// destination because of a --volume-from.
		if container.isDestinationMounted(destination) {
			continue
		}
		path, err := container.GetResourcePath(destination)
		if err != nil {
			return nil, nil, err
		}

		stat, err := os.Stat(path)
		if err == nil && !stat.IsDir() {
			return nil, nil, fmt.Errorf("cannot mount volume over existing file, file exists %s", path)
		}

		v, err := createVolume(name, config.VolumeDriver)
		if err != nil {
			return nil, nil, err
		}
		if err := label.Relabel(v.Path(), container.MountLabel, "z"); err != nil {
			return nil, nil, err
		}

		if err := container.copyImagePathContent(v, destination); err != nil {
			return nil, nil, err
		}

		container.addMountPointWithVolume(destination, v, true)
	}
	if err := container.ToDisk(); err != nil {
		return nil, nil, err
	}
	return container, warnings, nil
}

func (daemon *Daemon) GenerateSecurityOpt(ipcMode runconfig.IpcMode, pidMode runconfig.PidMode) ([]string, error) {
	if ipcMode.IsHost() || pidMode.IsHost() {
		return label.DisableSecOpt(), nil
	}
	if ipcContainer := ipcMode.Container(); ipcContainer != "" {
		c, err := daemon.Get(ipcContainer)
		if err != nil {
			return nil, err
		}

		return label.DupSecOpt(c.ProcessLabel), nil
	}
	return nil, nil
}
<commit_msg>Clean up: remove redundant '\n' in Create<commit_after>package daemon

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/docker/docker/graph"
	"github.com/docker/docker/image"
	"github.com/docker/docker/pkg/parsers"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/runconfig"
	"github.com/docker/libcontainer/label"
)

func (daemon *Daemon) ContainerCreate(name string, config *runconfig.Config, hostConfig *runconfig.HostConfig) (string, []string, error) {
	warnings, err := daemon.verifyHostConfig(hostConfig)
	if err != nil {
		return "", warnings, err
	}

	// The check for a valid workdir path is made on the server rather than in the
	// client. This is because we don't know the type of path (Linux or Windows)
	// to validate on the client.
	if config.WorkingDir != "" && !filepath.IsAbs(config.WorkingDir) {
		return "", warnings, fmt.Errorf("The working directory '%s' is invalid. It needs to be an absolute path.", config.WorkingDir)
	}

	container, buildWarnings, err := daemon.Create(config, hostConfig, name)
	if err != nil {
		if daemon.Graph().IsNotExist(err, config.Image) {
			_, tag := parsers.ParseRepositoryTag(config.Image)
			if tag == "" {
				tag = graph.DEFAULTTAG
			}
			return "", warnings, fmt.Errorf("No such image: %s (tag: %s)", config.Image, tag)
		}
		return "", warnings, err
	}

	container.LogEvent("create")
	warnings = append(warnings, buildWarnings...)

	return container.ID, warnings, nil
}

// Create creates a new container from the given configuration with a given name.
func (daemon *Daemon) Create(config *runconfig.Config, hostConfig *runconfig.HostConfig, name string) (*Container, []string, error) {
	var (
		container *Container
		warnings  []string
		img       *image.Image
		imgID     string
		err       error
	)

	if config.Image != "" {
		img, err = daemon.repositories.LookupImage(config.Image)
		if err != nil {
			return nil, nil, err
		}
		if err = img.CheckDepth(); err != nil {
			return nil, nil, err
		}
		imgID = img.ID
	}

	if warnings, err = daemon.mergeAndVerifyConfig(config, img); err != nil {
		return nil, nil, err
	}
	if !config.NetworkDisabled && daemon.SystemConfig().IPv4ForwardingDisabled {
		warnings = append(warnings, "IPv4 forwarding is disabled.")
	}
	if hostConfig == nil {
		hostConfig = &runconfig.HostConfig{}
	}
	if hostConfig.SecurityOpt == nil {
		hostConfig.SecurityOpt, err = daemon.GenerateSecurityOpt(hostConfig.IpcMode, hostConfig.PidMode)
		if err != nil {
			return nil, nil, err
		}
	}
	if container, err = daemon.newContainer(name, config, imgID); err != nil {
		return nil, nil, err
	}
	if err := daemon.Register(container); err != nil {
		return nil, nil, err
	}
	if err := daemon.createRootfs(container); err != nil {
		return nil, nil, err
	}
	if err := daemon.setHostConfig(container, hostConfig); err != nil {
		return nil, nil, err
	}
	if err := container.Mount(); err != nil {
		return nil, nil, err
	}
	defer container.Unmount()

	for spec := range config.Volumes {
		var (
			name, destination string
			parts             = strings.Split(spec, ":")
		)
		switch len(parts) {
		case 2:
			name, destination = parts[0], filepath.Clean(parts[1])
		default:
			name = stringid.GenerateRandomID()
			destination = filepath.Clean(parts[0])
		}
		// Skip volumes for which we already have something mounted on that
		// destination because of a --volume-from.
		if container.isDestinationMounted(destination) {
			continue
		}
		path, err := container.GetResourcePath(destination)
		if err != nil {
			return nil, nil, err
		}

		stat, err := os.Stat(path)
		if err == nil && !stat.IsDir() {
			return nil, nil, fmt.Errorf("cannot mount volume over existing file, file exists %s", path)
		}

		v, err := createVolume(name, config.VolumeDriver)
		if err != nil {
			return nil, nil, err
		}
		if err := label.Relabel(v.Path(), container.MountLabel, "z"); err != nil {
			return nil, nil, err
		}

		if err := container.copyImagePathContent(v, destination); err != nil {
			return nil, nil, err
		}

		container.addMountPointWithVolume(destination, v, true)
	}
	if err := container.ToDisk(); err != nil {
		return nil, nil, err
	}
	return container, warnings, nil
}

func (daemon *Daemon) GenerateSecurityOpt(ipcMode runconfig.IpcMode, pidMode runconfig.PidMode) ([]string, error) {
	if ipcMode.IsHost() || pidMode.IsHost() {
		return label.DisableSecOpt(), nil
	}
	if ipcContainer := ipcMode.Container(); ipcContainer != "" {
		c, err := daemon.Get(ipcContainer)
		if err != nil {
			return nil, err
		}

		return label.DupSecOpt(c.ProcessLabel), nil
	}
	return nil, nil
}
<|endoftext|>


<commit_before>package sd_test

import (
	"io"
	"testing"
	"time"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/sd"
	"github.com/go-kit/kit/sd/internal/instance"
)

func TestDefaultEndpointer(t *testing.T) {
	var (
		ca = make(closer)
		cb = make(closer)
		c  = map[string]io.Closer{"a": ca, "b": cb}
		f  = func(instance string) (endpoint.Endpoint, io.Closer, error) {
			return endpoint.Nop, c[instance], nil
		}
		instancer = &mockInstancer{
			cache: instance.NewCache(),
		}
	)
	// set initial state
	instancer.Update(sd.Event{Instances: []string{"a", "b"}})

	endpointer := sd.NewEndpointer(instancer, f, log.NewNopLogger(), sd.InvalidateOnError(time.Minute))
	if endpoints, err := endpointer.Endpoints(); err != nil {
		t.Errorf("unepected error %v", err)
	} else if want, have := 2, len(endpoints); want != have {
		t.Errorf("want %d, have %d", want, have)
	}

	instancer.Update(sd.Event{Instances: []string{}})
	select {
	case <-ca:
		t.Logf("endpoint a closed, good")
	case <-time.After(time.Millisecond):
		t.Errorf("didn't close the deleted instance in time")
	}
	select {
	case <-cb:
		t.Logf("endpoint b closed, good")
	case <-time.After(time.Millisecond):
		t.Errorf("didn't close the deleted instance in time")
	}
	if endpoints, err := endpointer.Endpoints(); err != nil {
		t.Errorf("unepected error %v", err)
	} else if want, have := 0, len(endpoints); want != have {
		t.Errorf("want %d, have %d", want, have)
	}

	endpointer.Close()
	instancer.Update(sd.Event{Instances: []string{"a"}})
	// TODO verify that on Close the endpointer fully disconnects from the instancer.
	// Unfortunately, because we use instance.Cache, this test cannot be in the sd package,
	// and therefore does not have access to the endpointer's private members.
}

type mockInstancer struct {
	cache *instance.Cache
}

func (m *mockInstancer) Update(event sd.Event) {
	m.cache.Update(event)
}

func (m *mockInstancer) Register(ch chan<- sd.Event) {
	m.cache.Register(ch)
}

func (m *mockInstancer) Deregister(ch chan<- sd.Event) {
	m.cache.Deregister(ch)
}

type closer chan struct{}

func (c closer) Close() error { close(c); return nil }
<commit_msg>sd: fix TestDefaultEndpointer flake, hopefully<commit_after>package sd_test

import (
	"io"
	"testing"
	"time"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/sd"
	"github.com/go-kit/kit/sd/internal/instance"
)

func TestDefaultEndpointer(t *testing.T) {
	var (
		ca = make(closer)
		cb = make(closer)
		c  = map[string]io.Closer{"a": ca, "b": cb}
		f  = func(instance string) (endpoint.Endpoint, io.Closer, error) {
			return endpoint.Nop, c[instance], nil
		}
		instancer = &mockInstancer{
			cache: instance.NewCache(),
		}
	)
	// set initial state
	instancer.Update(sd.Event{Instances: []string{"a", "b"}})

	endpointer := sd.NewEndpointer(instancer, f, log.NewNopLogger(), sd.InvalidateOnError(time.Minute))

	var (
		endpoints []endpoint.Endpoint
		err       error
	)
	if !within(time.Second, func() bool {
		endpoints, err = endpointer.Endpoints()
		return err == nil && len(endpoints) == 2
	}) {
		t.Errorf("wanted 2 endpoints, got %d (%v)", len(endpoints), err)
	}

	instancer.Update(sd.Event{Instances: []string{}})

	select {
	case <-ca:
		t.Logf("endpoint a closed, good")
	case <-time.After(time.Millisecond):
		t.Errorf("didn't close the deleted instance in time")
	}

	select {
	case <-cb:
		t.Logf("endpoint b closed, good")
	case <-time.After(time.Millisecond):
		t.Errorf("didn't close the deleted instance in time")
	}

	if endpoints, err := endpointer.Endpoints(); err != nil {
		t.Errorf("unepected error %v", err)
	} else if want, have := 0, len(endpoints); want != have {
		t.Errorf("want %d, have %d", want, have)
	}

	endpointer.Close()

	instancer.Update(sd.Event{Instances: []string{"a"}})
	// TODO verify that on Close the endpointer fully disconnects from the instancer.
	// Unfortunately, because we use instance.Cache, this test cannot be in the sd package,
	// and therefore does not have access to the endpointer's private members.
}

type mockInstancer struct {
	cache *instance.Cache
}

func (m *mockInstancer) Update(event sd.Event) {
	m.cache.Update(event)
}

func (m *mockInstancer) Register(ch chan<- sd.Event) {
	m.cache.Register(ch)
}

func (m *mockInstancer) Deregister(ch chan<- sd.Event) {
	m.cache.Deregister(ch)
}

type closer chan struct{}

func (c closer) Close() error { close(c); return nil }

func within(d time.Duration, f func() bool) bool {
	deadline := time.Now().Add(d)
	for time.Now().Before(deadline) {
		if f() {
			return true
		}
		time.Sleep(d / 10)
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Open-RnD Sp. z o.o.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package main

import (
	"chain"
	"flag"
	"fmt"
	"led"
	"os"
	"time"
)

const (
	REVOLUTIONS = 100
	REV_PROC = 100000 * REVOLUTIONS

	usage = "" +
		`Usage of 'blinktest' command
Flags:
	-usefake=[0|1]:   Use fake LEDs
`
)

var (
	use_fake  = flag.Bool("usefake", false, "Use fake LEDs")

	// Raspberry PI LEDs connected via GPIO
	gpio_leds = []led.Led{
		led.NewGPIO(4),
		led.NewGPIO(17),
		led.NewGPIO(27),
		led.NewGPIO(22),
	}

	fake_leds = []led.Led{
		led.NewFake(4),
		led.NewFake(17),
		led.NewFake(24),
		led.NewFake(22),
	}
)

func main() {
	flag.Usage = func() {
		fmt.Println(usage)
		os.Exit(1)
	}

	flag.Parse()

	var leds []led.Led

	if *use_fake {
		fmt.Println("using fake LEDs")
		leds = fake_leds
	} else {
		leds = gpio_leds
	}

	for _, l := range leds {
		l.Init()
	}

	chains := []*chain.Chain{
		chain.New(REV_PROC / 1000, 1000, leds[0]),
		chain.New(REV_PROC / 10000, 10000, leds[1]),
		chain.New(REV_PROC / 25000, 25000, leds[2]),
		chain.New(REV_PROC / 100000, 100000, leds[3]),
	}

	fmt.Println("spawning goroutines")
	// spawn all goroutines
	for _, ch := range chains {
		ch.Spawn()
	}

	time.Sleep(time.Second)
	// start all
	fmt.Println("trigger start")
	for _, ch := range chains {
		ch.Start()
	}

	for _, ch := range chains {
		ch.Wait()
	}
	fmt.Println("finished")

	for _, l := range leds {
		l.Close()
	}
}
<commit_msg>blinktest: fmt update<commit_after>// Copyright (c) 2015 Open-RnD Sp. z o.o.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package main

import (
	"chain"
	"flag"
	"fmt"
	"led"
	"os"
	"time"
)

const (
	REVOLUTIONS = 100
	REV_PROC = 100000 * REVOLUTIONS

	usage = "" +
		`Usage of 'blinktest' command
Flags:
	-usefake=[0|1]:   Use fake LEDs
`
)

var (
	use_fake = flag.Bool("usefake", false, "Use fake LEDs")

	// Raspberry PI LEDs connected via GPIO
	gpio_leds = []led.Led{
		led.NewGPIO(4),
		led.NewGPIO(17),
		led.NewGPIO(27),
		led.NewGPIO(22),
	}

	fake_leds = []led.Led{
		led.NewFake(4),
		led.NewFake(17),
		led.NewFake(24),
		led.NewFake(22),
	}
)

func main() {
	flag.Usage = func() {
		fmt.Println(usage)
		os.Exit(1)
	}

	flag.Parse()

	var leds []led.Led

	if *use_fake {
		fmt.Println("using fake LEDs")
		leds = fake_leds
	} else {
		leds = gpio_leds
	}

	for _, l := range leds {
		l.Init()
	}

	chains := []*chain.Chain{
		chain.New(REV_PROC / 1000, 1000, leds[0]),
		chain.New(REV_PROC / 10000, 10000, leds[1]),
		chain.New(REV_PROC / 25000, 25000, leds[2]),
		chain.New(REV_PROC / 100000, 100000, leds[3]),
	}

	fmt.Println("spawning goroutines")
	// spawn all goroutines
	for _, ch := range chains {
		ch.Spawn()
	}

	time.Sleep(time.Second)
	// start all
	fmt.Println("trigger start")
	for _, ch := range chains {
		ch.Start()
	}

	for _, ch := range chains {
		ch.Wait()
	}
	fmt.Println("finished")

	for _, l := range leds {
		l.Close()
	}
}
<|endoftext|>


<commit_before>package angularjs

import "github.com/gopherjs/gopherjs/js"

type Module struct{ js.Object }

func (m *Module) NewController(name string, constructor func(scope *Scope)) {
	m.Call("controller", name, func(dollar_scope js.Object) {
		constructor(&Scope{dollar_scope})
	})
}

type Scope struct{ js.Object }

func (s *Scope) Apply(f func()) {
	s.Call("$apply", f)
}

func (s *Scope) EvalAsync(f func()) {
	s.Call("$evalAsync", f)
}

type JQueryElement struct{ js.Object }

func (e *JQueryElement) Prop(name string) js.Object {
	return e.Call("prop", name)
}

func (e *JQueryElement) SetProp(name, value interface{}) {
	e.Call("prop", name, value)
}

func (e *JQueryElement) On(events string, handler func(*Event)) {
	e.Call("on", events, func(e js.Object) {
		handler(&Event{Object: e})
	})
}

func (e *JQueryElement) Val() js.Object {
	return e.Call("val")
}

func (e *JQueryElement) SetVal(value interface{}) {
	e.Call("val", value)
}

type Event struct {
	js.Object
	KeyCode int `js:"keyCode"`
}

func (e *Event) PreventDefault() {
	e.Call("preventDefault")
}

func NewModule(name string, requires []string, configFn func()) *Module {
	return &Module{js.Global.Get("angular").Call("module", name, requires, configFn)}
}

func ElementById(id string) *JQueryElement {
	return &JQueryElement{js.Global.Get("angular").Call("element", js.Global.Get("document").Call("getElementById", id))}
}

func Service(name string) js.Object {
	return js.Global.Get("angular").Call("element", js.Global.Get("document")).Call("injector").Call("get", name)
}

type HttpService struct{}

var HTTP = new(HttpService)

func (s *HttpService) Get(url string, callback func(data string, status int)) {
	future := Service("$http").Call("get", url)
	future.Call("success", func(data string, status int, headers js.Object, config js.Object) {
		callback(data, status)
	})
	future.Call("error", func(data string, status int, headers js.Object, config js.Object) {
		callback(data, status)
	})
}
<commit_msg>replaced js.Object by *js.Object<commit_after>package angularjs

import "github.com/gopherjs/gopherjs/js"

type Module struct{ *js.Object }

func (m *Module) NewController(name string, constructor func(scope *Scope)) {
	m.Call("controller", name, func(dollar_scope *js.Object) {
		constructor(&Scope{dollar_scope})
	})
}

type Scope struct{ *js.Object }

func (s *Scope) Apply(f func()) {
	s.Call("$apply", f)
}

func (s *Scope) EvalAsync(f func()) {
	s.Call("$evalAsync", f)
}

type JQueryElement struct{ *js.Object }

func (e *JQueryElement) Prop(name string) *js.Object {
	return e.Call("prop", name)
}

func (e *JQueryElement) SetProp(name, value interface{}) {
	e.Call("prop", name, value)
}

func (e *JQueryElement) On(events string, handler func(*Event)) {
	e.Call("on", events, func(e *js.Object) {
		handler(&Event{Object: e})
	})
}

func (e *JQueryElement) Val() *js.Object {
	return e.Call("val")
}

func (e *JQueryElement) SetVal(value interface{}) {
	e.Call("val", value)
}

type Event struct {
	*js.Object
	KeyCode int `js:"keyCode"`
}

func (e *Event) PreventDefault() {
	e.Call("preventDefault")
}

func NewModule(name string, requires []string, configFn func()) *Module {
	return &Module{js.Global.Get("angular").Call("module", name, requires, configFn)}
}

func ElementById(id string) *JQueryElement {
	return &JQueryElement{js.Global.Get("angular").Call("element", js.Global.Get("document").Call("getElementById", id))}
}

func Service(name string) *js.Object {
	return js.Global.Get("angular").Call("element", js.Global.Get("document")).Call("injector").Call("get", name)
}

type HttpService struct{}

var HTTP = new(HttpService)

func (s *HttpService) Get(url string, callback func(data string, status int)) {
	future := Service("$http").Call("get", url)
	future.Call("success", func(data string, status int, headers *js.Object, config *js.Object) {
		callback(data, status)
	})
	future.Call("error", func(data string, status int, headers *js.Object, config *js.Object) {
		callback(data, status)
	})
}
<|endoftext|>


<commit_before>//
// Copyright 2022 The Ent Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nodeservice

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"path"

	"github.com/google/ent/index"
	"github.com/google/ent/utils"
)

type IndexClient struct {
	BaseURL string
}

func (c IndexClient) Get(ctx context.Context, digest utils.Digest) ([]byte, error) {
	log.Printf("%s\n", string(digest))
	u := c.BaseURL + "/" + path.Join(index.DigestToPath(digest), index.EntryFilename)
	log.Printf("fetching entry from %s\n", u)
	entryRes, err := http.Get(u)
	if err != nil {
		return nil, fmt.Errorf("could not fetch index entry: %w", err)
	}
	entry := index.IndexEntry{}
	err = json.NewDecoder(entryRes.Body).Decode(&entry)
	if err != nil {
		return nil, fmt.Errorf("could not parse index entry as JSON: %w", err)
	}
	log.Printf("parsed entry: %+v", entry)
	targetRes, err := http.Get(entry.URLS[0])
	if err != nil {
		return nil, fmt.Errorf("could not fetch target: %w", err)
	}
	target, err := ioutil.ReadAll(targetRes.Body)
	if err != nil {
		return nil, fmt.Errorf("could not download target: %w", err)
	}
	targetDigest := utils.ComputeDigest(target)
	if targetDigest != digest {
		return nil, fmt.Errorf("digest mismatch, wanted: %q, got %q", digest, targetDigest)
	}
	return target, nil
}

func (c IndexClient) Has(ctx context.Context, h utils.Digest) (bool, error) {
	return false, nil
}
<commit_msg>Fix index errors<commit_after>//
// Copyright 2022 The Ent Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nodeservice

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"path"

	"github.com/google/ent/index"
	"github.com/google/ent/utils"
)

type IndexClient struct {
	BaseURL string
}

func (c IndexClient) Get(ctx context.Context, digest utils.Digest) ([]byte, error) {
	log.Printf("%s\n", string(digest))
	u := c.BaseURL + "/" + path.Join(index.DigestToPath(digest), index.EntryFilename)
	log.Printf("fetching entry from %s\n", u)
	entryRes, err := http.Get(u)
	if err != nil {
		return nil, fmt.Errorf("could not fetch index entry: %w", err)
	}
	if entryRes.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("could not fetch index entry: %s", entryRes.Status)
	}
	entry := index.IndexEntry{}
	err = json.NewDecoder(entryRes.Body).Decode(&entry)
	if err != nil {
		return nil, fmt.Errorf("could not parse index entry as JSON: %w", err)
	}
	log.Printf("parsed entry: %+v", entry)
	targetRes, err := http.Get(entry.URLS[0])
	if err != nil {
		return nil, fmt.Errorf("could not fetch target: %w", err)
	}
	target, err := ioutil.ReadAll(targetRes.Body)
	if err != nil {
		return nil, fmt.Errorf("could not download target: %w", err)
	}
	targetDigest := utils.ComputeDigest(target)
	if targetDigest != digest {
		return nil, fmt.Errorf("digest mismatch, wanted: %q, got %q", digest, targetDigest)
	}
	return target, nil
}

func (c IndexClient) Has(ctx context.Context, h utils.Digest) (bool, error) {
	return false, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"strings"
	"fmt"
	"os"

	"github.com/apex/log"
	"github.com/apex/log/handlers/cli"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var (
	cfgFile string
	logLevel string
	psqlDBStr string
)

var ctx = log.WithFields(log.Fields{
	"env": "production",
})

var RootCmd = &cobra.Command{
	Use:   "proteus-events",
	Short: "I receive events to propagate to probes",
	Long: `Is responsible for receiving events via the admin interface and triggering notifications via proteus-notify`,
}

func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.proteus.yaml)")
	RootCmd.PersistentFlags().StringVarP(&logLevel, "logLevel", "", "info", "Set the log level")
	RootCmd.PersistentFlags().StringP("db-url", "", "", "Set the url of the postgres database (ex. postgres://username:password@host/dbname?sslmode=verify-full)")
	viper.BindPFlag("database.url", RootCmd.PersistentFlags().Lookup("db-url"))
	viper.SetDefault("database.active-probes-table", "active_probes")
	viper.SetDefault("database.probe-updates-table", "probe_updates")
}

func initConfig() {
	if cfgFile != "" { // enable ability to specify config file via flag
		viper.SetConfigFile(cfgFile)
	}

	viper.SetConfigName("proteus-events")
	viper.AddConfigPath("/etc/proteus/")
	viper.AddConfigPath(".")
	viper.AutomaticEnv()

	replacer := strings.NewReplacer("-", "_") // Allows us to defined keys with -, but set them in via env variables with _
	viper.SetEnvKeyReplacer(replacer)

	err := viper.ReadInConfig()
	if err != nil {
			ctx.WithError(err).Error("failed to read config file")
			panic(err)
	}
	ctx.Infof("using config file:", viper.ConfigFileUsed())

	log.SetHandler(cli.Default)
	level, err := log.ParseLevel(logLevel)
	if err != nil {
		fmt.Println("Invalid log level. Must be one of debug, info, warn, error, fatal")
	}
	log.SetLevel(level)
}
<commit_msg>Fix parsing of log-level config file option<commit_after>package cmd

import (
	"strings"
	"fmt"
	"os"

	"github.com/apex/log"
	"github.com/apex/log/handlers/cli"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var (
	cfgFile string
	logLevel string
	psqlDBStr string
)

var ctx = log.WithFields(log.Fields{
	"env": "production",
})

var RootCmd = &cobra.Command{
	Use:   "proteus-events",
	Short: "I receive events to propagate to probes",
	Long: `Is responsible for receiving events via the admin interface and triggering notifications via proteus-notify`,
}

func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.proteus.yaml)")
	RootCmd.PersistentFlags().StringP("log-level", "", "info", "Set the log level")
	RootCmd.PersistentFlags().StringP("db-url", "", "", "Set the url of the postgres database (ex. postgres://username:password@host/dbname?sslmode=verify-full)")
	viper.BindPFlag("database.url", RootCmd.PersistentFlags().Lookup("db-url"))
	viper.BindPFlag("core.log-level", RootCmd.PersistentFlags().Lookup("log-level"))
	viper.SetDefault("database.active-probes-table", "active_probes")
	viper.SetDefault("database.probe-updates-table", "probe_updates")
}

func initConfig() {
	if cfgFile != "" { // enable ability to specify config file via flag
		viper.SetConfigFile(cfgFile)
	}

	viper.SetConfigName("proteus-events")
	viper.AddConfigPath("/etc/proteus/")
	viper.AddConfigPath(".")
	viper.AutomaticEnv()

	replacer := strings.NewReplacer("-", "_") // Allows us to defined keys with -, but set them in via env variables with _
	viper.SetEnvKeyReplacer(replacer)

	err := viper.ReadInConfig()
	if err != nil {
			ctx.WithError(err).Error("failed to read config file")
			panic(err)
	}
	ctx.Infof("using config file:", viper.ConfigFileUsed())

	log.SetHandler(cli.Default)
	level, err := log.ParseLevel(viper.GetString("core.log-level"))
	if err != nil {
		fmt.Println("Invalid log level. Must be one of debug, info, warn, error, fatal")
	}
	log.SetLevel(level)
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	"github.com/peterstace/grayt/grayt"
)

func BenchmarkClassic(b *testing.B) {
	grayt.BenchmarkTraceImage(b, scene())
}
<commit_msg>Remove speed test<commit_after><|endoftext|>


<commit_before>package wellington

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"testing"

	_ "github.com/wellington/wellington/handlers"
)

func decResp(t *testing.T, r io.Reader) Response {
	dec := json.NewDecoder(r)
	var resp Response
	err := dec.Decode(&resp)
	if err != nil {
		t.Errorf("decodeHTTPResponse: %s", err)
	}
	return resp
}

func TestFileHandler(t *testing.T) {

	tempDir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir: %v", err)
	}
	e := "Hello world"
	if err := ioutil.WriteFile(filepath.Join(tempDir, "foo.txt"),
		[]byte(e), 0644); err != nil {
		t.Fatalf("WriteFile: %v", err)
	}

	ts := httptest.NewServer(FileHandler(tempDir))
	defer ts.Close()
	get := func(suffix string) string {
		res, err := http.Get(ts.URL + suffix)
		if err != nil {
			t.Fatalf("Get %s: %v", suffix, err)
		}
		b, err := ioutil.ReadAll(res.Body)
		if err != nil {
			t.Fatalf("ReadAll %s: %v", suffix, err)
		}
		res.Body.Close()
		return string(b)
	}

	if s := get("/build/foo.txt"); e != s {
		t.Fatalf("got %q want %q", s, e)
	}

}

func TestHTTPHandler(t *testing.T) {
	gba := &BuildArgs{}
	u := "http://foo.com"
	hh := http.HandlerFunc(HTTPHandler(gba, u))
	req, err := http.NewRequest("GET", "", nil)
	req.Header.Set("Origin", u)
	if err != nil {
		t.Error(err)
	}
	w := httptest.NewRecorder()
	hh.ServeHTTP(w, req)

	if e := 200; w.Code != e {
		t.Errorf("got: %d wanted: %d", w.Code, e)
	}

	resp := decResp(t, w.Body)
	if e := "request is empty"; resp.Error != e {
		t.Errorf("got: %s wanted: %s", resp, e)
	}

	req, err = http.NewRequest("GET", "",
		bytes.NewBufferString(`div { p { color: red; } }`))
	if err != nil {
		t.Error(err)
	}
	w.Body.Reset()
	hh.ServeHTTP(w, req)

	if e := 200; w.Code != e {
		t.Errorf("got: %d wanted: %d", w.Code, e)
	}
	e := `div p {
  color: red; }
`
	resp = decResp(t, w.Body)
	if resp.Contents != e {
		t.Errorf("got: %s wanted: %s", resp.Contents, e)
	}

	ehead := map[string][]string{
		"Access-Control-Allow-Origin":      []string{"http://foo.com"},
		"Access-Control-Allow-Methods":     []string{"POST, GET, OPTIONS, PUT, DELETE"},
		"Access-Control-Allow-Headers":     []string{"Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token"},
		"Access-Control-Allow-Credentials": []string{"true"},
	}

	for i, h := range w.Header() {

		if ehead[i][0] != h[0] {
			t.Errorf("got:\n%q\nwanted:\n%q", h, ehead[i])
		}
	}

}

func TestHTTPHandler_spritepath(t *testing.T) {
	// TODO: maybe just check that compiler has the correct HTTPPath
	tdir, err := ioutil.TempDir("", "spritepath")
	if err != nil {
		t.Fatal(err)
	}
	gba := &BuildArgs{
		BuildDir: tdir,
		Gen:      filepath.Join(tdir, "im"),
	}
	u := "http://foo.com"
	hh := http.HandlerFunc(HTTPHandler(gba, u))
	w := httptest.NewRecorder()

	req, err := http.NewRequest("POST", "",
		bytes.NewBufferString(`$m: sprite-map("test/img/*.png");
div {
  file: sprite($m, "140");
}`))
	if err != nil {
		t.Error(err)
	}
	w.Body.Reset()
	hh.ServeHTTP(w, req)

	if e := 200; w.Code != e {
		t.Errorf("got: %d wanted: %d", w.Code, e)
	}
	e := `div {
  file: url("http://foo.com/build/20185e.png") 0px -139px; }
`

	resp := decResp(t, w.Body)
	if resp.Contents != e {
		t.Errorf("got: %s wanted: %s", resp.Contents, e)
	}

	if len(resp.Error) > 0 {
		t.Fatal(resp.Error)
	}

	ehead := map[string][]string{
		"Access-Control-Allow-Origin":      []string{"http://foo.com"},
		"Access-Control-Allow-Methods":     []string{"POST, GET, OPTIONS, PUT, DELETE"},
		"Access-Control-Allow-Headers":     []string{"Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token"},
		"Access-Control-Allow-Credentials": []string{"true"},
	}

	for i, h := range w.Header() {

		if ehead[i][0] != h[0] {
			t.Errorf("got:\n%q\nwanted:\n%q", h, ehead[i])
		}
	}

}

func TestHTTPHandler_error(t *testing.T) {
	hh := http.HandlerFunc(HTTPHandler(&BuildArgs{}, ""))
	// nil causes panic, is this a problem?
	req, err := http.NewRequest("GET", "",
		bytes.NewBufferString(`div { p { color: darken(); } };`))
	if err != nil {
		t.Error(err)
	}
	w := httptest.NewRecorder()
	hh.ServeHTTP(w, req)

	if e := 200; w.Code != e {
		t.Errorf("got: %d wanted: %d", w.Code, e)
	}

	e := `Error > stdin:1
required parameter $color is missing in call to Function darken
div { p { color: darken(); } };
`
	resp := decResp(t, w.Body)

	if resp.Error != e {
		t.Errorf("got:\n%s\nwanted:\n%s", resp.Error, e)
	}

	req, err = http.NewRequest("GET", "",
		bytes.NewBufferString(`div { p { color: red; } }`))
	if err != nil {
		t.Error(err)
	}
	w.Body.Reset()
	hh.ServeHTTP(w, req)

	if e := 200; w.Code != e {
		t.Errorf("got: %d wanted: %d", w.Code, e)
	}
	e = `div p {
  color: red; }
`
	resp = decResp(t, w.Body)

	if resp.Contents != e {
		t.Errorf("got:\n%s\nwanted:\n%s", resp.Contents, e)
	}

	// Second run shouldn't have an error in it
}
<commit_msg>gofmt -s<commit_after>package wellington

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"testing"

	_ "github.com/wellington/wellington/handlers"
)

func decResp(t *testing.T, r io.Reader) Response {
	dec := json.NewDecoder(r)
	var resp Response
	err := dec.Decode(&resp)
	if err != nil {
		t.Errorf("decodeHTTPResponse: %s", err)
	}
	return resp
}

func TestFileHandler(t *testing.T) {

	tempDir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir: %v", err)
	}
	e := "Hello world"
	if err := ioutil.WriteFile(filepath.Join(tempDir, "foo.txt"),
		[]byte(e), 0644); err != nil {
		t.Fatalf("WriteFile: %v", err)
	}

	ts := httptest.NewServer(FileHandler(tempDir))
	defer ts.Close()
	get := func(suffix string) string {
		res, err := http.Get(ts.URL + suffix)
		if err != nil {
			t.Fatalf("Get %s: %v", suffix, err)
		}
		b, err := ioutil.ReadAll(res.Body)
		if err != nil {
			t.Fatalf("ReadAll %s: %v", suffix, err)
		}
		res.Body.Close()
		return string(b)
	}

	if s := get("/build/foo.txt"); e != s {
		t.Fatalf("got %q want %q", s, e)
	}

}

func TestHTTPHandler(t *testing.T) {
	gba := &BuildArgs{}
	u := "http://foo.com"
	hh := http.HandlerFunc(HTTPHandler(gba, u))
	req, err := http.NewRequest("GET", "", nil)
	req.Header.Set("Origin", u)
	if err != nil {
		t.Error(err)
	}
	w := httptest.NewRecorder()
	hh.ServeHTTP(w, req)

	if e := 200; w.Code != e {
		t.Errorf("got: %d wanted: %d", w.Code, e)
	}

	resp := decResp(t, w.Body)
	if e := "request is empty"; resp.Error != e {
		t.Errorf("got: %s wanted: %s", resp, e)
	}

	req, err = http.NewRequest("GET", "",
		bytes.NewBufferString(`div { p { color: red; } }`))
	if err != nil {
		t.Error(err)
	}
	w.Body.Reset()
	hh.ServeHTTP(w, req)

	if e := 200; w.Code != e {
		t.Errorf("got: %d wanted: %d", w.Code, e)
	}
	e := `div p {
  color: red; }
`
	resp = decResp(t, w.Body)
	if resp.Contents != e {
		t.Errorf("got: %s wanted: %s", resp.Contents, e)
	}

	ehead := map[string][]string{
		"Access-Control-Allow-Origin":      {"http://foo.com"},
		"Access-Control-Allow-Methods":     {"POST, GET, OPTIONS, PUT, DELETE"},
		"Access-Control-Allow-Headers":     {"Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token"},
		"Access-Control-Allow-Credentials": {"true"},
	}

	for i, h := range w.Header() {

		if ehead[i][0] != h[0] {
			t.Errorf("got:\n%q\nwanted:\n%q", h, ehead[i])
		}
	}

}

func TestHTTPHandler_spritepath(t *testing.T) {
	// TODO: maybe just check that compiler has the correct HTTPPath
	tdir, err := ioutil.TempDir("", "spritepath")
	if err != nil {
		t.Fatal(err)
	}
	gba := &BuildArgs{
		BuildDir: tdir,
		Gen:      filepath.Join(tdir, "im"),
	}
	u := "http://foo.com"
	hh := http.HandlerFunc(HTTPHandler(gba, u))
	w := httptest.NewRecorder()

	req, err := http.NewRequest("POST", "",
		bytes.NewBufferString(`$m: sprite-map("test/img/*.png");
div {
  file: sprite($m, "140");
}`))
	if err != nil {
		t.Error(err)
	}
	w.Body.Reset()
	hh.ServeHTTP(w, req)

	if e := 200; w.Code != e {
		t.Errorf("got: %d wanted: %d", w.Code, e)
	}
	e := `div {
  file: url("http://foo.com/build/20185e.png") 0px -139px; }
`

	resp := decResp(t, w.Body)
	if resp.Contents != e {
		t.Errorf("got: %s wanted: %s", resp.Contents, e)
	}

	if len(resp.Error) > 0 {
		t.Fatal(resp.Error)
	}

	ehead := map[string][]string{
		"Access-Control-Allow-Origin":      {"http://foo.com"},
		"Access-Control-Allow-Methods":     {"POST, GET, OPTIONS, PUT, DELETE"},
		"Access-Control-Allow-Headers":     {"Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token"},
		"Access-Control-Allow-Credentials": {"true"},
	}

	for i, h := range w.Header() {

		if ehead[i][0] != h[0] {
			t.Errorf("got:\n%q\nwanted:\n%q", h, ehead[i])
		}
	}

}

func TestHTTPHandler_error(t *testing.T) {
	hh := http.HandlerFunc(HTTPHandler(&BuildArgs{}, ""))
	// nil causes panic, is this a problem?
	req, err := http.NewRequest("GET", "",
		bytes.NewBufferString(`div { p { color: darken(); } };`))
	if err != nil {
		t.Error(err)
	}
	w := httptest.NewRecorder()
	hh.ServeHTTP(w, req)

	if e := 200; w.Code != e {
		t.Errorf("got: %d wanted: %d", w.Code, e)
	}

	e := `Error > stdin:1
required parameter $color is missing in call to Function darken
div { p { color: darken(); } };
`
	resp := decResp(t, w.Body)

	if resp.Error != e {
		t.Errorf("got:\n%s\nwanted:\n%s", resp.Error, e)
	}

	req, err = http.NewRequest("GET", "",
		bytes.NewBufferString(`div { p { color: red; } }`))
	if err != nil {
		t.Error(err)
	}
	w.Body.Reset()
	hh.ServeHTTP(w, req)

	if e := 200; w.Code != e {
		t.Errorf("got: %d wanted: %d", w.Code, e)
	}
	e = `div p {
  color: red; }
`
	resp = decResp(t, w.Body)

	if resp.Contents != e {
		t.Errorf("got:\n%s\nwanted:\n%s", resp.Contents, e)
	}

	// Second run shouldn't have an error in it
}
<|endoftext|>


<commit_before>package docker

import (
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	titancommon "github.com/iron-io/titan/common"
	"github.com/iron-io/titan/runner/agent"
	"github.com/iron-io/titan/runner/drivers"
	drivercommon "github.com/iron-io/titan/runner/drivers/common"
	"golang.org/x/net/context"
)

type DockerDriver struct {
	conf     *drivercommon.Config
	docker   *docker.Client
	hostname string
	*titancommon.Environment
}

func NewDocker(env *titancommon.Environment, conf *drivercommon.Config) *DockerDriver {
	hostname, err := os.Hostname()
	if err != nil {
		logrus.WithError(err).Fatal("couldn't resolve hostname")
	}

	// docker, err := docker.NewClient(conf.Docker)
	docker, err := docker.NewClientFromEnv()
	if err != nil {
		logrus.WithError(err).Fatal("couldn't create docker client")
	}

	return &DockerDriver{
		conf:        conf,
		docker:      docker,
		hostname:    hostname,
		Environment: env,
	}
}

// Run executes the docker container. If task runs, drivers.RunResult will be returned. If something fails outside the task (ie: Docker), it will return error.
// todo: pass in context
func (drv *DockerDriver) Run(ctx context.Context, task drivers.ContainerTask) (drivers.RunResult, error) {
	container, err := drv.startTask(task)
	if err != nil {
		return nil, err
	}
	defer drv.removeContainer(container)

	sentence := make(chan string, 1)

	go drv.nanny(ctx, container, task, sentence)

	outTasker, errTasker := task.Logger()

	// We may need to capture some lines here using the Head or Tail writers to find substrings regarding memory usage or other Docker errors.
	// IW-125
	mwOut := io.MultiWriter(outTasker)
	mwErr := io.MultiWriter(errTasker)

	// Docker sometimes fails to close the attach response connection even after
	// the container stops, leaving the runner stuck. We use a non-blocking
	// attach so we can sleep a bit after WaitContainer returns and then forcibly
	// close the connection.
	closer, err := drv.docker.AttachToContainerNonBlocking(docker.AttachToContainerOptions{
		Container: container, OutputStream: mwOut, ErrorStream: mwErr,
		Stream: true, Logs: true, Stdout: true, Stderr: true})
	defer closer.Close()
	if err != nil {
		return nil, fmt.Errorf("attach to container: %v", err)
	}

	// It's possible the execution could be finished here, then what? http://docs.docker.com.s3-website-us-east-1.amazonaws.com/engine/reference/api/docker_remote_api_v1.20/#wait-a-container
	exitCode, err := drv.docker.WaitContainer(container)
	time.Sleep(10 * time.Millisecond)
	if err != nil {
		return nil, fmt.Errorf("wait container: %v", err)
	}

	status, err := drv.status(container, exitCode, sentence)

	// TODO: Check stdout/stderr for driver-specific errors like OOM.

	// the err returned above is an error from running user code, so we don't return it from this method.
	return &runResult{
		StatusValue: status,
		Err:         err,
	}, nil
}

func (drv *DockerDriver) startTask(task drivers.ContainerTask) (dockerId string, err error) {
	cID, err := drv.createContainer(task)
	if err != nil {
		return "", err
	}

	startTimer := drv.NewTimer("docker", "start_container", 1.0)
	err = drv.docker.StartContainer(cID, nil)
	startTimer.Measure()
	if err != nil {
		if cID != "" {
			// Remove the created container since we couldn't start it.
			defer drv.removeContainer(cID)
		}
		drv.Inc("docker", "container_start_error", 1, 1.0)
		return "", fmt.Errorf("docker.StartContainer: %v", err)
	}
	return cID, nil
}

func (drv *DockerDriver) createContainer(task drivers.ContainerTask) (string, error) {
	log := logrus.WithFields(logrus.Fields{"image": task.Image()}) // todo: add context fields here, job id, etc.

	if task.Image() == "" {
		return "", errors.New("no image specified, this runner cannot run this")
	}

	var cmd []string
	if task.Command() != "" {
		// TODO: maybe check for spaces or shell meta characters?
		// There's a possibility that the container doesn't have sh.
		cmd = []string{"sh", "-c", task.Command()}
	}

	envvars := make([]string, 0, len(task.EnvVars())+4)
	for name, val := range task.EnvVars() {
		envvars = append(envvars, name+"="+val)
	}

	container := docker.CreateContainerOptions{
		Name: "task-" + task.Id(),
		Config: &docker.Config{
			Env:       envvars,
			Cmd:       cmd,
			Memory:    int64(drv.conf.Memory),
			CPUShares: drv.conf.CPUShares,
			Hostname:  drv.hostname,
			Image:     task.Image(),
			Volumes:   map[string]struct{}{},
		},
		HostConfig: &docker.HostConfig{},
	}

	volumes := task.Volumes()
	for _, mapping := range volumes {
		if len(mapping) != 2 {
			return "", fmt.Errorf("Invalid volume tuple: %v. Tuple must be 2-element", mapping)
		}

		hostDir := mapping[0]
		containerDir := mapping[1]
		container.Config.Volumes[containerDir] = struct{}{}
		mapn := fmt.Sprintf("%s:%s", hostDir, containerDir)
		container.HostConfig.Binds = append(container.HostConfig.Binds, mapn)
		log.WithFields(logrus.Fields{"volumes": mapn}).Debug("setting volumes")
	}

	if wd := task.WorkDir(); wd != "" {
		log.WithFields(logrus.Fields{"wd": wd}).Debug("setting work dir")
		container.Config.WorkingDir = wd
	}

	createTimer := drv.NewTimer("docker", "create_container", 1.0)
	c, err := drv.docker.CreateContainer(container)
	createTimer.Measure()

	if err == docker.ErrNoSuchImage {
		log.Info("could not create container due to missing image, trying to pull...")

		regHost := "docker.io"
		repo, tag := docker.ParseRepositoryTag(task.Image())
		split := strings.Split(repo, "/")
		if len(split) >= 3 {
			// then we have an explicit registry
			regHost = split[0]
		}
		// todo: we should probably move all this auth stuff up a level, don't need to do it for every job
		authConfig := docker.AuthConfiguration{}
		auth := task.Auth()
		if auth != "" {
			log.WithFields(logrus.Fields{"auth": auth}).Debug("Using auth")
			read := strings.NewReader(fmt.Sprintf(`{"%s":{"auth":"%s"}}`, regHost, auth))
			ac, err := docker.NewAuthConfigurations(read)
			if err != nil {
				return "", fmt.Errorf("failed to create auth configurations: %v", err)
			}
			authConfig = ac.Configs[regHost]
		}

		pullTimer := drv.NewTimer("docker", "pull_image", 1.0)
		err = drv.docker.PullImage(docker.PullImageOptions{Repository: repo, Tag: tag}, authConfig)
		pullTimer.Measure()
		if err != nil {
			return "", fmt.Errorf("docker.PullImage: %v", err)
		}

		// should have it now
		createTimer := drv.NewTimer("docker", "create_container", 1.0)
		c, err = drv.docker.CreateContainer(container)
		createTimer.Measure()
		if err != nil {
			logDockerContainerConfig(log, container)
			drv.Inc("docker", "container_create_error", 1, 1.0)
			return "", createContainerErrorf("docker.CreateContainer try 2: %v", err)
		}
	} else if err != nil {
		logDockerContainerConfig(log, container)
		drv.Inc("docker", "container_create_error", 1, 1.0)
		return "", createContainerErrorf("docker.CreateContainer: %v", err)
	}

	return c.ID, nil
}

func createContainerErrorf(format string, err error) error {
	errmsg := fmt.Errorf(format, err)

	if err == docker.ErrConnectionRefused {
		return agent.UnrecoverableError(errmsg)
	}

	return errmsg
}

func (drv *DockerDriver) removeContainer(container string) {
	removeTimer := drv.NewTimer("docker", "remove_container", 1.0)
	// TODO: trap error
	drv.docker.RemoveContainer(docker.RemoveContainerOptions{
		ID: container, Force: true, RemoveVolumes: true})
	removeTimer.Measure()
}

// watch for cancel or timeout and kill process.
func (drv *DockerDriver) nanny(ctx context.Context, container string, task drivers.ContainerTask, sentence chan<- string) {
	select {
	case <-ctx.Done():
		switch ctx.Err() {
		case context.DeadlineExceeded:
			sentence <- drivers.StatusTimeout
			drv.cancel(container)
		case context.Canceled:
			sentence <- drivers.StatusKilled
			drv.cancel(container)
		}
	}
}

func (drv *DockerDriver) status(container string, exitCode int, sentence <-chan string) (string, error) {
	var status string
	var err error
	select {
	case status = <-sentence: // use this if killed / timed out
	default:
		switch exitCode {
		case 0:
			status = drivers.StatusSuccess
		case 137:
			// Probably an OOM kill

			cinfo, err := drv.docker.InspectContainer(container)
			if err != nil {
				logrus.WithError(err).WithFields(logrus.Fields{"container": container}).Error("inspecting container to check for oom")
				drv.Inc("docker", "possible_oom_inspect_container_error", 1, 1.0)
			} else {
				if !cinfo.State.OOMKilled {
					// It is possible that the host itself is running out of memory and
					// the host kernel killed one of the container processes.
					// See: https://github.com/docker/docker/issues/15621
					logrus.WithFields(logrus.Fields{"container": container}).Info("Setting task as OOM killed, but docker disagreed.")
					drv.Inc("docker", "possible_oom_false_alarm", 1, 1.0)
					// TODO: This may be a situation where we would like to shut down the runner completely.
				}
			}

			status = drivers.StatusKilled
			err = drivers.ErrOutOfMemory
		default:
			status = drivers.StatusError
			err = fmt.Errorf("exit code %d", exitCode)
		}
	}
	return status, err
}

// TODO we _sure_ it's dead?
func (drv *DockerDriver) cancel(container string) {
	stopTimer := drv.NewTimer("docker", "stop_container", 1.0)
	drv.docker.StopContainer(container, 5)
	// We will get a large skew due to the by default 5 second wait. Should we log times after subtracting this?
	stopTimer.Measure()
}

func logDockerContainerConfig(log logrus.FieldLogger, container docker.CreateContainerOptions) {
	// envvars are left out because they could have private information.
	log.WithFields(logrus.Fields{
		"command":    container.Config.Cmd,
		"memory":     container.Config.Memory,
		"cpu_shares": container.Config.CPUShares,
		"hostname":   container.Config.Hostname,
		"image":      container.Config.Image,
		"volumes":    container.Config.Volumes,
		"binds":      container.HostConfig.Binds,
	}).Error("Could not create container")
}
<commit_msg>[IW-153] attack of the logs<commit_after>package docker

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	titancommon "github.com/iron-io/titan/common"
	"github.com/iron-io/titan/runner/agent"
	"github.com/iron-io/titan/runner/drivers"
	drivercommon "github.com/iron-io/titan/runner/drivers/common"
	"golang.org/x/net/context"
)

type DockerDriver struct {
	conf     *drivercommon.Config
	docker   *docker.Client
	hostname string
	*titancommon.Environment
}

func NewDocker(env *titancommon.Environment, conf *drivercommon.Config) *DockerDriver {
	hostname, err := os.Hostname()
	if err != nil {
		logrus.WithError(err).Fatal("couldn't resolve hostname")
	}

	// docker, err := docker.NewClient(conf.Docker)
	docker, err := docker.NewClientFromEnv()
	if err != nil {
		logrus.WithError(err).Fatal("couldn't create docker client")
	}

	return &DockerDriver{
		conf:        conf,
		docker:      docker,
		hostname:    hostname,
		Environment: env,
	}
}

// Run executes the docker container. If task runs, drivers.RunResult will be returned. If something fails outside the task (ie: Docker), it will return error.
// todo: pass in context
func (drv *DockerDriver) Run(ctx context.Context, task drivers.ContainerTask) (drivers.RunResult, error) {
	container, err := drv.startTask(task)
	if err != nil {
		return nil, err
	}
	defer drv.removeContainer(container)

	sentence := make(chan string, 1)

	go drv.nanny(ctx, container, task, sentence)

	mwOut, mwErr := task.Logger()

	// Docker sometimes fails to close the attach response connection even after
	// the container stops, leaving the runner stuck. We use a non-blocking
	// attach so we can sleep a bit after WaitContainer returns and then forcibly
	// close the connection.
	closer, err := drv.docker.AttachToContainerNonBlocking(docker.AttachToContainerOptions{
		Container: container, OutputStream: mwOut, ErrorStream: mwErr,
		Stream: true, Logs: true, Stdout: true, Stderr: true})
	defer closer.Close()
	if err != nil {
		return nil, fmt.Errorf("attach to container: %v", err)
	}

	// It's possible the execution could be finished here, then what? http://docs.docker.com.s3-website-us-east-1.amazonaws.com/engine/reference/api/docker_remote_api_v1.20/#wait-a-container
	exitCode, err := drv.docker.WaitContainer(container)
	time.Sleep(10 * time.Millisecond)
	if err != nil {
		return nil, fmt.Errorf("wait container: %v", err)
	}

	status, err := drv.status(container, exitCode, sentence)

	// TODO: Check stdout/stderr for driver-specific errors like OOM.

	// the err returned above is an error from running user code, so we don't return it from this method.
	return &runResult{
		StatusValue: status,
		Err:         err,
	}, nil
}

func (drv *DockerDriver) startTask(task drivers.ContainerTask) (dockerId string, err error) {
	cID, err := drv.createContainer(task)
	if err != nil {
		return "", err
	}

	startTimer := drv.NewTimer("docker", "start_container", 1.0)
	err = drv.docker.StartContainer(cID, nil)
	startTimer.Measure()
	if err != nil {
		if cID != "" {
			// Remove the created container since we couldn't start it.
			defer drv.removeContainer(cID)
		}
		drv.Inc("docker", "container_start_error", 1, 1.0)
		return "", fmt.Errorf("docker.StartContainer: %v", err)
	}
	return cID, nil
}

func (drv *DockerDriver) createContainer(task drivers.ContainerTask) (string, error) {
	log := logrus.WithFields(logrus.Fields{"image": task.Image()}) // todo: add context fields here, job id, etc.

	if task.Image() == "" {
		return "", errors.New("no image specified, this runner cannot run this")
	}

	var cmd []string
	if task.Command() != "" {
		// TODO: maybe check for spaces or shell meta characters?
		// There's a possibility that the container doesn't have sh.
		cmd = []string{"sh", "-c", task.Command()}
	}

	envvars := make([]string, 0, len(task.EnvVars())+4)
	for name, val := range task.EnvVars() {
		envvars = append(envvars, name+"="+val)
	}

	container := docker.CreateContainerOptions{
		Name: "task-" + task.Id(),
		Config: &docker.Config{
			Env:       envvars,
			Cmd:       cmd,
			Memory:    int64(drv.conf.Memory),
			CPUShares: drv.conf.CPUShares,
			Hostname:  drv.hostname,
			Image:     task.Image(),
			Volumes:   map[string]struct{}{},
		},
		HostConfig: &docker.HostConfig{
			LogConfig: docker.LogConfig{
				// NOTE: this will make a docker log with 1 line, using 'none' meant we could not get logs after attaching.
				// attaching to the container will still give the full task log output, this just keeps docker from doubly
				// logging the same thing (in their very inefficient format) internally.
				Type:   "json-file",
				Config: map[string]string{"max-size": "0"},
			},
		},
	}

	volumes := task.Volumes()
	for _, mapping := range volumes {
		if len(mapping) != 2 {
			return "", fmt.Errorf("Invalid volume tuple: %v. Tuple must be 2-element", mapping)
		}

		hostDir := mapping[0]
		containerDir := mapping[1]
		container.Config.Volumes[containerDir] = struct{}{}
		mapn := fmt.Sprintf("%s:%s", hostDir, containerDir)
		container.HostConfig.Binds = append(container.HostConfig.Binds, mapn)
		log.WithFields(logrus.Fields{"volumes": mapn}).Debug("setting volumes")
	}

	if wd := task.WorkDir(); wd != "" {
		log.WithFields(logrus.Fields{"wd": wd}).Debug("setting work dir")
		container.Config.WorkingDir = wd
	}

	createTimer := drv.NewTimer("docker", "create_container", 1.0)
	c, err := drv.docker.CreateContainer(container)
	createTimer.Measure()

	if err == docker.ErrNoSuchImage {
		log.Info("could not create container due to missing image, trying to pull...")

		regHost := "docker.io"
		repo, tag := docker.ParseRepositoryTag(task.Image())
		split := strings.Split(repo, "/")
		if len(split) >= 3 {
			// then we have an explicit registry
			regHost = split[0]
		}
		// todo: we should probably move all this auth stuff up a level, don't need to do it for every job
		authConfig := docker.AuthConfiguration{}
		auth := task.Auth()
		if auth != "" {
			log.WithFields(logrus.Fields{"auth": auth}).Debug("Using auth")
			read := strings.NewReader(fmt.Sprintf(`{"%s":{"auth":"%s"}}`, regHost, auth))
			ac, err := docker.NewAuthConfigurations(read)
			if err != nil {
				return "", fmt.Errorf("failed to create auth configurations: %v", err)
			}
			authConfig = ac.Configs[regHost]
		}

		pullTimer := drv.NewTimer("docker", "pull_image", 1.0)
		err = drv.docker.PullImage(docker.PullImageOptions{Repository: repo, Tag: tag}, authConfig)
		pullTimer.Measure()
		if err != nil {
			return "", fmt.Errorf("docker.PullImage: %v", err)
		}

		// should have it now
		createTimer := drv.NewTimer("docker", "create_container", 1.0)
		c, err = drv.docker.CreateContainer(container)
		createTimer.Measure()
		if err != nil {
			logDockerContainerConfig(log, container)
			drv.Inc("docker", "container_create_error", 1, 1.0)
			return "", createContainerErrorf("docker.CreateContainer try 2: %v", err)
		}
	} else if err != nil {
		logDockerContainerConfig(log, container)
		drv.Inc("docker", "container_create_error", 1, 1.0)
		return "", createContainerErrorf("docker.CreateContainer: %v", err)
	}

	return c.ID, nil
}

func createContainerErrorf(format string, err error) error {
	errmsg := fmt.Errorf(format, err)

	if err == docker.ErrConnectionRefused {
		return agent.UnrecoverableError(errmsg)
	}

	return errmsg
}

func (drv *DockerDriver) removeContainer(container string) {
	removeTimer := drv.NewTimer("docker", "remove_container", 1.0)
	// TODO: trap error
	drv.docker.RemoveContainer(docker.RemoveContainerOptions{
		ID: container, Force: true, RemoveVolumes: true})
	removeTimer.Measure()
}

// watch for cancel or timeout and kill process.
func (drv *DockerDriver) nanny(ctx context.Context, container string, task drivers.ContainerTask, sentence chan<- string) {
	select {
	case <-ctx.Done():
		switch ctx.Err() {
		case context.DeadlineExceeded:
			sentence <- drivers.StatusTimeout
			drv.cancel(container)
		case context.Canceled:
			sentence <- drivers.StatusKilled
			drv.cancel(container)
		}
	}
}

func (drv *DockerDriver) status(container string, exitCode int, sentence <-chan string) (string, error) {
	var status string
	var err error
	select {
	case status = <-sentence: // use this if killed / timed out
	default:
		switch exitCode {
		case 0:
			status = drivers.StatusSuccess
		case 137:
			// Probably an OOM kill

			cinfo, err := drv.docker.InspectContainer(container)
			if err != nil {
				logrus.WithError(err).WithFields(logrus.Fields{"container": container}).Error("inspecting container to check for oom")
				drv.Inc("docker", "possible_oom_inspect_container_error", 1, 1.0)
			} else {
				if !cinfo.State.OOMKilled {
					// It is possible that the host itself is running out of memory and
					// the host kernel killed one of the container processes.
					// See: https://github.com/docker/docker/issues/15621
					logrus.WithFields(logrus.Fields{"container": container}).Info("Setting task as OOM killed, but docker disagreed.")
					drv.Inc("docker", "possible_oom_false_alarm", 1, 1.0)
					// TODO: This may be a situation where we would like to shut down the runner completely.
				}
			}

			status = drivers.StatusKilled
			err = drivers.ErrOutOfMemory
		default:
			status = drivers.StatusError
			err = fmt.Errorf("exit code %d", exitCode)
		}
	}
	return status, err
}

// TODO we _sure_ it's dead?
func (drv *DockerDriver) cancel(container string) {
	stopTimer := drv.NewTimer("docker", "stop_container", 1.0)
	drv.docker.StopContainer(container, 5)
	// We will get a large skew due to the by default 5 second wait. Should we log times after subtracting this?
	stopTimer.Measure()
}

func logDockerContainerConfig(log logrus.FieldLogger, container docker.CreateContainerOptions) {
	// envvars are left out because they could have private information.
	log.WithFields(logrus.Fields{
		"command":    container.Config.Cmd,
		"memory":     container.Config.Memory,
		"cpu_shares": container.Config.CPUShares,
		"hostname":   container.Config.Hostname,
		"image":      container.Config.Image,
		"volumes":    container.Config.Volumes,
		"binds":      container.HostConfig.Binds,
	}).Error("Could not create container")
}
<|endoftext|>