<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package google

import (
	"net/http"

	"code.google.com/p/goauth2/oauth"
	"code.google.com/p/goauth2/oauth/jwt"
	"code.google.com/p/google-api-go-client/compute/v1"
	"github.com/juju/errors"
)

const (
	driverScopes = "https://www.googleapis.com/auth/compute " +
		"https://www.googleapis.com/auth/devstorage.full_control"

	tokenURL = "https://accounts.google.com/o/oauth2/token"

	authURL = "https://accounts.google.com/o/oauth2/auth"
)

// Auth holds the information needed to authenticate on GCE.
type Auth struct {
	// ClientID is the GCE account's OAuth ID. It is part of the OAuth
	// config used in the OAuth-wrapping network transport.
	ClientID string
	// ClientEmail is the email address associatd with the GCE account.
	// It is used to generate a new OAuth token to use in the
	// OAuth-wrapping network transport.
	ClientEmail string
	// PrivateKey is the private key that matches the public key
	// associatd with the GCE account. It is used to generate a new
	// OAuth token to use in the OAuth-wrapping network transport.
	PrivateKey []byte
}

// newTransport builds a new network transport that wraps requests
// with the GCE-required OAuth authentication/authorization. The
// transport is built using the Auth values. The following GCE access
// scopes are used:
//   https://www.googleapis.com/auth/compute
// 	 https://www.googleapis.com/auth/devstorage.full_control
func (ga Auth) newTransport() (*oauth.Transport, error) {
	token, err := newToken(ga, driverScopes)
	if err != nil {
		return nil, errors.Trace(err)
	}

	transport := oauth.Transport{
		Config: &oauth.Config{
			ClientId: ga.ClientID,
			Scope:    driverScopes,
			TokenURL: tokenURL,
			AuthURL:  authURL,
		},
		Token: token,
	}
	return &transport, nil
}

// newToken generates a new OAuth token for use in the OAuth-wrapping
// network transport and returns it. This involves network calls to the
// GCE OAuth API.
var newToken = func(auth Auth, scopes string) (*oauth.Token, error) {
	jtok := jwt.NewToken(auth.ClientEmail, scopes, auth.PrivateKey)
	jtok.ClaimSet.Aud = tokenURL

	token, err := jtok.Assert(&http.Client{})
	if err != nil {
		msg := "retrieving auth token for %s"
		return nil, errors.Annotatef(err, msg, auth.ClientEmail)
	}
	return token, nil
}

// newConnection opens a new low-level connection to the GCE API using
// the Auth's data and returns it. This includes building the
// OAuth-wrapping network transport.
func (ga Auth) newConnection() (*compute.Service, error) {
	transport, err := ga.newTransport()
	if err != nil {
		return nil, errors.Trace(err)
	}
	service, err := newService(transport)
	return service, errors.Trace(err)
}

// newService is a simple wrapper around compute.New for use in testing.
var newService = func(transport *oauth.Transport) (*compute.Service, error) {
	return compute.New(transport.Client())
}
<commit_msg>Fix mixed tabs and spaces.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package google

import (
	"net/http"

	"code.google.com/p/goauth2/oauth"
	"code.google.com/p/goauth2/oauth/jwt"
	"code.google.com/p/google-api-go-client/compute/v1"
	"github.com/juju/errors"
)

const (
	driverScopes = "https://www.googleapis.com/auth/compute " +
		"https://www.googleapis.com/auth/devstorage.full_control"

	tokenURL = "https://accounts.google.com/o/oauth2/token"

	authURL = "https://accounts.google.com/o/oauth2/auth"
)

// Auth holds the information needed to authenticate on GCE.
type Auth struct {
	// ClientID is the GCE account's OAuth ID. It is part of the OAuth
	// config used in the OAuth-wrapping network transport.
	ClientID string
	// ClientEmail is the email address associatd with the GCE account.
	// It is used to generate a new OAuth token to use in the
	// OAuth-wrapping network transport.
	ClientEmail string
	// PrivateKey is the private key that matches the public key
	// associatd with the GCE account. It is used to generate a new
	// OAuth token to use in the OAuth-wrapping network transport.
	PrivateKey []byte
}

// newTransport builds a new network transport that wraps requests
// with the GCE-required OAuth authentication/authorization. The
// transport is built using the Auth values. The following GCE access
// scopes are used:
//   https://www.googleapis.com/auth/compute
//   https://www.googleapis.com/auth/devstorage.full_control
func (ga Auth) newTransport() (*oauth.Transport, error) {
	token, err := newToken(ga, driverScopes)
	if err != nil {
		return nil, errors.Trace(err)
	}

	transport := oauth.Transport{
		Config: &oauth.Config{
			ClientId: ga.ClientID,
			Scope:    driverScopes,
			TokenURL: tokenURL,
			AuthURL:  authURL,
		},
		Token: token,
	}
	return &transport, nil
}

// newToken generates a new OAuth token for use in the OAuth-wrapping
// network transport and returns it. This involves network calls to the
// GCE OAuth API.
var newToken = func(auth Auth, scopes string) (*oauth.Token, error) {
	jtok := jwt.NewToken(auth.ClientEmail, scopes, auth.PrivateKey)
	jtok.ClaimSet.Aud = tokenURL

	token, err := jtok.Assert(&http.Client{})
	if err != nil {
		msg := "retrieving auth token for %s"
		return nil, errors.Annotatef(err, msg, auth.ClientEmail)
	}
	return token, nil
}

// newConnection opens a new low-level connection to the GCE API using
// the Auth's data and returns it. This includes building the
// OAuth-wrapping network transport.
func (ga Auth) newConnection() (*compute.Service, error) {
	transport, err := ga.newTransport()
	if err != nil {
		return nil, errors.Trace(err)
	}
	service, err := newService(transport)
	return service, errors.Trace(err)
}

// newService is a simple wrapper around compute.New for use in testing.
var newService = func(transport *oauth.Transport) (*compute.Service, error) {
	return compute.New(transport.Client())
}
<|endoftext|>


<commit_before>//
// Copyright 2016-2018 Bryan T. Meyers <bmeyers@datadrake.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/DataDrake/cuppa/config"
	"github.com/DataDrake/cuppa/results"
	"net/http"
	"os"
	"strings"
	"time"
)

// GraphQLAPI is the location of the GraphQL Endpoint
const GraphQLAPI = "https://api.github.com/graphql"

// RepoQueryFormat is the text format for the necessary GraphQL query
const RepoQueryFormat = `
query {
    repository(owner: "%s", name: "%s") {
        releases (last: %d) {
            nodes {
                name
                publishedAt
                isPrerelease
                tag {
                    name
                }
            }
        }
        refs (refPrefix: "refs/tags/", last: %d){
            nodes {
                name
            }
        }
    }
}
`

// RepoQuery is the JSON payload for this request
type RepoQuery struct {
	Query string `json:"query"`
}

// RepoQueryResult is the JSON payload of the response
type RepoQueryResult struct {
	Data struct {
		Repository struct {
			Releases struct {
				Nodes []struct {
					Name         string `json:"name"`
					PublishedAt  string `json:"publishedAt"`
					IsPrerelease bool   `json:"isPrerelease"`
					Tag          struct {
						Name string `json:"name"`
					} `json:"tag"`
				} `json:"nodes"`
			} `json:"releases"`
			Refs struct {
				Nodes []struct {
					Name string `json:"name"`
				} `json:"nodes"`
			} `json:"refs"`
		} `json:"repository"`
	} `json:"data"`
}

// Convert turns a RepoQueryResult into a Cuppa ResultSet
func (rqr RepoQueryResult) Convert(name string) (rs *results.ResultSet) {
	rs = results.NewResultSet(name)
	if nodes := rqr.Data.Repository.Releases.Nodes; len(nodes) > 0 {
		for _, node := range nodes {
			if node.IsPrerelease {
				continue
			}
			published, _ := time.Parse(time.RFC3339, node.PublishedAt)
			r := results.NewResult(node.Name, node.Tag.Name, fmt.Sprintf(SourceFormat, name, node.Tag.Name), published)
			rs.AddResult(r)
		}
	}
	if rs.Len() == 0 {
		nodes := rqr.Data.Repository.Refs.Nodes
		for _, node := range nodes {
			r := results.NewResult(name, node.Name, fmt.Sprintf(SourceFormat, name, node.Name), time.Time{})
			rs.AddResult(r)
		}
	}
	return
}

// GetReleases gets a number of releases for a given repo
func (c Provider) GetReleases(name string, max int) (rs *results.ResultSet, s results.Status) {
	names := strings.Split(name, "/")

	query := RepoQuery{
		Query: fmt.Sprintf(RepoQueryFormat, names[0], names[1], max, max),
	}
	buff := new(bytes.Buffer)
	enc := json.NewEncoder(buff)
	err := enc.Encode(&query)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		s = results.Unavailable
		return
	}
	// Query the API
	req, _ := http.NewRequest("POST", GraphQLAPI, buff)
	if key := config.Global.Github.Key; len(key) > 0 {
		req.Header["Authorization"] = []string{"token " + key}
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		s = results.Unavailable
		return
	}
	defer resp.Body.Close()
	// Translate Status Code
	switch resp.StatusCode {
	case 200:
		s = results.OK
	case 404:
		s = results.NotFound
		return
	default:
		s = results.Unavailable
	}

	// Fail if not OK
	if s != results.OK {
		return
	}

	dec := json.NewDecoder(resp.Body)
	rqr := &RepoQueryResult{}
	err = dec.Decode(rqr)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		s = results.Unavailable
		return
	}
	rs = rqr.Convert(name)
	return
}
<commit_msg>Switch to tags-first approach to GitHub sources<commit_after>//
// Copyright 2016-2018 Bryan T. Meyers <bmeyers@datadrake.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/DataDrake/cuppa/config"
	"github.com/DataDrake/cuppa/results"
	"net/http"
	"os"
	"strings"
	"time"
)

// GraphQLAPI is the location of the GraphQL Endpoint
const GraphQLAPI = "https://api.github.com/graphql"

// RepoQueryFormat is the text format for the necessary GraphQL query
const RepoQueryFormat = `
query {
    repository(owner: "%s", name: "%s") {
        releases (last: %d) {
            nodes {
                name
                publishedAt
                isPrerelease
                tag {
                    name
                }
            }
        }
        refs (refPrefix: "refs/tags/", last: %d){
            nodes {
                name
            }
        }
    }
}
`

// RepoQuery is the JSON payload for this request
type RepoQuery struct {
	Query string `json:"query"`
}

// RepoQueryResult is the JSON payload of the response
type RepoQueryResult struct {
	Data struct {
		Repository struct {
			Releases struct {
				Nodes []struct {
					Name         string `json:"name"`
					PublishedAt  string `json:"publishedAt"`
					IsPrerelease bool   `json:"isPrerelease"`
					Tag          struct {
						Name string `json:"name"`
					} `json:"tag"`
				} `json:"nodes"`
			} `json:"releases"`
			Refs struct {
				Nodes []struct {
					Name string `json:"name"`
				} `json:"nodes"`
			} `json:"refs"`
		} `json:"repository"`
	} `json:"data"`
}

// Convert turns a RepoQueryResult into a Cuppa ResultSet
func (rqr RepoQueryResult) Convert(name string) (rs *results.ResultSet) {
	rs = results.NewResultSet(name)
	for _, tag := range rqr.Data.Repository.Refs.Nodes {
        pre   := false
        var published time.Time
		for _, node := range rqr.Data.Repository.Releases.Nodes {
            if node.Tag.Name == tag.Name {
			    if node.IsPrerelease {
				    pre = true
			    }
    			published, _ = time.Parse(time.RFC3339, node.PublishedAt)
            }
		}
        if pre {
            continue
        }
        r := results.NewResult(name, tag.Name, fmt.Sprintf(SourceFormat, name, tag.Name), published)
	    rs.AddResult(r)
	}
	return
}

// GetReleases gets a number of releases for a given repo
func (c Provider) GetReleases(name string, max int) (rs *results.ResultSet, s results.Status) {
	names := strings.Split(name, "/")

	query := RepoQuery{
		Query: fmt.Sprintf(RepoQueryFormat, names[0], names[1], max, max),
	}
	buff := new(bytes.Buffer)
	enc := json.NewEncoder(buff)
	err := enc.Encode(&query)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		s = results.Unavailable
		return
	}
	// Query the API
	req, _ := http.NewRequest("POST", GraphQLAPI, buff)
	if key := config.Global.Github.Key; len(key) > 0 {
		req.Header["Authorization"] = []string{"token " + key}
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		s = results.Unavailable
		return
	}
	defer resp.Body.Close()
	// Translate Status Code
	switch resp.StatusCode {
	case 200:
		s = results.OK
	case 404:
		s = results.NotFound
		return
	default:
		s = results.Unavailable
	}

	// Fail if not OK
	if s != results.OK {
		return
	}

	dec := json.NewDecoder(resp.Body)
	rqr := &RepoQueryResult{}
	err = dec.Decode(rqr)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		s = results.Unavailable
		return
	}
	rs = rqr.Convert(name)
	return
}
<|endoftext|>


<commit_before>package input

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"expvar"
	"fmt"
	"io"
	"net"
	"strings"
	"sync/atomic"
	"time"
)

var sequenceNumber int64
var stats = expvar.NewMap("input")

func init() {
	sequenceNumber = time.Now().UnixNano()
}

const (
	newlineTimeout = time.Duration(1000 * time.Millisecond)
	msgBufSize     = 256
)

// Collector specifies the interface all network collectors must implement.
type Collector interface {
	Start(chan<- *Event) error
	Addr() net.Addr
}

// TCPCollector represents a network collector that accepts and handler TCP connections.
type TCPCollector struct {
	iface  string
	format string

	addr      net.Addr
	tlsConfig *tls.Config
}

// UDPCollector represents a network collector that accepts UDP packets.
type UDPCollector struct {
	format string
	addr   *net.UDPAddr
}

// NewCollector returns a network collector of the specified type, that will bind
// to the given inteface on Start(). If config is non-nil, a secure Collector will
// be returned. Secure Collectors require the protocol be TCP.
func NewCollector(proto, iface, format string, tlsConfig *tls.Config) (Collector, error) {
	// Verify that a parser can be instantiated. The actual parser that is used will
	// be created by the connection handler.
	_, err := NewParser(format)
	if err != nil {
		return nil, err
	}

	if strings.ToLower(proto) == "tcp" {
		return &TCPCollector{
			iface:     iface,
			format:    format,
			tlsConfig: tlsConfig,
		}, nil
	} else if strings.ToLower(proto) == "udp" {
		addr, err := net.ResolveUDPAddr("udp", iface)
		if err != nil {
			return nil, err
		}

		return &UDPCollector{addr: addr, format: format}, nil
	}
	return nil, fmt.Errorf("unsupport collector protocol")
}

// Start instructs the TCPCollector to bind to the interface and accept connections.
func (s *TCPCollector) Start(c chan<- *Event) error {
	var ln net.Listener
	var err error
	if s.tlsConfig == nil {
		ln, err = net.Listen("tcp", s.iface)
	} else {
		ln, err = tls.Listen("tcp", s.iface, s.tlsConfig)
	}

	if err != nil {
		return err
	}

	s.addr = ln.Addr()

	go func() {
		for {
			conn, err := ln.Accept()
			if err != nil {
				continue
			}
			go s.handleConnection(conn, c)
		}
	}()
	return nil
}

// Addr returns the net.Addr that the Collector is bound to, in a race-say manner.
func (s *TCPCollector) Addr() net.Addr {
	return s.addr
}

func (s *TCPCollector) handleConnection(conn net.Conn, c chan<- *Event) {
	stats.Add("tcpConnections", 1)
	defer func() {
		stats.Add("tcpConnections", -1)
		conn.Close()
	}()

	parser, err := NewParser(s.format)
	if err != nil {
		panic(fmt.Sprintf("failed to create TCP connection parser:%s", err.Error()))
	}

	delimiter := NewSyslogDelimiter(msgBufSize)
	reader := bufio.NewReader(conn)
	var log string
	var match bool

	for {
		conn.SetReadDeadline(time.Now().Add(newlineTimeout))
		b, err := reader.ReadByte()
		if err != nil {
			stats.Add("tcpConnReadError", 1)
			if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
				stats.Add("tcpConnReadTimeout", 1)
			} else if err == io.EOF {
				stats.Add("tcpConnReadEOF", 1)
			} else {
				stats.Add("tcpConnUnrecoverError", 1)
				return
			}

			log, match = delimiter.Vestige()
		} else {
			stats.Add("tcpBytesRead", 1)
			log, match = delimiter.Push(b)
		}

		// Log line available?
		if match {
			stats.Add("tcpEventsRx", 1)
			if parser.Parse(bytes.NewBufferString(log).Bytes()) {
				c <- &Event{
					Text:          string(parser.Raw),
					Parsed:        parser.Result,
					ReceptionTime: time.Now().UTC(),
					Sequence:      atomic.AddInt64(&sequenceNumber, 1),
					SourceIP:      conn.RemoteAddr().String(),
				}
			}
		}

		// Was the connection closed?
		if err == io.EOF {
			return
		}
	}
}

// Start instructs the UDPCollector to start reading packets from the interface.
func (s *UDPCollector) Start(c chan<- *Event) error {
	conn, err := net.ListenUDP("udp", s.addr)
	if err != nil {
		return err
	}

	parser, err := NewParser(s.format)
	if err != nil {
		panic(fmt.Sprintf("failed to create UDP parser:%s", err.Error()))
	}

	go func() {
		buf := make([]byte, msgBufSize)
		for {
			n, addr, err := conn.ReadFromUDP(buf)
			stats.Add("udpBytesRead", int64(n))
			if err != nil {
				continue
			}
			log := strings.Trim(string(buf[:n]), "\r\n")
			if parser.Parse(bytes.NewBufferString(log).Bytes()) {
				c <- &Event{
					Text:          log,
					Parsed:        parser.Result,
					ReceptionTime: time.Now().UTC(),
					Sequence:      atomic.AddInt64(&sequenceNumber, 1),
					SourceIP:      addr.String(),
				}
			}
			stats.Add("udpEventsRx", 1)
		}
	}()
	return nil
}

// Addr returns the net.Addr to which the UDP collector is bound.
func (s *UDPCollector) Addr() net.Addr {
	return s.addr
}
<commit_msg>Tighten up blank lines<commit_after>package input

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"expvar"
	"fmt"
	"io"
	"net"
	"strings"
	"sync/atomic"
	"time"
)

var sequenceNumber int64
var stats = expvar.NewMap("input")

func init() {
	sequenceNumber = time.Now().UnixNano()
}

const (
	newlineTimeout = time.Duration(1000 * time.Millisecond)
	msgBufSize     = 256
)

// Collector specifies the interface all network collectors must implement.
type Collector interface {
	Start(chan<- *Event) error
	Addr() net.Addr
}

// TCPCollector represents a network collector that accepts and handler TCP connections.
type TCPCollector struct {
	iface  string
	format string

	addr      net.Addr
	tlsConfig *tls.Config
}

// UDPCollector represents a network collector that accepts UDP packets.
type UDPCollector struct {
	format string
	addr   *net.UDPAddr
}

// NewCollector returns a network collector of the specified type, that will bind
// to the given inteface on Start(). If config is non-nil, a secure Collector will
// be returned. Secure Collectors require the protocol be TCP.
func NewCollector(proto, iface, format string, tlsConfig *tls.Config) (Collector, error) {
	// Verify that a parser can be instantiated. The actual parser that is used will
	// be created by the connection handler.
	_, err := NewParser(format)
	if err != nil {
		return nil, err
	}

	if strings.ToLower(proto) == "tcp" {
		return &TCPCollector{
			iface:     iface,
			format:    format,
			tlsConfig: tlsConfig,
		}, nil
	} else if strings.ToLower(proto) == "udp" {
		addr, err := net.ResolveUDPAddr("udp", iface)
		if err != nil {
			return nil, err
		}

		return &UDPCollector{addr: addr, format: format}, nil
	}
	return nil, fmt.Errorf("unsupport collector protocol")
}

// Start instructs the TCPCollector to bind to the interface and accept connections.
func (s *TCPCollector) Start(c chan<- *Event) error {
	var ln net.Listener
	var err error
	if s.tlsConfig == nil {
		ln, err = net.Listen("tcp", s.iface)
	} else {
		ln, err = tls.Listen("tcp", s.iface, s.tlsConfig)
	}
	if err != nil {
		return err
	}
	s.addr = ln.Addr()

	go func() {
		for {
			conn, err := ln.Accept()
			if err != nil {
				continue
			}
			go s.handleConnection(conn, c)
		}
	}()
	return nil
}

// Addr returns the net.Addr that the Collector is bound to, in a race-say manner.
func (s *TCPCollector) Addr() net.Addr {
	return s.addr
}

func (s *TCPCollector) handleConnection(conn net.Conn, c chan<- *Event) {
	stats.Add("tcpConnections", 1)
	defer func() {
		stats.Add("tcpConnections", -1)
		conn.Close()
	}()

	parser, err := NewParser(s.format)
	if err != nil {
		panic(fmt.Sprintf("failed to create TCP connection parser:%s", err.Error()))
	}

	delimiter := NewSyslogDelimiter(msgBufSize)
	reader := bufio.NewReader(conn)
	var log string
	var match bool

	for {
		conn.SetReadDeadline(time.Now().Add(newlineTimeout))
		b, err := reader.ReadByte()
		if err != nil {
			stats.Add("tcpConnReadError", 1)
			if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
				stats.Add("tcpConnReadTimeout", 1)
			} else if err == io.EOF {
				stats.Add("tcpConnReadEOF", 1)
			} else {
				stats.Add("tcpConnUnrecoverError", 1)
				return
			}

			log, match = delimiter.Vestige()
		} else {
			stats.Add("tcpBytesRead", 1)
			log, match = delimiter.Push(b)
		}

		// Log line available?
		if match {
			stats.Add("tcpEventsRx", 1)
			if parser.Parse(bytes.NewBufferString(log).Bytes()) {
				c <- &Event{
					Text:          string(parser.Raw),
					Parsed:        parser.Result,
					ReceptionTime: time.Now().UTC(),
					Sequence:      atomic.AddInt64(&sequenceNumber, 1),
					SourceIP:      conn.RemoteAddr().String(),
				}
			}
		}

		// Was the connection closed?
		if err == io.EOF {
			return
		}
	}
}

// Start instructs the UDPCollector to start reading packets from the interface.
func (s *UDPCollector) Start(c chan<- *Event) error {
	conn, err := net.ListenUDP("udp", s.addr)
	if err != nil {
		return err
	}

	parser, err := NewParser(s.format)
	if err != nil {
		panic(fmt.Sprintf("failed to create UDP parser:%s", err.Error()))
	}

	go func() {
		buf := make([]byte, msgBufSize)
		for {
			n, addr, err := conn.ReadFromUDP(buf)
			stats.Add("udpBytesRead", int64(n))
			if err != nil {
				continue
			}
			log := strings.Trim(string(buf[:n]), "\r\n")
			if parser.Parse(bytes.NewBufferString(log).Bytes()) {
				c <- &Event{
					Text:          log,
					Parsed:        parser.Result,
					ReceptionTime: time.Now().UTC(),
					Sequence:      atomic.AddInt64(&sequenceNumber, 1),
					SourceIP:      addr.String(),
				}
			}
			stats.Add("udpEventsRx", 1)
		}
	}()
	return nil
}

// Addr returns the net.Addr to which the UDP collector is bound.
func (s *UDPCollector) Addr() net.Addr {
	return s.addr
}
<|endoftext|>


<commit_before>package install

import (
	"bytes"
	"fmt"
	"sort"

	"github.com/hashicorp/go-version"
)

// Memoizes cluster info for install operations
var Clusters = map[string]*SyncedCluster{}

var installCmds = map[string]string{
	"cassandra": `
echo "deb http://www.apache.org/dist/cassandra/debian 311x main" | \
	sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list;
curl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -;
sudo apt-get update;
sudo apt-get install -y cassandra;
sudo service cassandra stop;
`,

	"confluent": `
sudo apt-get update;
sudo apt-get install -y default-jdk-headless;
curl https://packages.confluent.io/archive/5.0/confluent-oss-5.0.0-2.11.tar.gz | sudo tar -C /usr/local -xz;
sudo ln -s /usr/local/confluent-5.0.0 /usr/local/confluent;
`,

	"docker": `
sudo apt-get update;
sudo apt-get install  -y \
    apt-transport-https \
    ca-certificates \
    curl \
    software-properties-common;
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -;
sudo add-apt-repository \
   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
   $(lsb_release -cs) \
   stable";

sudo apt-get update;
sudo apt-get install  -y docker-ce;
`,

	"gcc": `
sudo apt-get update;
sudo apt-get install -y gcc;
`,

	// graphviz and rlwrap are useful for pprof
	"go": `
sudo apt-get update;
sudo apt-get install -y graphviz rlwrap;

curl https://dl.google.com/go/go1.11.linux-amd64.tar.gz | sudo tar -C /usr/local -xz;
echo 'export PATH=$PATH:/usr/local/go/bin' | sudo tee /etc/profile.d/go.sh > /dev/null;
sudo chmod +x /etc/profile.d/go.sh;
`,

	"haproxy": `
sudo apt-get update;
sudo apt-get install -y haproxy;
`,

	"ntp": `
sudo apt-get update;
sudo apt-get install -y \
  ntp \
  ntpdate;
`,

	"tools": `
sudo apt-get update;
sudo apt-get install -y \
  fio \
  iftop \
  iotop \
  sysstat \
  linux-tools-common \
  linux-tools-4.10.0-35-generic \
  linux-cloud-tools-4.10.0-35-generic;
`,

	"zfs": `
sudo apt-get update;
sudo apt-get install -y \
  zfsutils-linux;
`,
}

func SortedCmds() []string {
	cmds := make([]string, 0, len(installCmds))
	for cmd := range installCmds {
		cmds = append(cmds, cmd)
	}
	sort.Strings(cmds)
	return cmds
}

func Install(c *SyncedCluster, args []string) error {
	do := func(title, cmd string) error {
		var buf bytes.Buffer
		err := c.Run(&buf, &buf, c.Nodes, "installing "+title, cmd)
		if err != nil {
			fmt.Print(buf.String())
		}
		return err
	}

	for _, arg := range args {
		cmd, ok := installCmds[arg]
		if !ok {
			return fmt.Errorf("unknown tool %q", arg)
		}

		if err := do(arg, cmd); err != nil {
			return err
		}
	}
	return nil
}

func VersionSatifies(v *version.Version, constraintString string) bool {
	constraints, err := version.NewConstraint(constraintString)
	if err != nil {
		panic(err)
	}
	return constraints.Check(v)
}
<commit_msg>add `roachprod install x:y charybdefs`<commit_after>package install

import (
	"bytes"
	"fmt"
	"sort"

	"github.com/hashicorp/go-version"
)

// Memoizes cluster info for install operations
var Clusters = map[string]*SyncedCluster{}

var installCmds = map[string]string{
	"cassandra": `
echo "deb http://www.apache.org/dist/cassandra/debian 311x main" | \
	sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list;
curl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -;
sudo apt-get update;
sudo apt-get install -y cassandra;
sudo service cassandra stop;
`,

	"charybdefs": `
  thrift_dir="/opt/thrift"

  if [ ! -f "/usr/bin/thrift" ]; then
	sudo apt-get update;
	sudo apt-get install -qy automake bison flex g++ git libboost-all-dev libevent-dev libssl-dev libtool make pkg-config python-setuptools libglib2.0-dev

    sudo mkdir -p "${thrift_dir}"
    sudo chmod 777 "${thrift_dir}"
    cd "${thrift_dir}"
    curl "http://www-eu.apache.org/dist/thrift/0.10.0/thrift-0.10.0.tar.gz" | sudo tar xvz --strip-components 1
    sudo ./configure --prefix=/usr
    sudo make -j$(nproc)
    sudo make install
    (cd "${thrift_dir}/lib/py" && sudo python setup.py install)
  fi

  charybde_dir="/opt/charybdefs"
  nemesis_path="${charybde_dir}/charybdefs-nemesis"

  if [ ! -f "${nemesis_path}" ]; then
    sudo apt-get install -qy build-essential cmake libfuse-dev fuse
    sudo rm -rf "${charybde_dir}" "${nemesis_path}" /usr/local/bin/charybdefs{,-nemesis}
    sudo mkdir -p "${charybde_dir}"
    sudo chmod 777 "${charybde_dir}"
    git clone --depth 1 "https://github.com/scylladb/charybdefs.git" "${charybde_dir}"

    cd "${charybde_dir}"
    thrift -r --gen cpp server.thrift
    cmake CMakeLists.txt
    make -j$(nproc)

    sudo modprobe fuse
    sudo ln -s "${charybde_dir}/charybdefs" /usr/local/bin/charybdefs
    cat > "${nemesis_path}" <<EOF
#!/bin/bash
cd /opt/charybdefs/cookbook
./recipes "\$@"
EOF
    chmod +x "${nemesis_path}"
	sudo ln -s "${nemesis_path}" /usr/local/bin/charybdefs-nemesis
fi
`,

	"confluent": `
sudo apt-get update;
sudo apt-get install -y default-jdk-headless;
curl https://packages.confluent.io/archive/5.0/confluent-oss-5.0.0-2.11.tar.gz | sudo tar -C /usr/local -xz;
sudo ln -s /usr/local/confluent-5.0.0 /usr/local/confluent;
`,

	"docker": `
sudo apt-get update;
sudo apt-get install  -y \
    apt-transport-https \
    ca-certificates \
    curl \
    software-properties-common;
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -;
sudo add-apt-repository \
   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
   $(lsb_release -cs) \
   stable";

sudo apt-get update;
sudo apt-get install  -y docker-ce;
`,

	"gcc": `
sudo apt-get update;
sudo apt-get install -y gcc;
`,

	// graphviz and rlwrap are useful for pprof
	"go": `
sudo apt-get update;
sudo apt-get install -y graphviz rlwrap;

curl https://dl.google.com/go/go1.11.linux-amd64.tar.gz | sudo tar -C /usr/local -xz;
echo 'export PATH=$PATH:/usr/local/go/bin' | sudo tee /etc/profile.d/go.sh > /dev/null;
sudo chmod +x /etc/profile.d/go.sh;
`,

	"haproxy": `
sudo apt-get update;
sudo apt-get install -y haproxy;
`,

	"ntp": `
sudo apt-get update;
sudo apt-get install -y \
  ntp \
  ntpdate;
`,

	"tools": `
sudo apt-get update;
sudo apt-get install -y \
  fio \
  iftop \
  iotop \
  sysstat \
  linux-tools-common \
  linux-tools-4.10.0-35-generic \
  linux-cloud-tools-4.10.0-35-generic;
`,

	"zfs": `
sudo apt-get update;
sudo apt-get install -y \
  zfsutils-linux;
`,
}

func SortedCmds() []string {
	cmds := make([]string, 0, len(installCmds))
	for cmd := range installCmds {
		cmds = append(cmds, cmd)
	}
	sort.Strings(cmds)
	return cmds
}

func Install(c *SyncedCluster, args []string) error {
	do := func(title, cmd string) error {
		var buf bytes.Buffer
		err := c.Run(&buf, &buf, c.Nodes, "installing "+title, cmd)
		if err != nil {
			fmt.Print(buf.String())
		}
		return err
	}

	for _, arg := range args {
		cmd, ok := installCmds[arg]
		if !ok {
			return fmt.Errorf("unknown tool %q", arg)
		}

		if err := do(arg, cmd); err != nil {
			return err
		}
	}
	return nil
}

func VersionSatifies(v *version.Version, constraintString string) bool {
	constraints, err := version.NewConstraint(constraintString)
	if err != nil {
		panic(err)
	}
	return constraints.Check(v)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"io"
	"io/ioutil"
	"net/http"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/gorilla/mux"
	"go.skia.org/infra/autoroll/go/config"
	"go.skia.org/infra/autoroll/go/config/db"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/ds"
	"go.skia.org/infra/go/firestore"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/webhook"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/option"
	"google.golang.org/protobuf/encoding/prototext"
)

// flags
var (
	configFile        = flag.String("config_file", "", "Configuration file to use.")
	firestoreInstance = flag.String("firestore_instance", "", "Firestore instance to use, eg. \"production\"")
	local             = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	port              = flag.String("port", ":8000", "HTTP service port (e.g., ':8000')")
	promPort          = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
	webhookSalt       = flag.String("webhook_request_salt", "", "Path to a file containing webhook request salt.")
)

func main() {
	common.InitWithMust(
		"google3-autoroll",
		common.PrometheusOpt(promPort),
		common.MetricsLoggingOpt(),
	)
	defer common.Defer()

	if *webhookSalt == "" {
		sklog.Fatal("--webhook_request_salt is required.")
	}

	var cfg config.Config
	if err := util.WithReadFile(*configFile, func(f io.Reader) error {
		configBytes, err := ioutil.ReadAll(f)
		if err != nil {
			return err
		}
		if err := prototext.Unmarshal(configBytes, &cfg); err != nil {
			return err
		}
		return nil
	}); err != nil {
		sklog.Fatal(err)
	}

	ctx := context.Background()

	ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail, auth.ScopeGerrit, datastore.ScopeDatastore)
	if err != nil {
		sklog.Fatal(err)
	}
	const namespace = ds.AUTOROLL_INTERNAL_NS
	if err := ds.InitWithOpt(common.PROJECT_ID, namespace, option.WithTokenSource(ts)); err != nil {
		sklog.Fatal(err)
	}
	client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()

	if !*local {
		// Update the roller config in the DB.
		configDB, err := db.NewDBWithParams(ctx, firestore.FIRESTORE_PROJECT, namespace, *firestoreInstance, ts)
		if err != nil {
			sklog.Fatal(err)
		}
		if err := configDB.Put(ctx, cfg.RollerName, &cfg); err != nil {
			sklog.Fatal(err)
		}
	}

	r := mux.NewRouter()
	if err := webhook.InitRequestSaltFromFile(*webhookSalt); err != nil {
		sklog.Fatal(err)
	}
	arb, err := NewAutoRoller(ctx, &cfg, client, ts)
	if err != nil {
		sklog.Fatal(err)
	}
	arb.AddHandlers(r)
	arb.Start(ctx, time.Minute, time.Minute)
	h := httputils.LoggingGzipRequestResponse(r)
	if !*local {
		h = httputils.HealthzAndHTTPS(h)
	}
	http.Handle("/", h)
	sklog.Fatal(http.ListenAndServe(*port, nil))
}
<commit_msg>[autoroll] Add --config to autoroll-google3<commit_after>package main

import (
	"context"
	"encoding/base64"
	"flag"
	"io"
	"io/ioutil"
	"net/http"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/gorilla/mux"
	"go.skia.org/infra/autoroll/go/config"
	"go.skia.org/infra/autoroll/go/config/db"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/ds"
	"go.skia.org/infra/go/firestore"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/webhook"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/option"
	"google.golang.org/protobuf/encoding/prototext"
)

// flags
var (
	configContents    = flag.String("config", "", "Base 64 encoded configuration in JSON format, mutually exclusive with --config_file.")
	configFile        = flag.String("config_file", "", "Configuration file to use.")
	firestoreInstance = flag.String("firestore_instance", "", "Firestore instance to use, eg. \"production\"")
	local             = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	port              = flag.String("port", ":8000", "HTTP service port (e.g., ':8000')")
	promPort          = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
	webhookSalt       = flag.String("webhook_request_salt", "", "Path to a file containing webhook request salt.")
)

func main() {
	common.InitWithMust(
		"google3-autoroll",
		common.PrometheusOpt(promPort),
		common.MetricsLoggingOpt(),
	)
	defer common.Defer()

	if *webhookSalt == "" {
		sklog.Fatal("--webhook_request_salt is required.")
	}

	// Decode the config.
	if (*configContents == "" && *configFile == "") || (*configContents != "" && *configFile != "") {
		sklog.Fatal("Exactly one of --config or --config_file is required.")
	}
	var configBytes []byte
	var err error
	if *configContents != "" {
		configBytes, err = base64.StdEncoding.DecodeString(*configContents)
	} else {
		err = util.WithReadFile(*configFile, func(f io.Reader) error {
			configBytes, err = ioutil.ReadAll(f)
			return err
		})
	}
	if err != nil {
		sklog.Fatal(err)
	}
	var cfg config.Config
	if err := prototext.Unmarshal(configBytes, &cfg); err != nil {
		sklog.Fatal(err)
	}

	ctx := context.Background()

	ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail, auth.ScopeGerrit, datastore.ScopeDatastore)
	if err != nil {
		sklog.Fatal(err)
	}
	const namespace = ds.AUTOROLL_INTERNAL_NS
	if err := ds.InitWithOpt(common.PROJECT_ID, namespace, option.WithTokenSource(ts)); err != nil {
		sklog.Fatal(err)
	}
	client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()

	if !*local {
		// Update the roller config in the DB.
		configDB, err := db.NewDBWithParams(ctx, firestore.FIRESTORE_PROJECT, namespace, *firestoreInstance, ts)
		if err != nil {
			sklog.Fatal(err)
		}
		if err := configDB.Put(ctx, cfg.RollerName, &cfg); err != nil {
			sklog.Fatal(err)
		}
	}

	r := mux.NewRouter()
	if err := webhook.InitRequestSaltFromFile(*webhookSalt); err != nil {
		sklog.Fatal(err)
	}
	arb, err := NewAutoRoller(ctx, &cfg, client, ts)
	if err != nil {
		sklog.Fatal(err)
	}
	arb.AddHandlers(r)
	arb.Start(ctx, time.Minute, time.Minute)
	h := httputils.LoggingGzipRequestResponse(r)
	if !*local {
		h = httputils.HealthzAndHTTPS(h)
	}
	http.Handle("/", h)
	sklog.Fatal(http.ListenAndServe(*port, nil))
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package codec

import (
	"bytes"
	"runtime"
	"unsafe"

	"github.com/juju/errors"
)

const (
	encGroupSize = 8
	encMarker    = byte(0xFF)
	encPad       = byte(0x0)
)

// EncodeBytes guarantees the encoded value is in ascending order for comparison,
// encoding with the following rule:
//  [group1][marker1]...[groupN][markerN]
//  group is 8 bytes slice which is padding with 0.
//  marker is `0xFF - padding 0 count`
// For example:
//   [] -> [0, 0, 0, 0, 0, 0, 0, 0, 247]
//   [1, 2, 3] -> [1, 2, 3, 0, 0, 0, 0, 0, 250]
//   [1, 2, 3, 0] -> [1, 2, 3, 0, 0, 0, 0, 0, 251]
//   [1, 2, 3, 4, 5, 6, 7, 8] -> [1, 2, 3, 4, 5, 6, 7, 8, 255, 0, 0, 0, 0, 0, 0, 0, 0, 247]
// Refer: https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format#memcomparable-format
func EncodeBytes(b []byte, data []byte) []byte {
	// Allocate more space to avoid unnecessary slice growing.
	// Assume that the byte slice size is about `(len(data) / encGroupSize + 1) * (encGroupSize + 1)` bytes,
	// that is `(len(data) / 8 + 1) * 9` in our implement.
	dl := len(data)
	reallocSize := (dl/encGroupSize + 1) * (encGroupSize + 1)
	bs := reallocBytes(b, reallocSize)
	for idx := 0; idx <= dl; idx += encGroupSize {
		remain := dl - idx
		padCount := 0
		if remain >= encGroupSize {
			bs = append(bs, data[idx:idx+encGroupSize]...)
		} else {
			padCount = encGroupSize - remain
			bs = append(bs, data[idx:]...)
			bs = append(bs, make([]byte, padCount)...)
		}

		marker := encMarker - byte(padCount)
		bs = append(bs, marker)
	}

	return bs
}

func decodeBytes(b []byte, reverse bool) ([]byte, []byte, error) {
	data := make([]byte, 0, len(b))
	for {
		if len(b) < encGroupSize+1 {
			return nil, nil, errors.New("insufficient bytes to decode value")
		}

		groupBytes := b[:encGroupSize+1]
		if reverse {
			reverseBytes(groupBytes)
		}

		group := groupBytes[:encGroupSize]
		marker := groupBytes[encGroupSize]

		// Check validity of marker.
		padCount := encMarker - marker
		realGroupSize := encGroupSize - padCount
		if padCount > encGroupSize {
			return nil, nil, errors.Errorf("invalid marker byte, group bytes %q", groupBytes)
		}

		data = append(data, group[:realGroupSize]...)
		b = b[encGroupSize+1:]

		if marker != encMarker {
			// Check validity of padding bytes.
			if bytes.Count(group[realGroupSize:], []byte{encPad}) != int(padCount) {
				return nil, nil, errors.Errorf("invalid padding byte, group bytes %q", groupBytes)
			}

			break
		}
	}

	return b, data, nil
}

// DecodeBytes decodes bytes which is encoded by EncodeBytes before,
// returns the leftover bytes and decoded value if no error.
func DecodeBytes(b []byte) ([]byte, []byte, error) {
	return decodeBytes(b, false)
}

// EncodeBytesDesc first encodes bytes using EncodeBytes, then bitwise reverses
// encoded value to guarantee the encoded value is in descending order for comparison.
func EncodeBytesDesc(b []byte, data []byte) []byte {
	n := len(b)
	b = EncodeBytes(b, data)
	reverseBytes(b[n:])
	return b
}

// DecodeBytesDesc decodes bytes which is encoded by EncodeBytesDesc before,
// returns the leftover bytes and decoded value if no error.
func DecodeBytesDesc(b []byte) ([]byte, []byte, error) {
	return decodeBytes(b, true)
}

// See https://golang.org/src/crypto/cipher/xor.go
const wordSize = int(unsafe.Sizeof(uintptr(0)))
const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64"

func fastReverseBytes(b []byte) {
	n := len(b)
	w := n / wordSize
	if w > 0 {
		bw := *(*[]uintptr)(unsafe.Pointer(&b))
		for i := 0; i < w; i++ {
			bw[i] = ^bw[i]
		}
	}

	for i := w * wordSize; i < n; i++ {
		b[i] = ^b[i]
	}
}

func safeReverseBytes(b []byte) {
	for i := range b {
		b[i] = ^b[i]
	}
}

func reverseBytes(b []byte) {
	if supportsUnaligned {
		fastReverseBytes(b)
		return
	}

	safeReverseBytes(b)
}

// like realloc.
func reallocBytes(b []byte, n int) []byte {
	if cap(b) < n {
		bs := make([]byte, len(b), len(b)+n)
		copy(bs, b)
		return bs
	}

	// slice b has capability to store n bytes
	return b
}
<commit_msg>util/codec: address comment.<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package codec

import (
	"bytes"
	"runtime"
	"unsafe"

	"github.com/juju/errors"
)

const (
	encGroupSize = 8
	encMarker    = byte(0xFF)
	encPad       = byte(0x0)
)

// EncodeBytes guarantees the encoded value is in ascending order for comparison,
// encoding with the following rule:
//  [group1][marker1]...[groupN][markerN]
//  group is 8 bytes slice which is padding with 0.
//  marker is `0xFF - padding 0 count`
// For example:
//   [] -> [0, 0, 0, 0, 0, 0, 0, 0, 247]
//   [1, 2, 3] -> [1, 2, 3, 0, 0, 0, 0, 0, 250]
//   [1, 2, 3, 0] -> [1, 2, 3, 0, 0, 0, 0, 0, 251]
//   [1, 2, 3, 4, 5, 6, 7, 8] -> [1, 2, 3, 4, 5, 6, 7, 8, 255, 0, 0, 0, 0, 0, 0, 0, 0, 247]
// Refer: https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format#memcomparable-format
func EncodeBytes(b []byte, data []byte) []byte {
	// Allocate more space to avoid unnecessary slice growing.
	// Assume that the byte slice size is about `(len(data) / encGroupSize + 1) * (encGroupSize + 1)` bytes,
	// that is `(len(data) / 8 + 1) * 9` in our implement.
	dLen := len(data)
	reallocSize := (dLen/encGroupSize + 1) * (encGroupSize + 1)
	result := reallocBytes(b, reallocSize)
	for idx := 0; idx <= dLen; idx += encGroupSize {
		remain := dLen - idx
		padCount := 0
		if remain >= encGroupSize {
			result = append(result, data[idx:idx+encGroupSize]...)
		} else {
			padCount = encGroupSize - remain
			result = append(result, data[idx:]...)
			result = append(result, make([]byte, padCount)...)
		}

		marker := encMarker - byte(padCount)
		result = append(result, marker)
	}

	return result
}

func decodeBytes(b []byte, reverse bool) ([]byte, []byte, error) {
	data := make([]byte, 0, len(b))
	for {
		if len(b) < encGroupSize+1 {
			return nil, nil, errors.New("insufficient bytes to decode value")
		}

		groupBytes := b[:encGroupSize+1]
		if reverse {
			reverseBytes(groupBytes)
		}

		group := groupBytes[:encGroupSize]
		marker := groupBytes[encGroupSize]

		// Check validity of marker.
		padCount := encMarker - marker
		realGroupSize := encGroupSize - padCount
		if padCount > encGroupSize {
			return nil, nil, errors.Errorf("invalid marker byte, group bytes %q", groupBytes)
		}

		data = append(data, group[:realGroupSize]...)
		b = b[encGroupSize+1:]

		if marker != encMarker {
			// Check validity of padding bytes.
			if bytes.Count(group[realGroupSize:], []byte{encPad}) != int(padCount) {
				return nil, nil, errors.Errorf("invalid padding byte, group bytes %q", groupBytes)
			}

			break
		}
	}

	return b, data, nil
}

// DecodeBytes decodes bytes which is encoded by EncodeBytes before,
// returns the leftover bytes and decoded value if no error.
func DecodeBytes(b []byte) ([]byte, []byte, error) {
	return decodeBytes(b, false)
}

// EncodeBytesDesc first encodes bytes using EncodeBytes, then bitwise reverses
// encoded value to guarantee the encoded value is in descending order for comparison.
func EncodeBytesDesc(b []byte, data []byte) []byte {
	n := len(b)
	b = EncodeBytes(b, data)
	reverseBytes(b[n:])
	return b
}

// DecodeBytesDesc decodes bytes which is encoded by EncodeBytesDesc before,
// returns the leftover bytes and decoded value if no error.
func DecodeBytesDesc(b []byte) ([]byte, []byte, error) {
	return decodeBytes(b, true)
}

// See https://golang.org/src/crypto/cipher/xor.go
const wordSize = int(unsafe.Sizeof(uintptr(0)))
const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64"

func fastReverseBytes(b []byte) {
	n := len(b)
	w := n / wordSize
	if w > 0 {
		bw := *(*[]uintptr)(unsafe.Pointer(&b))
		for i := 0; i < w; i++ {
			bw[i] = ^bw[i]
		}
	}

	for i := w * wordSize; i < n; i++ {
		b[i] = ^b[i]
	}
}

func safeReverseBytes(b []byte) {
	for i := range b {
		b[i] = ^b[i]
	}
}

func reverseBytes(b []byte) {
	if supportsUnaligned {
		fastReverseBytes(b)
		return
	}

	safeReverseBytes(b)
}

// like realloc.
func reallocBytes(b []byte, n int) []byte {
	if cap(b) < n {
		bs := make([]byte, len(b), len(b)+n)
		copy(bs, b)
		return bs
	}

	// slice b has capability to store n bytes
	return b
}
<|endoftext|>


<commit_before>package interpreter

import (
	"fmt"
	"reflect"
	"strings"
)

// Env holds the environment for interpreting a script.
type Env struct {
	name      string
	env       map[string]reflect.Value
	typ       map[string]reflect.Type
	parent    *Env
	interrupt *chan bool
}

// NewEnv creates new global scope.
func NewEnv() *Env {
	interrupt := make(chan bool, 1)
	env := &Env{
		env:       make(map[string]reflect.Value),
		typ:       make(map[string]reflect.Type),
		parent:    nil,
		interrupt: &interrupt,
	}
	env.loadBuiltins()
	return env
}

// NewEnv creates new child scope.
func (env *Env) NewEnv() *Env {
	return &Env{
		env:       make(map[string]reflect.Value),
		typ:       make(map[string]reflect.Type),
		parent:    env,
		name:      env.name,
		interrupt: env.interrupt,
	}
}

// NewPackage creates a new global package.
func NewPackage(n string) *Env {
	interrupt := make(chan bool, 1)
	return &Env{
		env:       make(map[string]reflect.Value),
		typ:       make(map[string]reflect.Type),
		parent:    nil,
		name:      n,
		interrupt: &interrupt,
	}
}

// NewPackage creates a new child package.
func (env *Env) NewPackage(n string) *Env {
	return &Env{
		env:       make(map[string]reflect.Value),
		typ:       make(map[string]reflect.Type),
		parent:    env,
		name:      n,
		interrupt: env.interrupt,
	}
}

// Destroy deletes current scope.
func (env *Env) Destroy() {
	if env.parent == nil {
		return
	}
	for k, v := range env.parent.env {
		if v.IsValid() && v.Interface() == env {
			delete(env.parent.env, k)
		}
	}
	env.parent = nil
	env.env = nil
}

// NewModule creates new module scope as global.
func (env *Env) NewModule(n string) *Env {
	m := &Env{
		env:    make(map[string]reflect.Value),
		parent: env,
		name:   n,
	}
	env.Define(n, m)
	return m
}

// SetName sets a name of the scope. This means that the scope is module.
func (env *Env) SetName(n string) {
	env.name = n
}

// GetName returns module name.
func (env *Env) GetName() string {
	return env.name
}

// Addr returns pointer value which specified the symbol.
func (env *Env) Addr(sym string) (reflect.Value, error) {
	if v, ok := env.env[sym]; ok {
		return v.Addr(), nil
	}
	if env.parent == nil {
		return NilValue, fmt.Errorf("Undefined symbol '%s'", sym)
	}
	return env.parent.Addr(sym)
}

// Type returns type which specified symbol.
func (env *Env) Type(sym string) (reflect.Type, error) {
	if v, ok := env.typ[sym]; ok {
		return v, nil
	}
	if env.parent == nil {
		return NilType, fmt.Errorf("Undefined type '%s'", sym)
	}
	return env.parent.Type(sym)
}

// Get returns value which specified symbol.
func (env *Env) Get(sym string) (reflect.Value, error) {
	if v, ok := env.env[sym]; ok {
		return v, nil
	}
	if env.parent == nil {
		return NilValue, fmt.Errorf("Undefined symbol '%s'", sym)
	}
	return env.parent.Get(sym)
}

// Set the symbol's value.
func (env *Env) Set(k string, v interface{}) error {
	if _, ok := env.env[k]; ok {
		val, ok := v.(reflect.Value)
		if !ok {
			val = reflect.ValueOf(v)
		}
		env.env[k] = val
		return nil
	}
	if env.parent == nil {
		return fmt.Errorf("Unknown symbol '%s'", k)
	}
	return env.parent.Set(k, v)
}

// DefineGlobal defines a symbol in global scope.
func (env *Env) DefineGlobal(k string, v interface{}) {
	if env.parent == nil {
		env.Define(k, v)
	} else {
		env.parent.DefineGlobal(k, v)
	}
}

// DefineType defines a type.
func (env *Env) DefineType(k string, t interface{}) {
	global := env
	keys := []string{k}

	for global.parent != nil {
		if global.name != "" {
			keys = append(keys, global.name)
		}
		global = global.parent
	}

	for i, j := 0, len(keys)-1; i < j; i, j = i+1, j-1 {
		keys[i], keys[j] = keys[j], keys[i]
	}

	typ, ok := t.(reflect.Type)
	if !ok {
		typ = reflect.TypeOf(t)
	}

	global.typ[strings.Join(keys, ".")] = typ
}

// Define a symbol in the current scope.
func (env *Env) Define(k string, v interface{}) {
	val, ok := v.(reflect.Value)
	if !ok {
		val = reflect.ValueOf(v)
	}
	env.env[k] = val
}

// String returns the name of current scope.
func (env *Env) String() string {
	return env.name
}

// Dump shows symbol values in the scope.
func (env *Env) Dump() {
	for k, v := range env.env {
		fmt.Printf("%v = %#v\n", k, v)
	}
}

// Interrupt the execution of any running statements in this environment.
//
// Note that the execution is not instantly aborted: after a call to Interrupt,
// the current running statement will finish, but the next statement will not run,
// and instead will return a NilValue and an ErrInterrupt.
func (env *Env) Interrupt() {
	*(env.interrupt) <- true
}

// RunSingleStmt executes one statement in this environment.
func (env *Env) RunSingleStmt(stmt Stmt) (reflect.Value, error) {
	select {
	case <-*(env.interrupt):
		return NilValue, ErrInterrupt
	default:
		return stmt.Execute(env)
	}
}

// Run executes statements in this environment.
func (env *Env) Run(stmts []Stmt) (reflect.Value, error) {
	rv := NilValue
	var err error
	for _, stmt := range stmts {
		if rv, err = env.RunSingleStmt(stmt); err != nil {
			return rv, err
		}
	}
	return rv, nil
}
<commit_msg>Only create the variable and type maps when needed<commit_after>package interpreter

import (
	"fmt"
	"reflect"
	"strings"
)

// Env holds the environment for interpreting a script.
type Env struct {
	name      string
	env       map[string]reflect.Value // Only created when needed to reduce memory thrashing
	typ       map[string]reflect.Type  // Only created when needed to reduce memory thrashing
	parent    *Env
	interrupt *chan bool
}

// NewEnv creates new global scope.
func NewEnv() *Env {
	interrupt := make(chan bool, 1)
	env := &Env{interrupt: &interrupt}
	env.loadBuiltins()
	return env
}

// NewEnv creates new child scope.
func (env *Env) NewEnv() *Env {
	return &Env{
		parent:    env,
		name:      env.name,
		interrupt: env.interrupt,
	}
}

// NewPackage creates a new global package.
func NewPackage(n string) *Env {
	interrupt := make(chan bool, 1)
	return &Env{
		name:      n,
		interrupt: &interrupt,
	}
}

// NewPackage creates a new child package.
func (env *Env) NewPackage(n string) *Env {
	return &Env{
		parent:    env,
		name:      n,
		interrupt: env.interrupt,
	}
}

// Destroy deletes current scope.
func (env *Env) Destroy() {
	if env.parent == nil {
		return
	}
	if env.parent.env != nil {
		for k, v := range env.parent.env {
			if v.IsValid() && v.Interface() == env {
				delete(env.parent.env, k)
			}
		}
	}
	env.parent = nil
	env.env = nil
}

// NewModule creates new module scope as global.
func (env *Env) NewModule(n string) *Env {
	m := &Env{
		parent: env,
		name:   n,
	}
	env.Define(n, m)
	return m
}

// SetName sets a name of the scope. This means that the scope is module.
func (env *Env) SetName(n string) {
	env.name = n
}

// GetName returns module name.
func (env *Env) GetName() string {
	return env.name
}

// Addr returns pointer value which specified the symbol.
func (env *Env) Addr(sym string) (reflect.Value, error) {
	if env.env != nil {
		if v, ok := env.env[sym]; ok {
			return v.Addr(), nil
		}
	}
	if env.parent == nil {
		return NilValue, fmt.Errorf("Undefined symbol '%s'", sym)
	}
	return env.parent.Addr(sym)
}

// Type returns type which specified symbol.
func (env *Env) Type(sym string) (reflect.Type, error) {
	if env.typ != nil {
		if v, ok := env.typ[sym]; ok {
			return v, nil
		}
	}
	if env.parent == nil {
		return NilType, fmt.Errorf("Undefined type '%s'", sym)
	}
	return env.parent.Type(sym)
}

// Get returns value which specified symbol.
func (env *Env) Get(sym string) (reflect.Value, error) {
	if env.env != nil {
		if v, ok := env.env[sym]; ok {
			return v, nil
		}
	}
	if env.parent == nil {
		return NilValue, fmt.Errorf("Undefined symbol '%s'", sym)
	}
	return env.parent.Get(sym)
}

// Set the symbol's value.
func (env *Env) Set(k string, v interface{}) error {
	if env.env != nil {
		if _, ok := env.env[k]; ok {
			val, ok := v.(reflect.Value)
			if !ok {
				val = reflect.ValueOf(v)
			}
			env.env[k] = val
			return nil
		}
	}
	if env.parent == nil {
		return fmt.Errorf("Unknown symbol '%s'", k)
	}
	return env.parent.Set(k, v)
}

// DefineGlobal defines a symbol in global scope.
func (env *Env) DefineGlobal(k string, v interface{}) {
	if env.parent == nil {
		env.Define(k, v)
	} else {
		env.parent.DefineGlobal(k, v)
	}
}

// DefineType defines a type.
func (env *Env) DefineType(k string, t interface{}) {
	global := env
	keys := []string{k}

	for global.parent != nil {
		if global.name != "" {
			keys = append(keys, global.name)
		}
		global = global.parent
	}

	for i, j := 0, len(keys)-1; i < j; i, j = i+1, j-1 {
		keys[i], keys[j] = keys[j], keys[i]
	}

	typ, ok := t.(reflect.Type)
	if !ok {
		typ = reflect.TypeOf(t)
	}
	if global.typ == nil {
		global.typ = make(map[string]reflect.Type)
	}
	global.typ[strings.Join(keys, ".")] = typ
}

// Define a symbol in the current scope.
func (env *Env) Define(k string, v interface{}) {
	val, ok := v.(reflect.Value)
	if !ok {
		val = reflect.ValueOf(v)
	}
	if env.env == nil {
		env.env = make(map[string]reflect.Value)
	}
	env.env[k] = val
}

// String returns the name of current scope.
func (env *Env) String() string {
	return env.name
}

// Dump shows symbol values in the scope.
func (env *Env) Dump() {
	if env.env != nil {
		for k, v := range env.env {
			fmt.Printf("%v = %#v\n", k, v)
		}
	}
}

// Interrupt the execution of any running statements in this environment.
//
// Note that the execution is not instantly aborted: after a call to Interrupt,
// the current running statement will finish, but the next statement will not run,
// and instead will return a NilValue and an ErrInterrupt.
func (env *Env) Interrupt() {
	*(env.interrupt) <- true
}

// RunSingleStmt executes one statement in this environment.
func (env *Env) RunSingleStmt(stmt Stmt) (reflect.Value, error) {
	select {
	case <-*(env.interrupt):
		return NilValue, ErrInterrupt
	default:
		return stmt.Execute(env)
	}
}

// Run executes statements in this environment.
func (env *Env) Run(stmts []Stmt) (reflect.Value, error) {
	rv := NilValue
	var err error
	for _, stmt := range stmts {
		if rv, err = env.RunSingleStmt(stmt); err != nil {
			return rv, err
		}
	}
	return rv, nil
}
<|endoftext|>


<commit_before>package utils

import (
	"bytes"
	"encoding/json"
	"errors"
	"net/http"
	"net/url"
	"time"

	"github.com/eirka/eirka-libs/config"
)

type CloudFlareBanIpPayload struct {
	Mode          string `json:"mode"`
	Configuration struct {
		Target string `json:"target"`
		Value  string `json:"value"`
	} `json:"configuration"`
	Notes string `json:"notes"`
}

func CloudFlareBanIp(ip, reason string) (err error) {

	if len(ip) == 0 {
		return errors.New("no ip provided")
	}

	// block ip request json
	data := CloudFlareBanIpPayload{
		Mode: "block",
		Configuration: Configuration{
			Target: "ip",
			Value:  ip,
		},
		Notes: reason,
	}

	payloadBytes, err := json.Marshal(data)
	if err != nil {
		// handle err
	}

	// api endpoint
	cloudflareUrl := &url.URL{
		Scheme: "https",
		Host:   "api.cloudflare.com",
		Path:   "/client/v4/user/firewall/access_rules/rules",
	}

	// our http request
	req, err := http.NewRequest(http.MethodPost, cloudflareUrl.String(), bytes.NewReader(payloadBytes))
	if err != nil {
		return errors.New("Error creating CloudFlare request")
	}

	req.Header.Set("X-Auth-Email", config.Settings.CloudFlare.Email)
	req.Header.Set("X-Auth-Key", config.Settings.CloudFlare.Key)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", "Eirka/1.2")

	// a client with a timeout
	var netClient = &http.Client{
		Timeout: time.Second * 10,
	}

	// do the request
	// TODO: add errors here to a system log
	_, err = netClient.Do(req)
	if err != nil {
		return errors.New("Error reaching CloudFlare")
	}

	return
}
<commit_msg>add ban ip controller<commit_after>package utils

import (
	"bytes"
	"encoding/json"
	"errors"
	"net/http"
	"net/url"
	"time"

	"github.com/eirka/eirka-libs/config"
)

type CloudFlareBanIpPayload struct {
	Mode          string                       `json:"mode"`
	Notes         string                       `json:"notes"`
	Configuration CloudFlareBanIpConfiguration `json:"configuration"`
}

type CloudFlareBanIpConfiguration struct {
	Target string `json:"target"`
	Value  string `json:"value"`
}

func CloudFlareBanIp(ip, reason string) (err error) {

	if len(ip) == 0 {
		return errors.New("no ip provided")
	}

	// block ip request json
	data := CloudFlareBanIpPayload{
		Mode: "block",
		Configuration: CloudFlareBanIpConfiguration{
			Target: "ip",
			Value:  ip,
		},
		Notes: reason,
	}

	payloadBytes, err := json.Marshal(data)
	if err != nil {
		// handle err
	}

	// api endpoint
	cloudflareUrl := &url.URL{
		Scheme: "https",
		Host:   "api.cloudflare.com",
		Path:   "/client/v4/user/firewall/access_rules/rules",
	}

	// our http request
	req, err := http.NewRequest(http.MethodPost, cloudflareUrl.String(), bytes.NewReader(payloadBytes))
	if err != nil {
		return errors.New("Error creating CloudFlare request")
	}

	req.Header.Set("X-Auth-Email", config.Settings.CloudFlare.Email)
	req.Header.Set("X-Auth-Key", config.Settings.CloudFlare.Key)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", "Eirka/1.2")

	// a client with a timeout
	var netClient = &http.Client{
		Timeout: time.Second * 10,
	}

	// do the request
	// TODO: add errors here to a system log
	_, err = netClient.Do(req)
	if err != nil {
		return errors.New("Error reaching CloudFlare")
	}

	return
}
<|endoftext|>


<commit_before>package provision

import (
	"bytes"
	"fmt"
	"text/template"

	"github.com/rancher/machine/libmachine/auth"
	"github.com/rancher/machine/libmachine/drivers"
	"github.com/rancher/machine/libmachine/engine"
	"github.com/rancher/machine/libmachine/log"
	"github.com/rancher/machine/libmachine/provision/pkgaction"
	"github.com/rancher/machine/libmachine/swarm"
	"github.com/rancher/machine/libmachine/versioncmp"
)

const (
	hostTmpl = `sudo tee /var/tmp/hostname.yml << EOF
#cloud-config

hostname: %s
EOF
`
)

func init() {
	Register("CoreOS", &RegisteredProvisioner{
		New: NewCoreOSProvisioner,
	})
}

func NewCoreOSProvisioner(d drivers.Driver) Provisioner {
	return &CoreOSProvisioner{
		NewSystemdProvisioner("coreos", d),
	}
}

type CoreOSProvisioner struct {
	SystemdProvisioner
}

func (provisioner *CoreOSProvisioner) String() string {
	return "coreOS"
}

func (provisioner *CoreOSProvisioner) SetHostname(hostname string) error {
	log.Debugf("SetHostname: %s", hostname)

	if _, err := provisioner.SSHCommand(fmt.Sprintf(hostTmpl, hostname)); err != nil {
		return err
	}

	if _, err := provisioner.SSHCommand("sudo systemctl start system-cloudinit@var-tmp-hostname.yml.service"); err != nil {
		return err
	}

	return nil
}

func (provisioner *CoreOSProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
	var (
		engineCfg bytes.Buffer
	)

	driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
	provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)

	dockerVersion, err := DockerClientVersion(provisioner)
	if err != nil {
		return nil, err
	}

	arg := "daemon"
	if versioncmp.GreaterThanOrEqualTo(dockerVersion, "1.12.0") {
		arg = ""
	}

	engineConfigTmpl := `[Service]
Environment=TMPDIR=/var/tmp
ExecStart=
ExecStart=/usr/lib/coreos/dockerd ` + arg + ` --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:{{.DockerPort}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}}{{ range .EngineOptions.Labels }} --label {{.}}{{ end }}{{ range .EngineOptions.InsecureRegistry }} --insecure-registry {{.}}{{ end }}{{ range .EngineOptions.RegistryMirror }} --registry-mirror {{.}}{{ end }}{{ range .EngineOptions.ArbitraryFlags }} --{{.}}{{ end }} \$DOCKER_OPTS \$DOCKER_OPT_BIP \$DOCKER_OPT_MTU \$DOCKER_OPT_IPMASQ
Environment={{range .EngineOptions.Env}}{{ printf "%q" . }} {{end}}
`

	t, err := template.New("engineConfig").Parse(engineConfigTmpl)
	if err != nil {
		return nil, err
	}

	engineConfigContext := EngineConfigContext{
		DockerPort:    dockerPort,
		AuthOptions:   provisioner.AuthOptions,
		EngineOptions: provisioner.EngineOptions,
	}

	t.Execute(&engineCfg, engineConfigContext)

	return &DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: provisioner.DaemonOptionsFile,
	}, nil
}

func (provisioner *CoreOSProvisioner) Package(name string, action pkgaction.PackageAction) error {
	return nil
}

func (provisioner *CoreOSProvisioner) Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error {
	provisioner.SwarmOptions = swarmOptions
	provisioner.AuthOptions = authOptions
	provisioner.EngineOptions = engineOptions

	if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
		return err
	}

	if err := makeDockerOptionsDir(provisioner); err != nil {
		return err
	}

	log.Debugf("Preparing certificates")
	provisioner.AuthOptions = setRemoteAuthOptions(provisioner)

	log.Debugf("Setting up certificates")
	if err := ConfigureAuth(provisioner); err != nil {
		return err
	}

	log.Debug("Configuring swarm")
	err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions)
	return err
}
<commit_msg>Ensure that OSes that use ID_LIKE=coreos, the CoreOSProvisioneer is indicated as compatible (for example: flatcar)<commit_after>package provision

import (
	"bytes"
	"fmt"
	"text/template"

	"github.com/rancher/machine/libmachine/auth"
	"github.com/rancher/machine/libmachine/drivers"
	"github.com/rancher/machine/libmachine/engine"
	"github.com/rancher/machine/libmachine/log"
	"github.com/rancher/machine/libmachine/provision/pkgaction"
	"github.com/rancher/machine/libmachine/swarm"
	"github.com/rancher/machine/libmachine/versioncmp"
)

const (
	hostTmpl = `sudo tee /var/tmp/hostname.yml << EOF
#cloud-config

hostname: %s
EOF
`
)

func init() {
	Register("CoreOS", &RegisteredProvisioner{
		New: NewCoreOSProvisioner,
	})
}

func NewCoreOSProvisioner(d drivers.Driver) Provisioner {
	return &CoreOSProvisioner{
		NewSystemdProvisioner("coreos", d),
	}
}

type CoreOSProvisioner struct {
	SystemdProvisioner
}

func (provisioner *CoreOSProvisioner) String() string {
	return "coreOS"
}

func (provisioner *CoreOSProvisioner) CompatibleWithHost() bool {
	return provisioner.OsReleaseInfo.ID == provisioner.OsReleaseID || provisioner.OsReleaseInfo.IDLike == provisioner.OsReleaseID
}

func (provisioner *CoreOSProvisioner) SetHostname(hostname string) error {
	log.Debugf("SetHostname: %s", hostname)

	if _, err := provisioner.SSHCommand(fmt.Sprintf(hostTmpl, hostname)); err != nil {
		return err
	}

	if _, err := provisioner.SSHCommand("sudo systemctl start system-cloudinit@var-tmp-hostname.yml.service"); err != nil {
		return err
	}

	return nil
}

func (provisioner *CoreOSProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
	var (
		engineCfg bytes.Buffer
	)

	driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
	provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)

	dockerVersion, err := DockerClientVersion(provisioner)
	if err != nil {
		return nil, err
	}

	arg := "daemon"
	if versioncmp.GreaterThanOrEqualTo(dockerVersion, "1.12.0") {
		arg = ""
	}

	engineConfigTmpl := `[Service]
Environment=TMPDIR=/var/tmp
ExecStart=
ExecStart=/usr/lib/coreos/dockerd ` + arg + ` --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:{{.DockerPort}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}}{{ range .EngineOptions.Labels }} --label {{.}}{{ end }}{{ range .EngineOptions.InsecureRegistry }} --insecure-registry {{.}}{{ end }}{{ range .EngineOptions.RegistryMirror }} --registry-mirror {{.}}{{ end }}{{ range .EngineOptions.ArbitraryFlags }} --{{.}}{{ end }} \$DOCKER_OPTS \$DOCKER_OPT_BIP \$DOCKER_OPT_MTU \$DOCKER_OPT_IPMASQ
Environment={{range .EngineOptions.Env}}{{ printf "%q" . }} {{end}}
`

	t, err := template.New("engineConfig").Parse(engineConfigTmpl)
	if err != nil {
		return nil, err
	}

	engineConfigContext := EngineConfigContext{
		DockerPort:    dockerPort,
		AuthOptions:   provisioner.AuthOptions,
		EngineOptions: provisioner.EngineOptions,
	}

	t.Execute(&engineCfg, engineConfigContext)

	return &DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: provisioner.DaemonOptionsFile,
	}, nil
}

func (provisioner *CoreOSProvisioner) Package(name string, action pkgaction.PackageAction) error {
	return nil
}

func (provisioner *CoreOSProvisioner) Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error {
	provisioner.SwarmOptions = swarmOptions
	provisioner.AuthOptions = authOptions
	provisioner.EngineOptions = engineOptions

	if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
		return err
	}

	if err := makeDockerOptionsDir(provisioner); err != nil {
		return err
	}

	log.Debugf("Preparing certificates")
	provisioner.AuthOptions = setRemoteAuthOptions(provisioner)

	log.Debugf("Setting up certificates")
	if err := ConfigureAuth(provisioner); err != nil {
		return err
	}

	log.Debug("Configuring swarm")
	err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions)
	return err
}
<|endoftext|>


<commit_before>package main

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import (
	"net/http"
	"testing"

	"fmt"

	"bytes"
	"context"
	"net/http/httptest"
)

func TestCreateRouteMap(t *testing.T) {
	authBase := AuthBase{false, "secret", nil, func(handlerFunc http.HandlerFunc) http.HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request) {
			ctx := context.WithValue(r.Context(), "authWasCalled", "true")
			handlerFunc(w, r.WithContext(ctx))
		}
	}}

	PathOneHandler := func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()

		authWasCalled := getAuthWasCalled(ctx)

		fmt.Fprintf(w, "%s %s", "path1", authWasCalled)
	}

	PathTwoHandler := func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()

		authWasCalled := getAuthWasCalled(ctx)

		fmt.Fprintf(w, "%s %s", "path2", authWasCalled)
	}

	routes := []Route{{1.2, http.MethodGet, `path1`, PathOneHandler, ServersPrivLevel, true, nil}, {1.2, http.MethodGet, `path2`, PathTwoHandler, 0, false, nil}}

	routeMap := CreateRouteMap(routes, authBase)

	route1Handler := routeMap["GET"][0].Handler

	w := httptest.NewRecorder()
	r, err := http.NewRequest("", "/", nil)
	if err != nil {
		t.Error("Error creating new request")
	}

	route1Handler(w, r)

	if bytes.Compare(w.Body.Bytes(), []byte("path1 true")) != 0 {
		t.Errorf("Got: %s \nExpected to receive path1 true\n", w.Body.Bytes())
	}

	route2Handler := routeMap["GET"][1].Handler

	w = httptest.NewRecorder()

	route2Handler(w, r)

	if bytes.Compare(w.Body.Bytes(), []byte("path2 false")) != 0 {
		t.Errorf("Got: %s \nExpected to receive path2 false\n", w.Body.Bytes())
	}
}

func getAuthWasCalled(ctx context.Context) string {
	val := ctx.Value("authWasCalled")
	if val != nil {
		return val.(string)
	}
	return "false"
}
<commit_msg>test for explicitly skipped headers (for passthrough to legacy system)<commit_after>package main

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import (
	"net/http"
	"testing"

	"fmt"

	"bytes"
	"context"
	"net/http/httptest"
)

func TestCreateRouteMap(t *testing.T) {
	authBase := AuthBase{false, "secret", nil, func(handlerFunc http.HandlerFunc) http.HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request) {
			ctx := context.WithValue(r.Context(), "authWasCalled", "true")
			handlerFunc(w, r.WithContext(ctx))
		}
	}}

	PathOneHandler := func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()
		authWasCalled := getAuthWasCalled(ctx)
		fmt.Fprintf(w, "%s %s", "path1", authWasCalled)
	}

	PathTwoHandler := func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()
		authWasCalled := getAuthWasCalled(ctx)
		fmt.Fprintf(w, "%s %s", "path2", authWasCalled)
	}

	PathThreeHandler := func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()
		authWasCalled := getAuthWasCalled(ctx)
		fmt.Fprintf(w, "%s %s", "path3", authWasCalled)
	}

	routes := []Route{
		{1.2, http.MethodGet, `path1`, PathOneHandler, ServersPrivLevel, true, nil},
		{1.2, http.MethodGet, `path2`, PathTwoHandler, 0, false, nil},
		{1.2, http.MethodGet, `path3`, PathThreeHandler, 0, false, []Middleware{}},
	}

	routeMap := CreateRouteMap(routes, authBase)

	route1Handler := routeMap["GET"][0].Handler

	w := httptest.NewRecorder()
	r, err := http.NewRequest("", "/", nil)
	if err != nil {
		t.Error("Error creating new request")
	}

	route1Handler(w, r)

	if bytes.Compare(w.Body.Bytes(), []byte("path1 true")) != 0 {
		t.Errorf("Got: %s \nExpected to receive path1 true\n", w.Body.Bytes())
	}

	route2Handler := routeMap["GET"][1].Handler

	w = httptest.NewRecorder()

	route2Handler(w, r)

	if bytes.Compare(w.Body.Bytes(), []byte("path2 false")) != 0 {
		t.Errorf("Got: %s \nExpected to receive path2 false\n", w.Body.Bytes())
	}

	if v, ok := w.HeaderMap["Access-Control-Allow-Credentials"]; !ok || len(v) != 1 || v[0] != "true" {
		t.Errorf(`Expected Access-Control-Allow-Credentials: [ "true" ]`)
	}

	route3Handler := routeMap["GET"][2].Handler
	w = httptest.NewRecorder()
	route3Handler(w, r)
	if bytes.Compare(w.Body.Bytes(), []byte("path3 false")) != 0 {
		t.Errorf("Got: %s \nExpected to receive path3 false\n", w.Body.Bytes())
	}
	if v, ok := w.HeaderMap["Access-Control-Allow-Credentials"]; ok {
		t.Errorf("Unexpected Access-Control-Allow-Credentials: %s", v)
	}
}

func getAuthWasCalled(ctx context.Context) string {
	val := ctx.Value("authWasCalled")
	if val != nil {
		return val.(string)
	}
	return "false"
}
<|endoftext|>


<commit_before>package meta

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
)

// raftState abstracts the interaction of the raft consensus layer
// across local or remote nodes.  It is a form of the state design pattern and allows
// the meta.Store to change its behavior with the raft layer at runtime.
type raftState interface {
	open() error
	remove() error
	shutdown() error
	initialize() error
	leader() string
	isLeader() bool
	sync(index uint64, timeout time.Duration) error
	setPeers(addrs []string) error
	addPeer(addr string) error
	removePeer(addr string) error
	peers() ([]string, error)
	invalidate() error
	close() error
	lastIndex() uint64
	apply(b []byte) error
	snapshot() error
}

// localRaft is a consensus strategy that uses a local raft implementation for
// consensus operations.
type localRaft struct {
	wg        sync.WaitGroup
	closing   chan struct{}
	store     *Store
	raft      *raft.Raft
	transport *raft.NetworkTransport
	peerStore raft.PeerStore
	raftStore *raftboltdb.BoltStore
	raftLayer *raftLayer
}

func (r *localRaft) remove() error {
	if err := os.RemoveAll(filepath.Join(r.store.path, "raft.db")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(r.store.path, "peers.json")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(r.store.path, "snapshots")); err != nil {
		return err
	}
	return nil
}

func (r *localRaft) shutdown() error {
	// Shutdown raft.
	if r.raft != nil {
		if err := r.raft.Shutdown().Error(); err != nil {
			return err
		}
	}
	r.raft = nil
	return nil
}

func (r *localRaft) updateMetaData(ms *Data) {
	if ms == nil {
		return
	}

	updated := false
	r.store.mu.RLock()
	if ms.Index > r.store.data.Index {
		updated = true
	}
	r.store.mu.RUnlock()

	if updated {
		r.store.Logger.Printf("Updating metastore to term=%v index=%v", ms.Term, ms.Index)
		r.store.mu.Lock()
		r.store.data = ms
		r.store.mu.Unlock()
	}
}

func (r *localRaft) invalidate() error {
	if r.store.IsLeader() {
		return nil
	}

	ms, err := r.store.rpc.fetchMetaData(false)
	if err != nil {
		return err
	}

	r.updateMetaData(ms)
	return nil
}

func (r *localRaft) open() error {
	r.closing = make(chan struct{})

	s := r.store
	// Setup raft configuration.
	config := raft.DefaultConfig()
	config.LogOutput = ioutil.Discard

	if s.clusterTracingEnabled {
		config.Logger = s.Logger
	}
	config.HeartbeatTimeout = s.HeartbeatTimeout
	config.ElectionTimeout = s.ElectionTimeout
	config.LeaderLeaseTimeout = s.LeaderLeaseTimeout
	config.CommitTimeout = s.CommitTimeout

	// If no peers are set in the config or there is one and we are it, then start as a single server.
	if len(s.peers) <= 1 {
		config.EnableSingleNode = true
		// Ensure we can always become the leader
		config.DisableBootstrapAfterElect = false
		// Don't shutdown raft automatically if we renamed our hostname back to a previous name
		config.ShutdownOnRemove = false
	}

	// Build raft layer to multiplex listener.
	r.raftLayer = newRaftLayer(s.RaftListener, s.RemoteAddr)

	// Create a transport layer
	r.transport = raft.NewNetworkTransport(r.raftLayer, 3, 10*time.Second, config.LogOutput)

	// Create peer storage.
	r.peerStore = raft.NewJSONPeers(s.path, r.transport)

	peers, err := r.peerStore.Peers()
	if err != nil {
		return err
	}

	// For single-node clusters, we can update the raft peers before we start the cluster if the hostname
	// has changed.
	if config.EnableSingleNode {
		if err := r.peerStore.SetPeers([]string{s.RemoteAddr.String()}); err != nil {
			return err
		}
		peers = []string{s.RemoteAddr.String()}
	}

	// If we have multiple nodes in the cluster, make sure our address is in the raft peers or
	// we won't be able to boot into the cluster because the other peers will reject our new hostname.  This
	// is difficult to resolve automatically because we need to have all the raft peers agree on the current members
	// of the cluster before we can change them.
	if len(peers) > 0 && !raft.PeerContained(peers, s.RemoteAddr.String()) {
		s.Logger.Printf("%v is not in the list of raft peers. Please update %v/peers.json on all raft nodes to have the same contents.", s.RemoteAddr.String(), s.Path())
		return fmt.Errorf("peers out of sync: %v not in %v", s.RemoteAddr.String(), peers)
	}

	// Create the log store and stable store.
	store, err := raftboltdb.NewBoltStore(filepath.Join(s.path, "raft.db"))
	if err != nil {
		return fmt.Errorf("new bolt store: %s", err)
	}
	r.raftStore = store

	// Create the snapshot store.
	snapshots, err := raft.NewFileSnapshotStore(s.path, raftSnapshotsRetained, os.Stderr)
	if err != nil {
		return fmt.Errorf("file snapshot store: %s", err)
	}

	// Create raft log.
	ra, err := raft.NewRaft(config, (*storeFSM)(s), store, store, snapshots, r.peerStore, r.transport)
	if err != nil {
		return fmt.Errorf("new raft: %s", err)
	}
	r.raft = ra

	r.wg.Add(1)
	go r.logLeaderChanges()

	return nil
}

func (r *localRaft) logLeaderChanges() {
	defer r.wg.Done()
	// Logs our current state (Node at 1.2.3.4:8088 [Follower])
	r.store.Logger.Printf(r.raft.String())
	for {
		select {
		case <-r.closing:
			return
		case <-r.raft.LeaderCh():
			peers, err := r.peers()
			if err != nil {
				r.store.Logger.Printf("failed to lookup peers: %v", err)
			}
			r.store.Logger.Printf("%v. peers=%v", r.raft.String(), peers)
		}
	}
}

func (r *localRaft) close() error {
	close(r.closing)
	r.wg.Wait()

	if r.transport != nil {
		r.transport.Close()
		r.transport = nil
	}

	if r.raftLayer != nil {
		r.raftLayer.Close()
		r.raftLayer = nil
	}

	// Shutdown raft.
	if r.raft != nil {
		if err := r.raft.Shutdown().Error(); err != nil {
			return err
		}
		r.raft = nil
	}

	if r.raftStore != nil {
		r.raftStore.Close()
		r.raftStore = nil
	}

	return nil
}

func (r *localRaft) initialize() error {
	s := r.store
	// If we have committed entries then the store is already in the cluster.
	if index, err := r.raftStore.LastIndex(); err != nil {
		return fmt.Errorf("last index: %s", err)
	} else if index > 0 {
		return nil
	}

	// Force set peers.
	if err := r.setPeers(s.peers); err != nil {
		return fmt.Errorf("set raft peers: %s", err)
	}

	return nil
}

// apply applies a serialized command to the raft log.
func (r *localRaft) apply(b []byte) error {
	// Apply to raft log.
	f := r.raft.Apply(b, 0)
	if err := f.Error(); err != nil {
		return err
	}

	// Return response if it's an error.
	// No other non-nil objects should be returned.
	resp := f.Response()
	if err, ok := resp.(error); ok {
		return lookupError(err)
	}
	assert(resp == nil, "unexpected response: %#v", resp)

	return nil
}

func (r *localRaft) lastIndex() uint64 {
	return r.raft.LastIndex()
}

func (r *localRaft) sync(index uint64, timeout time.Duration) error {
	ticker := time.NewTicker(100 * time.Millisecond)
	defer ticker.Stop()

	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for {
		// Wait for next tick or timeout.
		select {
		case <-ticker.C:
		case <-timer.C:
			return errors.New("timeout")
		}

		// Compare index against current metadata.
		r.store.mu.Lock()
		ok := (r.store.data.Index >= index)
		r.store.mu.Unlock()

		// Exit if we are at least at the given index.
		if ok {
			return nil
		}
	}
}

func (r *localRaft) snapshot() error {
	future := r.raft.Snapshot()
	return future.Error()
}

// addPeer adds addr to the list of peers in the cluster.
func (r *localRaft) addPeer(addr string) error {
	peers, err := r.peerStore.Peers()
	if err != nil {
		return err
	}

	if len(peers) >= 3 {
		return nil
	}

	if fut := r.raft.AddPeer(addr); fut.Error() != nil {
		return fut.Error()
	}
	return nil
}

// removePeer removes addr from the list of peers in the cluster.
func (r *localRaft) removePeer(addr string) error {
	// Only do this on the leader
	if r.isLeader() {
		if fut := r.raft.RemovePeer(addr); fut.Error() != nil {
			return fut.Error()
		}
	}

	return nil
}

// setPeers sets a list of peers in the cluster.
func (r *localRaft) setPeers(addrs []string) error {
	return r.raft.SetPeers(addrs).Error()
}

func (r *localRaft) peers() ([]string, error) {
	return r.peerStore.Peers()
}

func (r *localRaft) leader() string {
	if r.raft == nil {
		return ""
	}

	return r.raft.Leader()
}

func (r *localRaft) isLeader() bool {
	if r.raft == nil {
		return false
	}
	return r.raft.State() == raft.Leader
}

// remoteRaft is a consensus strategy that uses a remote raft cluster for
// consensus operations.
type remoteRaft struct {
	store *Store
}

func (r *remoteRaft) remove() error {
	return nil
}

func (r *remoteRaft) shutdown() error {
	return nil
}

func (r *remoteRaft) updateMetaData(ms *Data) {
	if ms == nil {
		return
	}

	updated := false
	r.store.mu.RLock()
	if ms.Index > r.store.data.Index {
		updated = true
	}
	r.store.mu.RUnlock()

	if updated {
		r.store.Logger.Printf("Updating metastore to term=%v index=%v", ms.Term, ms.Index)
		r.store.mu.Lock()
		r.store.data = ms
		r.store.mu.Unlock()
	}
}

func (r *remoteRaft) invalidate() error {
	ms, err := r.store.rpc.fetchMetaData(false)
	if err != nil {
		return err
	}

	r.updateMetaData(ms)
	return nil
}

func (r *remoteRaft) setPeers(addrs []string) error {
	// Convert to JSON
	var buf bytes.Buffer
	enc := json.NewEncoder(&buf)
	if err := enc.Encode(addrs); err != nil {
		return err
	}

	// Write out as JSON
	return ioutil.WriteFile(filepath.Join(r.store.path, "peers.json"), buf.Bytes(), 0755)
}

// addPeer adds addr to the list of peers in the cluster.
func (r *remoteRaft) addPeer(addr string) error {
	return fmt.Errorf("cannot add peer using remote raft")
}

// removePeer does nothing for remoteRaft.
func (r *remoteRaft) removePeer(addr string) error {
	return nil
}

func (r *remoteRaft) peers() ([]string, error) {
	return readPeersJSON(filepath.Join(r.store.path, "peers.json"))
}

func (r *remoteRaft) open() error {
	if err := r.setPeers(r.store.peers); err != nil {
		return err
	}

	go func() {
		for {
			select {
			case <-r.store.closing:
				return
			default:
			}

			ms, err := r.store.rpc.fetchMetaData(true)
			if err != nil {
				r.store.Logger.Printf("fetch metastore: %v", err)
				time.Sleep(time.Second)
				continue
			}
			r.updateMetaData(ms)
		}
	}()
	return nil
}

func (r *remoteRaft) close() error {
	return nil
}

// apply applies a serialized command to the raft log.
func (r *remoteRaft) apply(b []byte) error {
	return fmt.Errorf("cannot apply log while in remote raft state")
}

func (r *remoteRaft) initialize() error {
	return nil
}

func (r *remoteRaft) leader() string {
	if len(r.store.peers) == 0 {
		return ""
	}

	return r.store.peers[rand.Intn(len(r.store.peers))]
}

func (r *remoteRaft) isLeader() bool {
	return false
}

func (r *remoteRaft) lastIndex() uint64 {
	return r.store.cachedData().Index
}

func (r *remoteRaft) sync(index uint64, timeout time.Duration) error {
	//FIXME: jwilder: check index and timeout
	return r.store.invalidate()
}

func (r *remoteRaft) snapshot() error {
	return fmt.Errorf("cannot snapshot while in remote raft state")
}

func readPeersJSON(path string) ([]string, error) {
	// Read the file
	buf, err := ioutil.ReadFile(path)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	// Check for no peers
	if len(buf) == 0 {
		return nil, nil
	}

	// Decode the peers
	var peers []string
	dec := json.NewDecoder(bytes.NewReader(buf))
	if err := dec.Decode(&peers); err != nil {
		return nil, err
	}

	return peers, nil
}
<commit_msg>to much code depends on setting this to nil yet<commit_after>package meta

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
)

// raftState abstracts the interaction of the raft consensus layer
// across local or remote nodes.  It is a form of the state design pattern and allows
// the meta.Store to change its behavior with the raft layer at runtime.
type raftState interface {
	open() error
	remove() error
	shutdown() error
	initialize() error
	leader() string
	isLeader() bool
	sync(index uint64, timeout time.Duration) error
	setPeers(addrs []string) error
	addPeer(addr string) error
	removePeer(addr string) error
	peers() ([]string, error)
	invalidate() error
	close() error
	lastIndex() uint64
	apply(b []byte) error
	snapshot() error
}

// localRaft is a consensus strategy that uses a local raft implementation for
// consensus operations.
type localRaft struct {
	wg        sync.WaitGroup
	closing   chan struct{}
	store     *Store
	raft      *raft.Raft
	transport *raft.NetworkTransport
	peerStore raft.PeerStore
	raftStore *raftboltdb.BoltStore
	raftLayer *raftLayer
}

func (r *localRaft) remove() error {
	if err := os.RemoveAll(filepath.Join(r.store.path, "raft.db")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(r.store.path, "peers.json")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(r.store.path, "snapshots")); err != nil {
		return err
	}
	return nil
}

func (r *localRaft) shutdown() error {
	// Shutdown raft.
	if r.raft != nil {
		if err := r.raft.Shutdown().Error(); err != nil {
			return err
		}
	}
	return nil
}

func (r *localRaft) updateMetaData(ms *Data) {
	if ms == nil {
		return
	}

	updated := false
	r.store.mu.RLock()
	if ms.Index > r.store.data.Index {
		updated = true
	}
	r.store.mu.RUnlock()

	if updated {
		r.store.Logger.Printf("Updating metastore to term=%v index=%v", ms.Term, ms.Index)
		r.store.mu.Lock()
		r.store.data = ms
		r.store.mu.Unlock()
	}
}

func (r *localRaft) invalidate() error {
	if r.store.IsLeader() {
		return nil
	}

	ms, err := r.store.rpc.fetchMetaData(false)
	if err != nil {
		return err
	}

	r.updateMetaData(ms)
	return nil
}

func (r *localRaft) open() error {
	r.closing = make(chan struct{})

	s := r.store
	// Setup raft configuration.
	config := raft.DefaultConfig()
	config.LogOutput = ioutil.Discard

	if s.clusterTracingEnabled {
		config.Logger = s.Logger
	}
	config.HeartbeatTimeout = s.HeartbeatTimeout
	config.ElectionTimeout = s.ElectionTimeout
	config.LeaderLeaseTimeout = s.LeaderLeaseTimeout
	config.CommitTimeout = s.CommitTimeout

	// If no peers are set in the config or there is one and we are it, then start as a single server.
	if len(s.peers) <= 1 {
		config.EnableSingleNode = true
		// Ensure we can always become the leader
		config.DisableBootstrapAfterElect = false
		// Don't shutdown raft automatically if we renamed our hostname back to a previous name
		config.ShutdownOnRemove = false
	}

	// Build raft layer to multiplex listener.
	r.raftLayer = newRaftLayer(s.RaftListener, s.RemoteAddr)

	// Create a transport layer
	r.transport = raft.NewNetworkTransport(r.raftLayer, 3, 10*time.Second, config.LogOutput)

	// Create peer storage.
	r.peerStore = raft.NewJSONPeers(s.path, r.transport)

	peers, err := r.peerStore.Peers()
	if err != nil {
		return err
	}

	// For single-node clusters, we can update the raft peers before we start the cluster if the hostname
	// has changed.
	if config.EnableSingleNode {
		if err := r.peerStore.SetPeers([]string{s.RemoteAddr.String()}); err != nil {
			return err
		}
		peers = []string{s.RemoteAddr.String()}
	}

	// If we have multiple nodes in the cluster, make sure our address is in the raft peers or
	// we won't be able to boot into the cluster because the other peers will reject our new hostname.  This
	// is difficult to resolve automatically because we need to have all the raft peers agree on the current members
	// of the cluster before we can change them.
	if len(peers) > 0 && !raft.PeerContained(peers, s.RemoteAddr.String()) {
		s.Logger.Printf("%v is not in the list of raft peers. Please update %v/peers.json on all raft nodes to have the same contents.", s.RemoteAddr.String(), s.Path())
		return fmt.Errorf("peers out of sync: %v not in %v", s.RemoteAddr.String(), peers)
	}

	// Create the log store and stable store.
	store, err := raftboltdb.NewBoltStore(filepath.Join(s.path, "raft.db"))
	if err != nil {
		return fmt.Errorf("new bolt store: %s", err)
	}
	r.raftStore = store

	// Create the snapshot store.
	snapshots, err := raft.NewFileSnapshotStore(s.path, raftSnapshotsRetained, os.Stderr)
	if err != nil {
		return fmt.Errorf("file snapshot store: %s", err)
	}

	// Create raft log.
	ra, err := raft.NewRaft(config, (*storeFSM)(s), store, store, snapshots, r.peerStore, r.transport)
	if err != nil {
		return fmt.Errorf("new raft: %s", err)
	}
	r.raft = ra

	r.wg.Add(1)
	go r.logLeaderChanges()

	return nil
}

func (r *localRaft) logLeaderChanges() {
	defer r.wg.Done()
	// Logs our current state (Node at 1.2.3.4:8088 [Follower])
	r.store.Logger.Printf(r.raft.String())
	for {
		select {
		case <-r.closing:
			return
		case <-r.raft.LeaderCh():
			peers, err := r.peers()
			if err != nil {
				r.store.Logger.Printf("failed to lookup peers: %v", err)
			}
			r.store.Logger.Printf("%v. peers=%v", r.raft.String(), peers)
		}
	}
}

func (r *localRaft) close() error {
	close(r.closing)
	r.wg.Wait()

	if r.transport != nil {
		r.transport.Close()
		r.transport = nil
	}

	if r.raftLayer != nil {
		r.raftLayer.Close()
		r.raftLayer = nil
	}

	// Shutdown raft.
	if r.raft != nil {
		if err := r.raft.Shutdown().Error(); err != nil {
			return err
		}
		r.raft = nil
	}

	if r.raftStore != nil {
		r.raftStore.Close()
		r.raftStore = nil
	}

	return nil
}

func (r *localRaft) initialize() error {
	s := r.store
	// If we have committed entries then the store is already in the cluster.
	if index, err := r.raftStore.LastIndex(); err != nil {
		return fmt.Errorf("last index: %s", err)
	} else if index > 0 {
		return nil
	}

	// Force set peers.
	if err := r.setPeers(s.peers); err != nil {
		return fmt.Errorf("set raft peers: %s", err)
	}

	return nil
}

// apply applies a serialized command to the raft log.
func (r *localRaft) apply(b []byte) error {
	// Apply to raft log.
	f := r.raft.Apply(b, 0)
	if err := f.Error(); err != nil {
		return err
	}

	// Return response if it's an error.
	// No other non-nil objects should be returned.
	resp := f.Response()
	if err, ok := resp.(error); ok {
		return lookupError(err)
	}
	assert(resp == nil, "unexpected response: %#v", resp)

	return nil
}

func (r *localRaft) lastIndex() uint64 {
	return r.raft.LastIndex()
}

func (r *localRaft) sync(index uint64, timeout time.Duration) error {
	ticker := time.NewTicker(100 * time.Millisecond)
	defer ticker.Stop()

	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for {
		// Wait for next tick or timeout.
		select {
		case <-ticker.C:
		case <-timer.C:
			return errors.New("timeout")
		}

		// Compare index against current metadata.
		r.store.mu.Lock()
		ok := (r.store.data.Index >= index)
		r.store.mu.Unlock()

		// Exit if we are at least at the given index.
		if ok {
			return nil
		}
	}
}

func (r *localRaft) snapshot() error {
	future := r.raft.Snapshot()
	return future.Error()
}

// addPeer adds addr to the list of peers in the cluster.
func (r *localRaft) addPeer(addr string) error {
	peers, err := r.peerStore.Peers()
	if err != nil {
		return err
	}

	if len(peers) >= 3 {
		return nil
	}

	if fut := r.raft.AddPeer(addr); fut.Error() != nil {
		return fut.Error()
	}
	return nil
}

// removePeer removes addr from the list of peers in the cluster.
func (r *localRaft) removePeer(addr string) error {
	// Only do this on the leader
	if r.isLeader() {
		if fut := r.raft.RemovePeer(addr); fut.Error() != nil {
			return fut.Error()
		}
	}

	return nil
}

// setPeers sets a list of peers in the cluster.
func (r *localRaft) setPeers(addrs []string) error {
	return r.raft.SetPeers(addrs).Error()
}

func (r *localRaft) peers() ([]string, error) {
	return r.peerStore.Peers()
}

func (r *localRaft) leader() string {
	if r.raft == nil {
		return ""
	}

	return r.raft.Leader()
}

func (r *localRaft) isLeader() bool {
	if r.raft == nil {
		return false
	}
	return r.raft.State() == raft.Leader
}

// remoteRaft is a consensus strategy that uses a remote raft cluster for
// consensus operations.
type remoteRaft struct {
	store *Store
}

func (r *remoteRaft) remove() error {
	return nil
}

func (r *remoteRaft) shutdown() error {
	return nil
}

func (r *remoteRaft) updateMetaData(ms *Data) {
	if ms == nil {
		return
	}

	updated := false
	r.store.mu.RLock()
	if ms.Index > r.store.data.Index {
		updated = true
	}
	r.store.mu.RUnlock()

	if updated {
		r.store.Logger.Printf("Updating metastore to term=%v index=%v", ms.Term, ms.Index)
		r.store.mu.Lock()
		r.store.data = ms
		r.store.mu.Unlock()
	}
}

func (r *remoteRaft) invalidate() error {
	ms, err := r.store.rpc.fetchMetaData(false)
	if err != nil {
		return err
	}

	r.updateMetaData(ms)
	return nil
}

func (r *remoteRaft) setPeers(addrs []string) error {
	// Convert to JSON
	var buf bytes.Buffer
	enc := json.NewEncoder(&buf)
	if err := enc.Encode(addrs); err != nil {
		return err
	}

	// Write out as JSON
	return ioutil.WriteFile(filepath.Join(r.store.path, "peers.json"), buf.Bytes(), 0755)
}

// addPeer adds addr to the list of peers in the cluster.
func (r *remoteRaft) addPeer(addr string) error {
	return fmt.Errorf("cannot add peer using remote raft")
}

// removePeer does nothing for remoteRaft.
func (r *remoteRaft) removePeer(addr string) error {
	return nil
}

func (r *remoteRaft) peers() ([]string, error) {
	return readPeersJSON(filepath.Join(r.store.path, "peers.json"))
}

func (r *remoteRaft) open() error {
	if err := r.setPeers(r.store.peers); err != nil {
		return err
	}

	go func() {
		for {
			select {
			case <-r.store.closing:
				return
			default:
			}

			ms, err := r.store.rpc.fetchMetaData(true)
			if err != nil {
				r.store.Logger.Printf("fetch metastore: %v", err)
				time.Sleep(time.Second)
				continue
			}
			r.updateMetaData(ms)
		}
	}()
	return nil
}

func (r *remoteRaft) close() error {
	return nil
}

// apply applies a serialized command to the raft log.
func (r *remoteRaft) apply(b []byte) error {
	return fmt.Errorf("cannot apply log while in remote raft state")
}

func (r *remoteRaft) initialize() error {
	return nil
}

func (r *remoteRaft) leader() string {
	if len(r.store.peers) == 0 {
		return ""
	}

	return r.store.peers[rand.Intn(len(r.store.peers))]
}

func (r *remoteRaft) isLeader() bool {
	return false
}

func (r *remoteRaft) lastIndex() uint64 {
	return r.store.cachedData().Index
}

func (r *remoteRaft) sync(index uint64, timeout time.Duration) error {
	//FIXME: jwilder: check index and timeout
	return r.store.invalidate()
}

func (r *remoteRaft) snapshot() error {
	return fmt.Errorf("cannot snapshot while in remote raft state")
}

func readPeersJSON(path string) ([]string, error) {
	// Read the file
	buf, err := ioutil.ReadFile(path)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	// Check for no peers
	if len(buf) == 0 {
		return nil, nil
	}

	// Decode the peers
	var peers []string
	dec := json.NewDecoder(bytes.NewReader(buf))
	if err := dec.Decode(&peers); err != nil {
		return nil, err
	}

	return peers, nil
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "net/http"
    "strconv"
    "strings"

    "github.com/golang/groupcache/lru"
)

var cache *lru.Cache

func handler(w http.ResponseWriter, r *http.Request) {
    var resp string

    respSize := r.URL.Path[1:]
    s, err := strconv.Atoi(respSize)
    if err != nil {
        s = 1024
    }

    val, ok := cache.Get(s)
    if ok {
        resp = val.(string)
    } else {
        resp = strings.Repeat("X", s)
        cache.Add(s, resp)
    }

    fmt.Fprintf(w, resp)
}

func main() {
    cache = lru.New(10)
    http.HandleFunc("/", handler)
    fmt.Println("Serving on :25000")
    http.ListenAndServe(":25000", nil)
}
<commit_msg>Fix gohttp performance<commit_after>package main

import (
    "bytes"
    "fmt"
    "net/http"
    "strconv"

    "github.com/golang/groupcache/lru"
)

var cache *lru.Cache

func handler(w http.ResponseWriter, r *http.Request) {
    var resp []byte

    respSize := r.URL.Path[1:]
    s, err := strconv.Atoi(respSize)
    if err != nil {
        s = 1024
    }

    val, ok := cache.Get(s)
    if ok {
        resp = val.([]byte)
    } else {
        resp = bytes.Repeat([]byte{'X'}, s)
        cache.Add(s, resp)
    }

    w.Write(resp)
}

func main() {
    cache = lru.New(10)
    http.HandleFunc("/", handler)
    fmt.Println("Serving on :25000")
    http.ListenAndServe(":25000", nil)
}
<|endoftext|>