<commit_before>package registry

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"runtime"
	"sync"
	"time"

	consul "github.com/hashicorp/consul/api"
	hash "github.com/mitchellh/hashstructure"
)

type consulRegistry struct {
	Address string
	Client  *consul.Client
	Options Options

	sync.Mutex
	register map[string]uint64
}

func newTransport(config *tls.Config) *http.Transport {
	if config == nil {
		config = &tls.Config{
			InsecureSkipVerify: true,
		}
	}

	t := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig:     config,
	}
	runtime.SetFinalizer(&t, func(tr **http.Transport) {
		(*tr).CloseIdleConnections()
	})
	return t
}

func newConsulRegistry(opts ...Option) Registry {
	var options Options
	for _, o := range opts {
		o(&options)
	}

	// use default config
	config := consul.DefaultConfig()

	// set timeout
	if options.Timeout > 0 {
		config.HttpClient.Timeout = options.Timeout
	}

	// check if there are any addrs
	if len(options.Addrs) > 0 {
		addr, port, err := net.SplitHostPort(options.Addrs[0])
		if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
			port = "8500"
			addr = options.Addrs[0]
			config.Address = fmt.Sprintf("%s:%s", addr, port)
		} else if err == nil {
			config.Address = fmt.Sprintf("%s:%s", addr, port)
		}
	}

	// requires secure connection?
	if options.Secure || options.TLSConfig != nil {
		config.Scheme = "https"
		// We're going to support InsecureSkipVerify
		config.HttpClient.Transport = newTransport(options.TLSConfig)
	}

	// create the client
	client, _ := consul.NewClient(config)

	cr := &consulRegistry{
		Address: config.Address,
		Client:  client,
		Options: options,
	}

	return cr
}

func (c *consulRegistry) Deregister(s *Service) error {
	if len(s.Nodes) == 0 {
		return errors.New("Require at least one node")
	}

	// delete our hash of the service
	c.Lock()
	delete(c.register, s.Name)
	c.Unlock()

	node := s.Nodes[0]
	return c.Client.Agent().ServiceDeregister(node.Id)
}

func (c *consulRegistry) Register(s *Service, opts ...RegisterOption) error {
	if len(s.Nodes) == 0 {
		return errors.New("Require at least one node")
	}

	var options RegisterOptions
	for _, o := range opts {
		o(&options)
	}

	// create hash of service; uint64
	h, err := hash.Hash(s, nil)
	if err != nil {
		return err
	}

	// use first node
	node := s.Nodes[0]

	// get existing hash
	c.Lock()
	v, ok := c.register[s.Name]
	c.Unlock()

	// if it's already registered and matches then just pass the check
	if ok && v == h {
		// if the err is nil we're all good, bail out
		// if not, we don't know what the state is, so full re-register
		if err := c.Client.Agent().PassTTL("service:"+node.Id, ""); err == nil {
			return nil
		}
	}

	// encode the tags
	tags := encodeMetadata(node.Metadata)
	tags = append(tags, encodeEndpoints(s.Endpoints)...)
	tags = append(tags, encodeVersion(s.Version)...)

	var check *consul.AgentServiceCheck

	// if the TTL is greater than 0 create an associated check
	if options.TTL > time.Duration(0) {
		check = &consul.AgentServiceCheck{
			TTL: fmt.Sprintf("%v", options.TTL),
		}
	}

	// register the service
	if err := c.Client.Agent().ServiceRegister(&consul.AgentServiceRegistration{
		ID:      node.Id,
		Name:    s.Name,
		Tags:    tags,
		Port:    node.Port,
		Address: node.Address,
		Check:   check,
	}); err != nil {
		return err
	}

	// save our hash of the service
	c.Lock()
	c.register[s.Name] = h
	c.Unlock()

	// if the TTL is 0 we don't mess with the checks
	if options.TTL == time.Duration(0) {
		return nil
	}

	// pass the healthcheck
	return c.Client.Agent().PassTTL("service:"+node.Id, "")
}

func (c *consulRegistry) GetService(name string) ([]*Service, error) {
	rsp, _, err := c.Client.Health().Service(name, "", true, nil)
	if err != nil {
		return nil, err
	}

	serviceMap := map[string]*Service{}

	for _, s := range rsp {
		if s.Service.Service != name {
			continue
		}

		// version is now a tag
		version, found := decodeVersion(s.Service.Tags)
		// service ID is now the node id
		id := s.Service.ID
		// key is always the version
		key := version
		// address is service address
		address := s.Service.Address

		// if we can't get the new type of version
		// use old the old ways
		if !found {
			// id was set as node
			id = s.Node.Node
			// key was service id
			key = s.Service.ID
			// version was service id
			version = s.Service.ID
			// address was address
			address = s.Node.Address
		}

		svc, ok := serviceMap[key]
		if !ok {
			svc = &Service{
				Endpoints: decodeEndpoints(s.Service.Tags),
				Name:      s.Service.Service,
				Version:   version,
			}
			serviceMap[key] = svc
		}

		svc.Nodes = append(svc.Nodes, &Node{
			Id:       id,
			Address:  address,
			Port:     s.Service.Port,
			Metadata: decodeMetadata(s.Service.Tags),
		})
	}

	var services []*Service
	for _, service := range serviceMap {
		services = append(services, service)
	}
	return services, nil
}

func (c *consulRegistry) ListServices() ([]*Service, error) {
	rsp, _, err := c.Client.Catalog().Services(nil)
	if err != nil {
		return nil, err
	}

	var services []*Service

	for service, _ := range rsp {
		services = append(services, &Service{Name: service})
	}

	return services, nil
}

func (c *consulRegistry) Watch() (Watcher, error) {
	return newConsulWatcher(c)
}

func (c *consulRegistry) String() string {
	return "consul"
}
<commit_msg>God damn you nil map<commit_after>package registry

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"runtime"
	"sync"
	"time"

	consul "github.com/hashicorp/consul/api"
	hash "github.com/mitchellh/hashstructure"
)

type consulRegistry struct {
	Address string
	Client  *consul.Client
	Options Options

	sync.Mutex
	register map[string]uint64
}

func newTransport(config *tls.Config) *http.Transport {
	if config == nil {
		config = &tls.Config{
			InsecureSkipVerify: true,
		}
	}

	t := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig:     config,
	}
	runtime.SetFinalizer(&t, func(tr **http.Transport) {
		(*tr).CloseIdleConnections()
	})
	return t
}

func newConsulRegistry(opts ...Option) Registry {
	var options Options
	for _, o := range opts {
		o(&options)
	}

	// use default config
	config := consul.DefaultConfig()

	// set timeout
	if options.Timeout > 0 {
		config.HttpClient.Timeout = options.Timeout
	}

	// check if there are any addrs
	if len(options.Addrs) > 0 {
		addr, port, err := net.SplitHostPort(options.Addrs[0])
		if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
			port = "8500"
			addr = options.Addrs[0]
			config.Address = fmt.Sprintf("%s:%s", addr, port)
		} else if err == nil {
			config.Address = fmt.Sprintf("%s:%s", addr, port)
		}
	}

	// requires secure connection?
	if options.Secure || options.TLSConfig != nil {
		config.Scheme = "https"
		// We're going to support InsecureSkipVerify
		config.HttpClient.Transport = newTransport(options.TLSConfig)
	}

	// create the client
	client, _ := consul.NewClient(config)

	cr := &consulRegistry{
		Address:  config.Address,
		Client:   client,
		Options:  options,
		register: make(map[string]uint64),
	}

	return cr
}

func (c *consulRegistry) Deregister(s *Service) error {
	if len(s.Nodes) == 0 {
		return errors.New("Require at least one node")
	}

	// delete our hash of the service
	c.Lock()
	delete(c.register, s.Name)
	c.Unlock()

	node := s.Nodes[0]
	return c.Client.Agent().ServiceDeregister(node.Id)
}

func (c *consulRegistry) Register(s *Service, opts ...RegisterOption) error {
	if len(s.Nodes) == 0 {
		return errors.New("Require at least one node")
	}

	var options RegisterOptions
	for _, o := range opts {
		o(&options)
	}

	// create hash of service; uint64
	h, err := hash.Hash(s, nil)
	if err != nil {
		return err
	}

	// use first node
	node := s.Nodes[0]

	// get existing hash
	c.Lock()
	v, ok := c.register[s.Name]
	c.Unlock()

	// if it's already registered and matches then just pass the check
	if ok && v == h {
		// if the err is nil we're all good, bail out
		// if not, we don't know what the state is, so full re-register
		if err := c.Client.Agent().PassTTL("service:"+node.Id, ""); err == nil {
			return nil
		}
	}

	// encode the tags
	tags := encodeMetadata(node.Metadata)
	tags = append(tags, encodeEndpoints(s.Endpoints)...)
	tags = append(tags, encodeVersion(s.Version)...)

	var check *consul.AgentServiceCheck

	// if the TTL is greater than 0 create an associated check
	if options.TTL > time.Duration(0) {
		check = &consul.AgentServiceCheck{
			TTL: fmt.Sprintf("%v", options.TTL),
		}
	}

	// register the service
	if err := c.Client.Agent().ServiceRegister(&consul.AgentServiceRegistration{
		ID:      node.Id,
		Name:    s.Name,
		Tags:    tags,
		Port:    node.Port,
		Address: node.Address,
		Check:   check,
	}); err != nil {
		return err
	}

	// save our hash of the service
	c.Lock()
	c.register[s.Name] = h
	c.Unlock()

	// if the TTL is 0 we don't mess with the checks
	if options.TTL == time.Duration(0) {
		return nil
	}

	// pass the healthcheck
	return c.Client.Agent().PassTTL("service:"+node.Id, "")
}

func (c *consulRegistry) GetService(name string) ([]*Service, error) {
	rsp, _, err := c.Client.Health().Service(name, "", true, nil)
	if err != nil {
		return nil, err
	}

	serviceMap := map[string]*Service{}

	for _, s := range rsp {
		if s.Service.Service != name {
			continue
		}

		// version is now a tag
		version, found := decodeVersion(s.Service.Tags)
		// service ID is now the node id
		id := s.Service.ID
		// key is always the version
		key := version
		// address is service address
		address := s.Service.Address

		// if we can't get the new type of version
		// use old the old ways
		if !found {
			// id was set as node
			id = s.Node.Node
			// key was service id
			key = s.Service.ID
			// version was service id
			version = s.Service.ID
			// address was address
			address = s.Node.Address
		}

		svc, ok := serviceMap[key]
		if !ok {
			svc = &Service{
				Endpoints: decodeEndpoints(s.Service.Tags),
				Name:      s.Service.Service,
				Version:   version,
			}
			serviceMap[key] = svc
		}

		svc.Nodes = append(svc.Nodes, &Node{
			Id:       id,
			Address:  address,
			Port:     s.Service.Port,
			Metadata: decodeMetadata(s.Service.Tags),
		})
	}

	var services []*Service
	for _, service := range serviceMap {
		services = append(services, service)
	}
	return services, nil
}

func (c *consulRegistry) ListServices() ([]*Service, error) {
	rsp, _, err := c.Client.Catalog().Services(nil)
	if err != nil {
		return nil, err
	}

	var services []*Service

	for service, _ := range rsp {
		services = append(services, &Service{Name: service})
	}

	return services, nil
}

func (c *consulRegistry) Watch() (Watcher, error) {
	return newConsulWatcher(c)
}

func (c *consulRegistry) String() string {
	return "consul"
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2015 Arnaud Vazard
//
// See LICENSE file.
package xkcd

import (
	"encoding/json"
	"fmt"
	"github.com/romainletendart/goxxx/core"
	"github.com/thoj/go-ircevent"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
)

const (
	HELP_XKCD     string = "\t!xkcd \t\t\t\t\t\t=> Return the last XKCD comic"
	HELP_XKCD_NUM string = "\t!xkcd <comic number> \t\t=> Return the XKCD comic corresponding to the number"

	URL_SITE        string = "https://xkcd.com/%d/"
	URL_JSON        string = "https://xkcd.com/%d/info.0.json"
	URL_JSON_LATEST string = "https://xkcd.com/info.0.json"
)

type xkcd struct {
	Img   string `json:"img"`
	Link  string `json:"link"`
	Num   int64  `json:"num"`
	Title string `json:"title"`
}

func getComic(number int64) *xkcd {
	var url string
	if number <= 0 { // Get latest comic
		url = URL_JSON_LATEST
	} else { // Get the comic corresponding to "number"
		url = fmt.Sprintf(URL_JSON, number)
	}

	response, err := http.Get(url)
	if err != nil {
		log.Println(err)
		return nil
	}
	defer response.Body.Close()

	jsonDataFromHttp, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Println(err)
		return nil
	}
	response.Body.Close()

	result := new(xkcd)
	err = json.Unmarshal(jsonDataFromHttp, result)
	if err != nil {
		log.Println(err)
		return nil
	}
	result.Link = fmt.Sprintf(URL_SITE, result.Num)
	return result
}

func HandleXKCDCmd(event *irc.Event, callback func(*core.ReplyCallbackData)) bool {
	if callback == nil {
		log.Println("Callback nil for the HandleXKCDCmd function")
		return false
	}

	fields := strings.Fields(event.Message())
	// fields[0]  => Command
	// fields[1]  => Comic #

	count := len(fields)
	if count == 0 || fields[0] != "!xkcd" {
		log.Println("XKCD: Not an XKCD command")
		return false
	}

	var message string
	if count < 2 {
		comic := getComic(0)
		if comic == nil {
			log.Println("XKCD: No comic return by getComic")
			return false
		}
		message = fmt.Sprintf("Last XKCD Comic: %s => %s", comic.Title, comic.Link)
	} else {
		number, err := strconv.ParseInt(fields[1], 10, 64)
		if err != nil {
			log.Println(err)
			return false
		}

		if number < 0 || getComic(0).Num < number {
			message = fmt.Sprintf("There is no XKCD comic #%d", number)
		} else {
			comic := getComic(number)
			if comic == nil {
				log.Println("XKCD: No comic return by getComic")
				return false
			}
			message = fmt.Sprintf("XKCD Comic #%d: %s => %s", comic.Num, comic.Title, comic.Link)
		}
	}
	log.Println(message)
	callback(&core.ReplyCallbackData{Message: message, Nick: event.Nick})
	return true
}
<commit_msg>XKCD: Added some comments<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2015 Arnaud Vazard
//
// See LICENSE file.

// Package to retrieve XKCD Comics
package xkcd

import (
	"encoding/json"
	"fmt"
	"github.com/romainletendart/goxxx/core"
	"github.com/thoj/go-ircevent"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
)

const (
	HELP_XKCD     string = "\t!xkcd \t\t\t\t\t\t=> Return the last XKCD comic"                               // Help message for the !xkcd command
	HELP_XKCD_NUM string = "\t!xkcd <comic number> \t\t=> Return the XKCD comic corresponding to the number" // Help message for the !xkcd <comic number> command

	URL_SITE        string = "https://xkcd.com/%d/"            // Website URL format string
	URL_JSON        string = "https://xkcd.com/%d/info.0.json" // JSON URL format string
	URL_JSON_LATEST string = "https://xkcd.com/info.0.json"    // JSON URL for the current comic
)

type xkcd struct {
	Img   string `json:"img"`
	Link  string `json:"link"`
	Num   int64  `json:"num"`
	Title string `json:"title"`
}

// If number is superior to 0 attempt to get informations on the corresponding comic, else return the inforamtions for the current comic.
// In case of error return nil
func getComic(number int64) *xkcd {
	var url string
	if number <= 0 {
		// Get latest comic
		url = URL_JSON_LATEST
	} else {
		// Get the comic corresponding to "number"
		url = fmt.Sprintf(URL_JSON, number)
	}

	response, err := http.Get(url)
	if err != nil {
		log.Println(err)
		return nil
	}
	defer response.Body.Close()

	jsonDataFromHttp, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Println(err)
		return nil
	}
	response.Body.Close()

	result := new(xkcd)
	err = json.Unmarshal(jsonDataFromHttp, result)
	if err != nil {
		log.Println(err)
		return nil
	}
	// Add the full website link to the structure
	result.Link = fmt.Sprintf(URL_SITE, result.Num)
	return result
}

// Handler for the XKCD commands
func HandleXKCDCmd(event *irc.Event, callback func(*core.ReplyCallbackData)) bool {
	if callback == nil {
		log.Println("Callback nil for the HandleXKCDCmd function")
		return false
	}

	fields := strings.Fields(event.Message())
	// fields[0]  => Command
	// fields[1]  => Comic #

	count := len(fields)
	if count == 0 || fields[0] != "!xkcd" {
		return false
	}

	var message string
	if count < 2 {
		comic := getComic(0)
		if comic == nil {
			log.Println("XKCD: No comic return by getComic")
			return false
		}
		message = fmt.Sprintf("Last XKCD Comic: %s => %s", comic.Title, comic.Link)
	} else {
		number, err := strconv.ParseInt(fields[1], 10, 64)
		if err != nil {
			log.Println(err)
			return false
		}

		if number < 0 || getComic(0).Num < number {
			message = fmt.Sprintf("There is no XKCD comic #%d", number)
		} else {
			comic := getComic(number)
			if comic == nil {
				log.Println("XKCD: No comic return by getComic")
				return false
			}
			message = fmt.Sprintf("XKCD Comic #%d: %s => %s", comic.Num, comic.Title, comic.Link)
		}
	}
	log.Println(message)
	callback(&core.ReplyCallbackData{Message: message, Nick: event.Nick})
	return true
}
<|endoftext|>


<commit_before>package requests

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"mime"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/cactus/go-statsd-client/statsd"
	"github.com/twitchscience/aws_utils/logger"
	"github.com/twitchscience/scoop_protocol/spade"
	"github.com/twitchscience/spade_edge/loggers"
)

var (
	hostSamplingRate = float32(0.01)
	xDomainContents  = func() (b []byte) {
		filename := os.Getenv("CROSS_DOMAIN_LOCATION")
		if filename == "" {
			filename = "../build/config/crossdomain.xml"
		}
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			logger.WithError(err).Fatal("Cross domain file not found")
		}
		return
	}()
	xmlApplicationType = mime.TypeByExtension(".xml")
	xarth              = []byte("XARTH")
	dataFlag           = []byte("data=")
	// from https://commons.wikimedia.org/wiki/File:Transparent.gif
	transparentPixel = []byte{
		71, 73, 70, 56, 57, 97, 1, 0, 1, 0,
		128, 0, 0, 0, 0, 0, 255, 255, 255,
		33, 249, 4, 1, 0, 0, 0, 0, 44, 0,
		0, 0, 0, 1, 0, 1, 0, 0, 2, 1, 68, 0, 59,
	}
)

const corsMaxAge = "86400" // One day

// EdgeLoggers represent the different kind of loggers for Spade events
type EdgeLoggers struct {
	sync.WaitGroup
	closed             chan struct{}
	S3EventLogger      loggers.SpadeEdgeLogger
	KinesisEventLogger loggers.SpadeEdgeLogger
}

// NewEdgeLoggers returns a new instance of an EdgeLoggers struct pre-filled
// wuth UndefinedLogger logger instances
func NewEdgeLoggers() *EdgeLoggers {
	return &EdgeLoggers{
		closed:             make(chan struct{}),
		S3EventLogger:      loggers.UndefinedLogger{},
		KinesisEventLogger: loggers.UndefinedLogger{},
	}
}

func (e *EdgeLoggers) log(event *spade.Event, context *requestContext) error {
	e.Add(1)
	defer e.Done()

	// If reading from the `closed` channel succeeds, the logger is closed.
	select {
	case <-e.closed:
		return errors.New("Loggers are shutting down")
	default: // Make this a non-blocking select
	}

	eventErr := e.S3EventLogger.Log(event)
	kinesisErr := e.KinesisEventLogger.Log(event)

	context.recordLoggerAttempt(eventErr, "event")
	context.recordLoggerAttempt(kinesisErr, "kinesis")

	if eventErr != nil &&
		kinesisErr != nil {
		return errors.New("Failed to store the event in any of the loggers")
	}

	return nil
}

// Close closes the loggers
func (e *EdgeLoggers) Close() {
	close(e.closed)
	e.Wait()

	e.KinesisEventLogger.Close()
	e.S3EventLogger.Close()
}

// SpadeHandler handles http requests and forwards them to the EdgeLoggers
type SpadeHandler struct {
	StatLogger  statsd.Statter
	EdgeLoggers *EdgeLoggers
	Time        func() time.Time // Defaults to time.Now
	EdgeType    string
	corsOrigins map[string]bool
	instanceID  string

	// eventCount counts the number of event requests handled. It is used in
	// uuid generation. eventCount is read and written from multiple go routines
	// so any access to it should go through sync/atomic
	eventCount             uint64
	eventInURISamplingRate float32
}

// NewSpadeHandler returns a new instance of SpadeHandler
func NewSpadeHandler(stats statsd.Statter, loggers *EdgeLoggers, instanceID string, CORSOrigins []string, eventInURISamplingRate float32, edgeType string) *SpadeHandler {
	h := &SpadeHandler{
		StatLogger:             stats,
		EdgeLoggers:            loggers,
		Time:                   time.Now,
		EdgeType:               edgeType,
		instanceID:             instanceID,
		corsOrigins:            make(map[string]bool),
		eventInURISamplingRate: eventInURISamplingRate,
	}

	for _, origin := range CORSOrigins {
		trimmedOrigin := strings.TrimSpace(origin)
		if trimmedOrigin != "" {
			h.corsOrigins[trimmedOrigin] = true
		}
	}
	return h
}

func parseLastForwarder(header string) net.IP {
	var clientIP string
	comma := strings.LastIndex(header, ",")
	if comma > -1 && comma < len(header)+1 {
		clientIP = header[comma+1:]
	} else {
		clientIP = header
	}

	return net.ParseIP(strings.TrimSpace(clientIP))
}

const (
	ipForwardHeader      = "X-Forwarded-For"
	badEndpoint          = "FourOhFour"
	nTimers              = 5
	maxBytesPerRequest   = 500 * 1024
	largeBodyErrorString = "http: request body too large" // Magic error string from the http pkg
	maxUserAgentBytes    = 1024
)

var allowedMethods = map[string]bool{
	"GET":     true,
	"POST":    true,
	"OPTIONS": true,
}
var allowedMethodsHeader string // Comma-separated version of allowedMethods

func (s *SpadeHandler) logLargeRequestError(r *http.Request, data string) {
	_ = s.StatLogger.Inc("large_request", 1, 0.1)
	head := truncate(data, 100)
	logger.WithField("sent_from", r.Header.Get("X-Forwarded-For")).
		WithField("user_agent", r.Header.Get("User-Agent")).
		WithField("content_length", r.ContentLength).
		WithField("data_head", head).
		Warn("Request larger than 500KB, rejecting.")
}

func (s *SpadeHandler) logLargeUserAgentError(r *http.Request, data string) {
	_ = s.StatLogger.Inc("large_user_agent", 1, 0.1)
	head := truncate(data, 100)
	userAgent := truncate(r.Header.Get("User-Agent"), 100)
	logger.WithField("user_agent", userAgent).
		WithField("data_head", head).
		Warn(fmt.Sprintf("User agent larger than %d bytes, dropping.", maxUserAgentBytes))
}

func truncate(s string, max int) string {
	if len(s) > max {
		return s[:max]
	}

	return s
}

func sanitizeHostValue(host string) string {
	if host == "" {
		return ""
	}
	hostWithoutPort := strings.Split(strings.ToLower(strings.TrimSpace(host)), ":")[0]
	return strings.Replace(hostWithoutPort, ".", "_", -1)
}

func (s *SpadeHandler) handleSpadeRequests(r *http.Request, values url.Values, context *requestContext) int {
	statTimer := newTimerInstance()

	xForwardedFor := r.Header.Get(context.IPHeader)
	clientIP := parseLastForwarder(xForwardedFor)

	context.Timers["ip"] = statTimer.stopTiming()

	err := r.ParseForm()
	if err != nil {
		if err.Error() == largeBodyErrorString {
			s.logLargeRequestError(r, "")
			return http.StatusRequestEntityTooLarge
		}
		return http.StatusBadRequest
	}

	if _, ok := values["data"]; ok {
		_ = s.StatLogger.Inc("event_in_URI", 1, s.eventInURISamplingRate)
	}

	if len(r.RequestURI) > 8192 {
		_ = s.StatLogger.Inc("large_URI", 1, 1)
	}

	if host := sanitizeHostValue(r.Host); len(host) > 0 {
		_ = s.StatLogger.Inc(fmt.Sprintf("requests.hosts.%s", host), 1, hostSamplingRate)
	}

	data := r.Form.Get("data")
	if data == "" && r.Method == "POST" {
		// if we're here then our clients have POSTed us something weird,
		// for example, something that maybe
		// application/x-www-form-urlencoded but with the Content-Type
		// header set incorrectly... best effort here on out

		var b []byte
		b, err = ioutil.ReadAll(r.Body)
		if err != nil {
			if err.Error() == largeBodyErrorString {
				s.logLargeRequestError(r, string(b))
				return http.StatusRequestEntityTooLarge
			}
			return http.StatusBadRequest
		}
		if bytes.Equal(b[:5], dataFlag) {
			context.BadClient = true
			b = b[5:]
		}
		data = string(b)

	}
	if data == "" {
		return http.StatusBadRequest
	}

	bData := []byte(data)
	if len(bData) > maxBytesPerRequest {
		s.logLargeRequestError(r, data)
		return http.StatusRequestEntityTooLarge
	}

	context.Timers["data"] = statTimer.stopTiming()

	count := atomic.AddUint64(&s.eventCount, 1)
	uuid := fmt.Sprintf("%s-%08x-%08x", s.instanceID, context.Now.Unix(), count)
	context.Timers["uuid"] = statTimer.stopTiming()

	var userAgent string
	if values.Get("ua") == "1" {
		userAgent = r.Header.Get("User-Agent")
		// anything over the max is likely garbage data
		if len(userAgent) > maxUserAgentBytes {
			s.logLargeUserAgentError(r, data)
			userAgent = ""
		}
	}

	event := spade.NewEvent(
		context.Now,
		clientIP,
		xForwardedFor,
		uuid,
		data,
		userAgent,
		s.EdgeType,
	)

	defer func() {
		context.Timers["write"] = statTimer.stopTiming()
	}()

	err = s.EdgeLoggers.log(event, context)

	if err != nil {
		return http.StatusBadRequest
	}

	if shouldWritePixel(values) {
		return http.StatusOK
	}

	return http.StatusNoContent
}

func (s *SpadeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !allowedMethods[r.Method] {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	w.Header().Set("Vary", "Origin")

	origin := r.Header.Get("Origin")
	if s.corsOrigins[origin] {
		w.Header().Set("Access-Control-Allow-Origin", origin)
		w.Header().Set("Access-Control-Allow-Methods", allowedMethodsHeader)
	}

	if r.Method == "OPTIONS" {
		w.Header().Set("Access-Control-Max-Age", corsMaxAge)
		w.WriteHeader(http.StatusOK)
		return
	}

	context := &requestContext{
		Now:       s.Time(),
		Method:    r.Method,
		Endpoint:  r.URL.Path,
		IPHeader:  ipForwardHeader,
		Timers:    make(map[string]time.Duration, nTimers),
		BadClient: false,
	}
	timer := newTimerInstance()
	status := s.serve(w, r, context)
	_ = s.StatLogger.Inc(fmt.Sprintf("status_code.%d", status), 1, 0.001)
	context.setStatus(status)
	context.Timers["http"] = timer.stopTiming()

	context.recordStats(s.StatLogger)
}

func (s *SpadeHandler) serve(w http.ResponseWriter, r *http.Request, context *requestContext) int {
	var status int
	path := r.URL.Path
	if strings.HasPrefix(path, "/v1/") {
		path = "/track"
	}
	switch path {
	case "/crossdomain.xml":
		w.Header().Add("Content-Type", xmlApplicationType)
		_, err := w.Write(xDomainContents)
		if err != nil {
			logger.WithError(err).Error("Unable to write crossdomain.xml contents")
			return http.StatusInternalServerError
		}
		return http.StatusOK
	case "/healthcheck":
		status = http.StatusOK
	case "/xarth":
		_, err := w.Write(xarth)
		if err != nil {
			logger.WithError(err).Error("Error writing XARTH response")
			return http.StatusInternalServerError
		}
		return http.StatusOK
	// Accepted tracking endpoints.
	case "/", "/track", "/track/":
		values := r.URL.Query()
		status = s.handleSpadeRequests(r, values, context)

		if shouldWritePixel(values) {
			if err := writePixel(w); err != nil {
				logger.WithError(err).Error("Error writing transparent pixel response")
				status = http.StatusInternalServerError
			} else {
				// header and body have already been written
				return http.StatusOK
			}
		}
	// dont track everything else
	default:
		context.Endpoint = badEndpoint
		status = http.StatusNotFound
	}
	w.WriteHeader(status)
	return status
}

func shouldWritePixel(values url.Values) bool {
	return values.Get("img") == "1"
}

func writePixel(w http.ResponseWriter) error {
	w.Header().Set("Content-Type", "image/gif")
	w.Header().Set("Cache-Control", "no-cache, max-age=0")
	_, err := w.Write(transparentPixel)
	return err
}

func init() {
	var allowedMethodsList []string
	for k := range allowedMethods {
		allowedMethodsList = append(allowedMethodsList, k)
	}
	allowedMethodsHeader = strings.Join(allowedMethodsList, ", ")
}
<commit_msg>Adding stats tracking of bad request statuses<commit_after>package requests

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"mime"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/cactus/go-statsd-client/statsd"
	"github.com/twitchscience/aws_utils/logger"
	"github.com/twitchscience/scoop_protocol/spade"
	"github.com/twitchscience/spade_edge/loggers"
)

var (
	hostSamplingRate = float32(0.01)
	xDomainContents  = func() (b []byte) {
		filename := os.Getenv("CROSS_DOMAIN_LOCATION")
		if filename == "" {
			filename = "../build/config/crossdomain.xml"
		}
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			logger.WithError(err).Fatal("Cross domain file not found")
		}
		return
	}()
	xmlApplicationType = mime.TypeByExtension(".xml")
	xarth              = []byte("XARTH")
	dataFlag           = []byte("data=")
	// from https://commons.wikimedia.org/wiki/File:Transparent.gif
	transparentPixel = []byte{
		71, 73, 70, 56, 57, 97, 1, 0, 1, 0,
		128, 0, 0, 0, 0, 0, 255, 255, 255,
		33, 249, 4, 1, 0, 0, 0, 0, 44, 0,
		0, 0, 0, 1, 0, 1, 0, 0, 2, 1, 68, 0, 59,
	}
)

const corsMaxAge = "86400" // One day

// EdgeLoggers represent the different kind of loggers for Spade events
type EdgeLoggers struct {
	sync.WaitGroup
	closed             chan struct{}
	S3EventLogger      loggers.SpadeEdgeLogger
	KinesisEventLogger loggers.SpadeEdgeLogger
}

// NewEdgeLoggers returns a new instance of an EdgeLoggers struct pre-filled
// wuth UndefinedLogger logger instances
func NewEdgeLoggers() *EdgeLoggers {
	return &EdgeLoggers{
		closed:             make(chan struct{}),
		S3EventLogger:      loggers.UndefinedLogger{},
		KinesisEventLogger: loggers.UndefinedLogger{},
	}
}

func (e *EdgeLoggers) log(event *spade.Event, context *requestContext) error {
	e.Add(1)
	defer e.Done()

	// If reading from the `closed` channel succeeds, the logger is closed.
	select {
	case <-e.closed:
		return errors.New("Loggers are shutting down")
	default: // Make this a non-blocking select
	}

	eventErr := e.S3EventLogger.Log(event)
	kinesisErr := e.KinesisEventLogger.Log(event)

	context.recordLoggerAttempt(eventErr, "event")
	context.recordLoggerAttempt(kinesisErr, "kinesis")

	if eventErr != nil &&
		kinesisErr != nil {
		return errors.New("Failed to store the event in any of the loggers")
	}

	return nil
}

// Close closes the loggers
func (e *EdgeLoggers) Close() {
	close(e.closed)
	e.Wait()

	e.KinesisEventLogger.Close()
	e.S3EventLogger.Close()
}

// SpadeHandler handles http requests and forwards them to the EdgeLoggers
type SpadeHandler struct {
	StatLogger  statsd.Statter
	EdgeLoggers *EdgeLoggers
	Time        func() time.Time // Defaults to time.Now
	EdgeType    string
	corsOrigins map[string]bool
	instanceID  string

	// eventCount counts the number of event requests handled. It is used in
	// uuid generation. eventCount is read and written from multiple go routines
	// so any access to it should go through sync/atomic
	eventCount             uint64
	eventInURISamplingRate float32
}

// NewSpadeHandler returns a new instance of SpadeHandler
func NewSpadeHandler(stats statsd.Statter, loggers *EdgeLoggers, instanceID string, CORSOrigins []string, eventInURISamplingRate float32, edgeType string) *SpadeHandler {
	h := &SpadeHandler{
		StatLogger:             stats,
		EdgeLoggers:            loggers,
		Time:                   time.Now,
		EdgeType:               edgeType,
		instanceID:             instanceID,
		corsOrigins:            make(map[string]bool),
		eventInURISamplingRate: eventInURISamplingRate,
	}

	for _, origin := range CORSOrigins {
		trimmedOrigin := strings.TrimSpace(origin)
		if trimmedOrigin != "" {
			h.corsOrigins[trimmedOrigin] = true
		}
	}
	return h
}

func parseLastForwarder(header string) net.IP {
	var clientIP string
	comma := strings.LastIndex(header, ",")
	if comma > -1 && comma < len(header)+1 {
		clientIP = header[comma+1:]
	} else {
		clientIP = header
	}

	return net.ParseIP(strings.TrimSpace(clientIP))
}

const (
	ipForwardHeader      = "X-Forwarded-For"
	badEndpoint          = "FourOhFour"
	nTimers              = 5
	maxBytesPerRequest   = 500 * 1024
	largeBodyErrorString = "http: request body too large" // Magic error string from the http pkg
	maxUserAgentBytes    = 1024
)

var allowedMethods = map[string]bool{
	"GET":     true,
	"POST":    true,
	"OPTIONS": true,
}
var allowedMethodsHeader string // Comma-separated version of allowedMethods

func (s *SpadeHandler) logLargeRequestError(r *http.Request, data string) {
	_ = s.StatLogger.Inc("large_request", 1, 0.1)
	head := truncate(data, 100)
	logger.WithField("sent_from", r.Header.Get("X-Forwarded-For")).
		WithField("user_agent", r.Header.Get("User-Agent")).
		WithField("content_length", r.ContentLength).
		WithField("data_head", head).
		Warn("Request larger than 500KB, rejecting.")
}

func (s *SpadeHandler) logLargeUserAgentError(r *http.Request, data string) {
	_ = s.StatLogger.Inc("large_user_agent", 1, 0.1)
	head := truncate(data, 100)
	userAgent := truncate(r.Header.Get("User-Agent"), 100)
	logger.WithField("user_agent", userAgent).
		WithField("data_head", head).
		Warn(fmt.Sprintf("User agent larger than %d bytes, dropping.", maxUserAgentBytes))
}

func truncate(s string, max int) string {
	if len(s) > max {
		return s[:max]
	}

	return s
}

func sanitizeHostValue(host string) string {
	if host == "" {
		return ""
	}
	hostWithoutPort := strings.Split(strings.ToLower(strings.TrimSpace(host)), ":")[0]
	return strings.Replace(hostWithoutPort, ".", "_", -1)
}

func (s *SpadeHandler) handleSpadeRequests(r *http.Request, values url.Values, context *requestContext) int {
	statTimer := newTimerInstance()

	xForwardedFor := r.Header.Get(context.IPHeader)
	clientIP := parseLastForwarder(xForwardedFor)

	context.Timers["ip"] = statTimer.stopTiming()

	err := r.ParseForm()
	if err != nil {
		if err.Error() == largeBodyErrorString {
			s.logLargeRequestError(r, "")
			return http.StatusRequestEntityTooLarge
		}
		_ = s.StatLogger.Inc("bad_request.parse_form", 1, 0.01)
		return http.StatusBadRequest
	}

	if _, ok := values["data"]; ok {
		_ = s.StatLogger.Inc("event_in_URI", 1, s.eventInURISamplingRate)
	}

	if len(r.RequestURI) > 8192 {
		_ = s.StatLogger.Inc("large_URI", 1, 1)
	}

	if host := sanitizeHostValue(r.Host); len(host) > 0 {
		_ = s.StatLogger.Inc(fmt.Sprintf("requests.hosts.%s", host), 1, hostSamplingRate)
	}

	data := r.Form.Get("data")
	if data == "" && r.Method == "POST" {
		// if we're here then our clients have POSTed us something weird,
		// for example, something that maybe
		// application/x-www-form-urlencoded but with the Content-Type
		// header set incorrectly... best effort here on out

		var b []byte
		b, err = ioutil.ReadAll(r.Body)
		if err != nil {
			if err.Error() == largeBodyErrorString {
				s.logLargeRequestError(r, string(b))
				return http.StatusRequestEntityTooLarge
			}
			_ = s.StatLogger.Inc("bad_request.read_data", 1, 0.01)
			return http.StatusBadRequest
		}
		if bytes.Equal(b[:5], dataFlag) {
			context.BadClient = true
			b = b[5:]
		}
		data = string(b)

	}
	if data == "" {
		_ = s.StatLogger.Inc("bad_request.empty", 1, 0.01)
		return http.StatusBadRequest
	}

	bData := []byte(data)
	if len(bData) > maxBytesPerRequest {
		s.logLargeRequestError(r, data)
		return http.StatusRequestEntityTooLarge
	}

	context.Timers["data"] = statTimer.stopTiming()

	count := atomic.AddUint64(&s.eventCount, 1)
	uuid := fmt.Sprintf("%s-%08x-%08x", s.instanceID, context.Now.Unix(), count)
	context.Timers["uuid"] = statTimer.stopTiming()

	var userAgent string
	if values.Get("ua") == "1" {
		userAgent = r.Header.Get("User-Agent")
		// anything over the max is likely garbage data
		if len(userAgent) > maxUserAgentBytes {
			s.logLargeUserAgentError(r, data)
			userAgent = ""
		}
	}

	event := spade.NewEvent(
		context.Now,
		clientIP,
		xForwardedFor,
		uuid,
		data,
		userAgent,
		s.EdgeType,
	)

	defer func() {
		context.Timers["write"] = statTimer.stopTiming()
	}()

	err = s.EdgeLoggers.log(event, context)

	if err != nil {
		_ = s.StatLogger.Inc("bad_request.write_failure", 1, 0.01)
		return http.StatusBadRequest
	}

	if shouldWritePixel(values) {
		return http.StatusOK
	}

	return http.StatusNoContent
}

func (s *SpadeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !allowedMethods[r.Method] {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	w.Header().Set("Vary", "Origin")

	origin := r.Header.Get("Origin")
	if s.corsOrigins[origin] {
		w.Header().Set("Access-Control-Allow-Origin", origin)
		w.Header().Set("Access-Control-Allow-Methods", allowedMethodsHeader)
	}

	if r.Method == "OPTIONS" {
		w.Header().Set("Access-Control-Max-Age", corsMaxAge)
		w.WriteHeader(http.StatusOK)
		return
	}

	context := &requestContext{
		Now:       s.Time(),
		Method:    r.Method,
		Endpoint:  r.URL.Path,
		IPHeader:  ipForwardHeader,
		Timers:    make(map[string]time.Duration, nTimers),
		BadClient: false,
	}
	timer := newTimerInstance()
	status := s.serve(w, r, context)
	_ = s.StatLogger.Inc(fmt.Sprintf("status_code.%d", status), 1, 0.001)
	context.setStatus(status)
	context.Timers["http"] = timer.stopTiming()

	context.recordStats(s.StatLogger)
}

func (s *SpadeHandler) serve(w http.ResponseWriter, r *http.Request, context *requestContext) int {
	var status int
	path := r.URL.Path
	if strings.HasPrefix(path, "/v1/") {
		path = "/track"
	}
	switch path {
	case "/crossdomain.xml":
		w.Header().Add("Content-Type", xmlApplicationType)
		_, err := w.Write(xDomainContents)
		if err != nil {
			logger.WithError(err).Error("Unable to write crossdomain.xml contents")
			return http.StatusInternalServerError
		}
		return http.StatusOK
	case "/healthcheck":
		status = http.StatusOK
	case "/xarth":
		_, err := w.Write(xarth)
		if err != nil {
			logger.WithError(err).Error("Error writing XARTH response")
			return http.StatusInternalServerError
		}
		return http.StatusOK
	// Accepted tracking endpoints.
	case "/", "/track", "/track/":
		values := r.URL.Query()
		status = s.handleSpadeRequests(r, values, context)

		if shouldWritePixel(values) {
			if err := writePixel(w); err != nil {
				logger.WithError(err).Error("Error writing transparent pixel response")
				status = http.StatusInternalServerError
			} else {
				// header and body have already been written
				return http.StatusOK
			}
		}
	// dont track everything else
	default:
		context.Endpoint = badEndpoint
		status = http.StatusNotFound
	}
	w.WriteHeader(status)
	return status
}

func shouldWritePixel(values url.Values) bool {
	return values.Get("img") == "1"
}

func writePixel(w http.ResponseWriter) error {
	w.Header().Set("Content-Type", "image/gif")
	w.Header().Set("Cache-Control", "no-cache, max-age=0")
	_, err := w.Write(transparentPixel)
	return err
}

func init() {
	var allowedMethodsList []string
	for k := range allowedMethods {
		allowedMethodsList = append(allowedMethodsList, k)
	}
	allowedMethodsHeader = strings.Join(allowedMethodsList, ", ")
}
<|endoftext|>


<commit_before>package operationtype

// Type is a numeric code indentifying the type of an Operation.
type Type int64

// Possible values for Type
//
// WARNING: The type codes are stored in the database, so this list of
//          definitions should be normally append-only. Any other change
//          requires a database update.
const (
	Unknown Type = iota
	ClusterBootstrap
	ClusterJoin
	BackupCreate
	BackupRename
	BackupRestore
	BackupRemove
	ConsoleShow
	InstanceCreate
	InstanceUpdate
	InstanceRename
	InstanceMigrate
	InstanceLiveMigrate
	InstanceFreeze
	InstanceUnfreeze
	InstanceDelete
	InstanceStart
	InstanceStop
	InstanceRestart
	CommandExec
	SnapshotCreate
	SnapshotRename
	SnapshotRestore
	SnapshotTransfer
	SnapshotUpdate
	SnapshotDelete
	ImageDownload
	ImageDelete
	ImageToken
	ImageRefresh
	VolumeCopy
	VolumeCreate
	VolumeMigrate
	VolumeMove
	VolumeSnapshotCreate
	VolumeSnapshotDelete
	VolumeSnapshotUpdate
	ProjectRename
	ImagesExpire
	ImagesPruneLeftover
	ImagesUpdate
	ImagesSynchronize
	LogsExpire
	InstanceTypesUpdate
	BackupsExpire
	SnapshotsExpire
	CustomVolumeSnapshotsExpire
	CustomVolumeBackupCreate
	CustomVolumeBackupRemove
	CustomVolumeBackupRename
	CustomVolumeBackupRestore
	WarningsPruneResolved
	ClusterJoinToken
	VolumeSnapshotRename
	ClusterMemberEvacuate
	ClusterMemberRestore
	CertificateAddToken
	RemoveOrphanedOperations
)

// Description return a human-readable description of the operation type.
func (t Type) Description() string {
	switch t {
	case ClusterBootstrap:
		return "Creating bootstrap node"
	case ClusterJoin:
		return "Joining cluster"
	case BackupCreate:
		return "Backing up instance"
	case BackupRename:
		return "Renaming instance backup"
	case BackupRestore:
		return "Restoring backup"
	case BackupRemove:
		return "Removing instance backup"
	case ConsoleShow:
		return "Showing console"
	case InstanceCreate:
		return "Creating instance"
	case InstanceUpdate:
		return "Updating instance"
	case InstanceRename:
		return "Renaming instance"
	case InstanceMigrate:
		return "Migrating instance"
	case InstanceLiveMigrate:
		return "Live-migrating instance"
	case InstanceFreeze:
		return "Freezing instance"
	case InstanceUnfreeze:
		return "Unfreezing instance"
	case InstanceDelete:
		return "Deleting instance"
	case InstanceStart:
		return "Starting instance"
	case InstanceStop:
		return "Stopping instance"
	case InstanceRestart:
		return "Restarting instance"
	case CommandExec:
		return "Executing command"
	case SnapshotCreate:
		return "Snapshotting instance"
	case SnapshotRename:
		return "Renaming snapshot"
	case SnapshotRestore:
		return "Restoring snapshot"
	case SnapshotTransfer:
		return "Transferring snapshot"
	case SnapshotUpdate:
		return "Updating snapshot"
	case SnapshotDelete:
		return "Deleting snapshot"
	case ImageDownload:
		return "Downloading image"
	case ImageDelete:
		return "Deleting image"
	case ImageToken:
		return "Image download token"
	case ImageRefresh:
		return "Refreshing image"
	case VolumeCopy:
		return "Copying storage volume"
	case VolumeCreate:
		return "Creating storage volume"
	case VolumeMigrate:
		return "Migrating storage volume"
	case VolumeMove:
		return "Moving storage volume"
	case VolumeSnapshotCreate:
		return "Creating storage volume snapshot"
	case VolumeSnapshotDelete:
		return "Deleting storage volume snapshot"
	case VolumeSnapshotUpdate:
		return "Updating storage volume snapshot"
	case VolumeSnapshotRename:
		return "Renaming storage volume snapshot"
	case ProjectRename:
		return "Renaming project"
	case ImagesExpire:
		return "Cleaning up expired images"
	case ImagesPruneLeftover:
		return "Pruning leftover image files"
	case ImagesUpdate:
		return "Updating images"
	case ImagesSynchronize:
		return "Synchronizing images"
	case LogsExpire:
		return "Expiring log files"
	case InstanceTypesUpdate:
		return "Updating instance types"
	case BackupsExpire:
		return "Cleaning up expired instance backups"
	case SnapshotsExpire:
		return "Cleaning up expired instance snapshots"
	case CustomVolumeSnapshotsExpire:
		return "Cleaning up expired volume snapshots"
	case CustomVolumeBackupCreate:
		return "Creating custom volume backup"
	case CustomVolumeBackupRemove:
		return "Deleting custom volume backup"
	case CustomVolumeBackupRename:
		return "Renaming custom volume backup"
	case CustomVolumeBackupRestore:
		return "Restoring custom volume backup"
	case WarningsPruneResolved:
		return "Pruning resolved warnings"
	case ClusterMemberEvacuate:
		return "Evacuating cluster member"
	case ClusterMemberRestore:
		return "Restoring cluster member"
	case RemoveOrphanedOperations:
		return "Remove orphaned operations"
	default:
		return "Executing operation"
	}
}

// Permission returns the needed RBAC permission to cancel the oepration
func (t Type) Permission() string {
	switch t {
	case BackupCreate:
		return "operate-containers"
	case BackupRename:
		return "operate-containers"
	case BackupRestore:
		return "operate-containers"
	case BackupRemove:
		return "operate-containers"
	case ConsoleShow:
		return "operate-containers"
	case InstanceFreeze:
		return "operate-containers"
	case InstanceUnfreeze:
		return "operate-containers"
	case InstanceStart:
		return "operate-containers"
	case InstanceStop:
		return "operate-containers"
	case InstanceRestart:
		return "operate-containers"
	case CommandExec:
		return "operate-containers"
	case SnapshotCreate:
		return "operate-containers"
	case SnapshotRename:
		return "operate-containers"
	case SnapshotTransfer:
		return "operate-containers"
	case SnapshotUpdate:
		return "operate-containers"
	case SnapshotDelete:
		return "operate-containers"

	case InstanceCreate:
		return "manage-containers"
	case InstanceUpdate:
		return "manage-containers"
	case InstanceRename:
		return "manage-containers"
	case InstanceMigrate:
		return "manage-containers"
	case InstanceLiveMigrate:
		return "manage-containers"
	case InstanceDelete:
		return "manage-containers"
	case SnapshotRestore:
		return "manage-containers"

	case ImageDownload:
		return "manage-images"
	case ImageDelete:
		return "manage-images"
	case ImageToken:
		return "manage-images"
	case ImageRefresh:
		return "manage-images"
	case ImagesUpdate:
		return "manage-images"
	case ImagesSynchronize:
		return "manage-images"

	case CustomVolumeSnapshotsExpire:
		return "operate-volumes"
	case CustomVolumeBackupCreate:
		return "manage-storage-volumes"
	case CustomVolumeBackupRemove:
		return "manage-storage-volumes"
	case CustomVolumeBackupRename:
		return "manage-storage-volumes"
	case CustomVolumeBackupRestore:
		return "manage-storage-volumes"
	}

	return ""
}
<commit_msg>lxd/db/operationtype: Ends all comments with a full-stop.<commit_after>package operationtype

// Type is a numeric code indentifying the type of an Operation.
type Type int64

// Possible values for Type
//
// WARNING: The type codes are stored in the database, so this list of
//          definitions should be normally append-only. Any other change
//          requires a database update.
const (
	Unknown Type = iota
	ClusterBootstrap
	ClusterJoin
	BackupCreate
	BackupRename
	BackupRestore
	BackupRemove
	ConsoleShow
	InstanceCreate
	InstanceUpdate
	InstanceRename
	InstanceMigrate
	InstanceLiveMigrate
	InstanceFreeze
	InstanceUnfreeze
	InstanceDelete
	InstanceStart
	InstanceStop
	InstanceRestart
	CommandExec
	SnapshotCreate
	SnapshotRename
	SnapshotRestore
	SnapshotTransfer
	SnapshotUpdate
	SnapshotDelete
	ImageDownload
	ImageDelete
	ImageToken
	ImageRefresh
	VolumeCopy
	VolumeCreate
	VolumeMigrate
	VolumeMove
	VolumeSnapshotCreate
	VolumeSnapshotDelete
	VolumeSnapshotUpdate
	ProjectRename
	ImagesExpire
	ImagesPruneLeftover
	ImagesUpdate
	ImagesSynchronize
	LogsExpire
	InstanceTypesUpdate
	BackupsExpire
	SnapshotsExpire
	CustomVolumeSnapshotsExpire
	CustomVolumeBackupCreate
	CustomVolumeBackupRemove
	CustomVolumeBackupRename
	CustomVolumeBackupRestore
	WarningsPruneResolved
	ClusterJoinToken
	VolumeSnapshotRename
	ClusterMemberEvacuate
	ClusterMemberRestore
	CertificateAddToken
	RemoveOrphanedOperations
)

// Description return a human-readable description of the operation type.
func (t Type) Description() string {
	switch t {
	case ClusterBootstrap:
		return "Creating bootstrap node"
	case ClusterJoin:
		return "Joining cluster"
	case BackupCreate:
		return "Backing up instance"
	case BackupRename:
		return "Renaming instance backup"
	case BackupRestore:
		return "Restoring backup"
	case BackupRemove:
		return "Removing instance backup"
	case ConsoleShow:
		return "Showing console"
	case InstanceCreate:
		return "Creating instance"
	case InstanceUpdate:
		return "Updating instance"
	case InstanceRename:
		return "Renaming instance"
	case InstanceMigrate:
		return "Migrating instance"
	case InstanceLiveMigrate:
		return "Live-migrating instance"
	case InstanceFreeze:
		return "Freezing instance"
	case InstanceUnfreeze:
		return "Unfreezing instance"
	case InstanceDelete:
		return "Deleting instance"
	case InstanceStart:
		return "Starting instance"
	case InstanceStop:
		return "Stopping instance"
	case InstanceRestart:
		return "Restarting instance"
	case CommandExec:
		return "Executing command"
	case SnapshotCreate:
		return "Snapshotting instance"
	case SnapshotRename:
		return "Renaming snapshot"
	case SnapshotRestore:
		return "Restoring snapshot"
	case SnapshotTransfer:
		return "Transferring snapshot"
	case SnapshotUpdate:
		return "Updating snapshot"
	case SnapshotDelete:
		return "Deleting snapshot"
	case ImageDownload:
		return "Downloading image"
	case ImageDelete:
		return "Deleting image"
	case ImageToken:
		return "Image download token"
	case ImageRefresh:
		return "Refreshing image"
	case VolumeCopy:
		return "Copying storage volume"
	case VolumeCreate:
		return "Creating storage volume"
	case VolumeMigrate:
		return "Migrating storage volume"
	case VolumeMove:
		return "Moving storage volume"
	case VolumeSnapshotCreate:
		return "Creating storage volume snapshot"
	case VolumeSnapshotDelete:
		return "Deleting storage volume snapshot"
	case VolumeSnapshotUpdate:
		return "Updating storage volume snapshot"
	case VolumeSnapshotRename:
		return "Renaming storage volume snapshot"
	case ProjectRename:
		return "Renaming project"
	case ImagesExpire:
		return "Cleaning up expired images"
	case ImagesPruneLeftover:
		return "Pruning leftover image files"
	case ImagesUpdate:
		return "Updating images"
	case ImagesSynchronize:
		return "Synchronizing images"
	case LogsExpire:
		return "Expiring log files"
	case InstanceTypesUpdate:
		return "Updating instance types"
	case BackupsExpire:
		return "Cleaning up expired instance backups"
	case SnapshotsExpire:
		return "Cleaning up expired instance snapshots"
	case CustomVolumeSnapshotsExpire:
		return "Cleaning up expired volume snapshots"
	case CustomVolumeBackupCreate:
		return "Creating custom volume backup"
	case CustomVolumeBackupRemove:
		return "Deleting custom volume backup"
	case CustomVolumeBackupRename:
		return "Renaming custom volume backup"
	case CustomVolumeBackupRestore:
		return "Restoring custom volume backup"
	case WarningsPruneResolved:
		return "Pruning resolved warnings"
	case ClusterMemberEvacuate:
		return "Evacuating cluster member"
	case ClusterMemberRestore:
		return "Restoring cluster member"
	case RemoveOrphanedOperations:
		return "Remove orphaned operations"
	default:
		return "Executing operation"
	}
}

// Permission returns the needed RBAC permission to cancel the operation.
func (t Type) Permission() string {
	switch t {
	case BackupCreate:
		return "operate-containers"
	case BackupRename:
		return "operate-containers"
	case BackupRestore:
		return "operate-containers"
	case BackupRemove:
		return "operate-containers"
	case ConsoleShow:
		return "operate-containers"
	case InstanceFreeze:
		return "operate-containers"
	case InstanceUnfreeze:
		return "operate-containers"
	case InstanceStart:
		return "operate-containers"
	case InstanceStop:
		return "operate-containers"
	case InstanceRestart:
		return "operate-containers"
	case CommandExec:
		return "operate-containers"
	case SnapshotCreate:
		return "operate-containers"
	case SnapshotRename:
		return "operate-containers"
	case SnapshotTransfer:
		return "operate-containers"
	case SnapshotUpdate:
		return "operate-containers"
	case SnapshotDelete:
		return "operate-containers"

	case InstanceCreate:
		return "manage-containers"
	case InstanceUpdate:
		return "manage-containers"
	case InstanceRename:
		return "manage-containers"
	case InstanceMigrate:
		return "manage-containers"
	case InstanceLiveMigrate:
		return "manage-containers"
	case InstanceDelete:
		return "manage-containers"
	case SnapshotRestore:
		return "manage-containers"

	case ImageDownload:
		return "manage-images"
	case ImageDelete:
		return "manage-images"
	case ImageToken:
		return "manage-images"
	case ImageRefresh:
		return "manage-images"
	case ImagesUpdate:
		return "manage-images"
	case ImagesSynchronize:
		return "manage-images"

	case CustomVolumeSnapshotsExpire:
		return "operate-volumes"
	case CustomVolumeBackupCreate:
		return "manage-storage-volumes"
	case CustomVolumeBackupRemove:
		return "manage-storage-volumes"
	case CustomVolumeBackupRename:
		return "manage-storage-volumes"
	case CustomVolumeBackupRestore:
		return "manage-storage-volumes"
	}

	return ""
}
<|endoftext|>


<commit_before>package ssh

import (
	"bitbucket.org/taruti/bigendian"
	"bufio"
	"bytes"
	"crypto/cipher"
	"hash"
	"io"
	"net"
	"os"
)

type ssh struct {
	c net.Conn
	r *bufio.Reader
	rc cipher.BlockMode
	wc cipher.BlockMode
	rh hash.Hash
	wh hash.Hash
	rseq, wseq uint32
	rident string
	ckex, skex []byte
	session_id []byte
}

func connect(host string) (*ssh,os.Error) {
	c,err := net.Dial("tcp", host+":22")
	if err!=nil {
		return nil,err
	}

	_,err = c.Write([]byte(ident+"\r\n"))
	if err!=nil {
		return nil,err
	}

	r := bufio.NewReader(c)

	var line []byte
	var prefix bool
	for {
		line, prefix, err = r.ReadLine()
		if err!=nil {
			return nil,err
		}
		if prefix {
			return nil,os.NewError("Too long identification line")
		}
		Log(1,"%s",line)
		if bytes.HasPrefix(line, []byte("SSH-2.0-")) {
			break
		}
		if bytes.HasPrefix(line, []byte("SSH-")) {
			return nil,os.NewError("Unsupported ssh version")
		}
	}

	return &ssh{c, r, NullCrypto{}, NullCrypto{}, NullHash{}, NullHash{}, 0, 0, string(line), nil, nil, nil}, nil
}

func readPacket(c *ssh) ([]byte, os.Error) {
	l := c.rc.BlockSize()
	if l < 16 {
		l = 16
	}

	// read packet header
	b := make([]byte, l)
	_, err := io.ReadFull(c.r, b)
	if err!=nil {
		return nil,err
	}

	Log(9,"WIRE = %X",b)
	// decrypt header
	c.rc.CryptBlocks(b,b)
	Log(9,"DEC  = %X",b)

	lfield   := int(bigendian.U32(b))
	lpadding := int(b[4])
	// FIXME add checks here to validate packet
	lhash    := c.rh.Size()
	lrest    := 4 + lfield + lhash - len(b)

	if lrest > 128*1024 || lfield<0 {
		return nil, os.NewError("too large packet")
	}

	packet := make([]byte, len(b)+lrest)
	copy(packet, b)
	rest := packet[len(b):]
	_,err = io.ReadFull(c.r,rest)
	if err!=nil {
		return nil,err
	}
	c.rc.CryptBlocks(rest[0:lrest-lhash], rest[0:lrest-lhash])

	var rseqb [4]byte
	bigendian.PutU32(rseqb[:], c.rseq)
	c.rseq++

	c.rh.Reset()
	c.rh.Write(rseqb[:])
	c.rh.Write(packet[0:len(packet)-lhash])
	if !constEq(c.rh.Sum(), packet[len(packet)-lhash:]) {
		return nil,os.NewError("Invalid mac")
	}

	return packet[5:4+lfield-lpadding],nil
}

func writePacket(c *ssh, fun func(*bigendian.Printer)) {
	p := bigendian.NewPrinterWith(make([]byte, 5, 32))
	fun(p)
	b := p.Out()

	bs := c.wc.BlockSize()
	if bs < 16 {
		bs = 16
	}

	padding := bs - (len(b) % bs)
	if padding < 4 {
		padding += bs
	}

	b[4] = byte(padding)
	b = append(b, rand(padding)...)
	bigendian.PutU32(b, uint32(len(b)-4))

	var wseqb [4]byte
	bigendian.PutU32(wseqb[:], c.wseq)
	c.wseq++
	c.wh.Reset()
	c.wh.Write(wseqb[:])
	c.wh.Write(b)

	c.wc.CryptBlocks(b,b)

	b = append(b, c.wh.Sum()...)

	c.c.Write(b)
}

func writeKexInit(c *ssh) {
	p := bigendian.NewPrinter()
	p.Byte(msgKexInit).Bytes(rand(16))
	p.U32String(kexKex).U32String(kexShk)
	p.U32String(kexEnc).U32String(kexEnc)
	p.U32String(kexMac).U32String(kexMac)
	p.U32String(kexCom).U32String(kexCom)
	p.U32String(kexLan).U32String(kexLan)
	p.Byte(0).U32(0)
	c.ckex = p.Out()

	writePacket(c, func(p *bigendian.Printer) { p.Bytes(c.ckex) })
}

type kexres struct {
	Kex, Shk, EncCS, EncSC, MacCS, MacSC, ComCS, ComSC string
}

func parseKexInit(c *ssh, b []byte) (*kexres,os.Error) {
	var cookie []byte
	var kex, shk, ecs, esc, mcs, msc, ccs, csc, lcs, lsc string
	var follows byte
	var reserved uint32

	p := bigendian.NewParser(b).NBytes(16, &cookie)
	p.U32String(&kex).U32String(&shk)
	p.U32String(&ecs).U32String(&esc)
	p.U32String(&mcs).U32String(&msc)
	p.U32String(&ccs).U32String(&csc)
	p.U32String(&lcs).U32String(&lsc)
	p.Byte(&follows).U32(&reserved).End()

	guessed := true
	var r kexres
	var g bool
	var err os.Error

	r.Kex,g,err = namelistCheck(kexKex, kex)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed

	r.Shk,g,err = namelistCheck(kexShk, shk)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed

	r.EncCS,g,err = namelistCheck(kexEnc, ecs)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed
	r.EncSC,g,err = namelistCheck(kexEnc, esc)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed

	r.MacCS,g,err = namelistCheck(kexMac, mcs)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed
	r.MacSC,g,err = namelistCheck(kexMac, msc)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed

	r.ComCS,g,err = namelistCheck(kexCom, ccs)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed
	r.ComSC,g,err = namelistCheck(kexCom, csc)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed

	if guessed==false && follows>0 {
		readPacket(c)
	}

	return &r, nil
}

<commit_msg>Enlarge maximum accepted packet size<commit_after>package ssh

import (
	"bitbucket.org/taruti/bigendian"
	"bufio"
	"bytes"
	"crypto/cipher"
	"hash"
	"io"
	"net"
	"os"
)

type ssh struct {
	c net.Conn
	r *bufio.Reader
	rc cipher.BlockMode
	wc cipher.BlockMode
	rh hash.Hash
	wh hash.Hash
	rseq, wseq uint32
	rident string
	ckex, skex []byte
	session_id []byte
}

func connect(host string) (*ssh,os.Error) {
	c,err := net.Dial("tcp", host+":22")
	if err!=nil {
		return nil,err
	}

	_,err = c.Write([]byte(ident+"\r\n"))
	if err!=nil {
		return nil,err
	}

	r := bufio.NewReader(c)

	var line []byte
	var prefix bool
	for {
		line, prefix, err = r.ReadLine()
		if err!=nil {
			return nil,err
		}
		if prefix {
			return nil,os.NewError("Too long identification line")
		}
		Log(1,"%s",line)
		if bytes.HasPrefix(line, []byte("SSH-2.0-")) {
			break
		}
		if bytes.HasPrefix(line, []byte("SSH-")) {
			return nil,os.NewError("Unsupported ssh version")
		}
	}

	return &ssh{c, r, NullCrypto{}, NullCrypto{}, NullHash{}, NullHash{}, 0, 0, string(line), nil, nil, nil}, nil
}

func readPacket(c *ssh) ([]byte, os.Error) {
	l := c.rc.BlockSize()
	if l < 16 {
		l = 16
	}

	// read packet header
	b := make([]byte, l)
	_, err := io.ReadFull(c.r, b)
	if err!=nil {
		return nil,err
	}

	Log(9,"WIRE = %X",b)
	// decrypt header
	c.rc.CryptBlocks(b,b)
	Log(9,"DEC  = %X",b)

	lfield   := int(bigendian.U32(b))
	lpadding := int(b[4])
	// FIXME add checks here to validate packet
	lhash    := c.rh.Size()
	lrest    := 4 + lfield + lhash - len(b)

	if lrest > 12*1024*1024 || lfield<0 {
		return nil, os.NewError("too large packet")
	}

	packet := make([]byte, len(b)+lrest)
	copy(packet, b)
	rest := packet[len(b):]
	_,err = io.ReadFull(c.r,rest)
	if err!=nil {
		return nil,err
	}
	c.rc.CryptBlocks(rest[0:lrest-lhash], rest[0:lrest-lhash])

	var rseqb [4]byte
	bigendian.PutU32(rseqb[:], c.rseq)
	c.rseq++

	c.rh.Reset()
	c.rh.Write(rseqb[:])
	c.rh.Write(packet[0:len(packet)-lhash])
	if !constEq(c.rh.Sum(), packet[len(packet)-lhash:]) {
		return nil,os.NewError("Invalid mac")
	}

	return packet[5:4+lfield-lpadding],nil
}

func writePacket(c *ssh, fun func(*bigendian.Printer)) {
	p := bigendian.NewPrinterWith(make([]byte, 5, 32))
	fun(p)
	b := p.Out()

	bs := c.wc.BlockSize()
	if bs < 16 {
		bs = 16
	}

	padding := bs - (len(b) % bs)
	if padding < 4 {
		padding += bs
	}

	b[4] = byte(padding)
	b = append(b, rand(padding)...)
	bigendian.PutU32(b, uint32(len(b)-4))

	var wseqb [4]byte
	bigendian.PutU32(wseqb[:], c.wseq)
	c.wseq++
	c.wh.Reset()
	c.wh.Write(wseqb[:])
	c.wh.Write(b)

	c.wc.CryptBlocks(b,b)

	b = append(b, c.wh.Sum()...)

	c.c.Write(b)
}

func writeKexInit(c *ssh) {
	p := bigendian.NewPrinter()
	p.Byte(msgKexInit).Bytes(rand(16))
	p.U32String(kexKex).U32String(kexShk)
	p.U32String(kexEnc).U32String(kexEnc)
	p.U32String(kexMac).U32String(kexMac)
	p.U32String(kexCom).U32String(kexCom)
	p.U32String(kexLan).U32String(kexLan)
	p.Byte(0).U32(0)
	c.ckex = p.Out()

	writePacket(c, func(p *bigendian.Printer) { p.Bytes(c.ckex) })
}

type kexres struct {
	Kex, Shk, EncCS, EncSC, MacCS, MacSC, ComCS, ComSC string
}

func parseKexInit(c *ssh, b []byte) (*kexres,os.Error) {
	var cookie []byte
	var kex, shk, ecs, esc, mcs, msc, ccs, csc, lcs, lsc string
	var follows byte
	var reserved uint32

	p := bigendian.NewParser(b).NBytes(16, &cookie)
	p.U32String(&kex).U32String(&shk)
	p.U32String(&ecs).U32String(&esc)
	p.U32String(&mcs).U32String(&msc)
	p.U32String(&ccs).U32String(&csc)
	p.U32String(&lcs).U32String(&lsc)
	p.Byte(&follows).U32(&reserved).End()

	guessed := true
	var r kexres
	var g bool
	var err os.Error

	r.Kex,g,err = namelistCheck(kexKex, kex)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed

	r.Shk,g,err = namelistCheck(kexShk, shk)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed

	r.EncCS,g,err = namelistCheck(kexEnc, ecs)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed
	r.EncSC,g,err = namelistCheck(kexEnc, esc)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed

	r.MacCS,g,err = namelistCheck(kexMac, mcs)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed
	r.MacSC,g,err = namelistCheck(kexMac, msc)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed

	r.ComCS,g,err = namelistCheck(kexCom, ccs)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed
	r.ComSC,g,err = namelistCheck(kexCom, csc)
	if err!=nil {
		return nil,err
	}
	guessed = g && guessed

	if guessed==false && follows>0 {
		readPacket(c)
	}

	return &r, nil
}

<|endoftext|>


<commit_before>package MQTTg

import (
	"net"
)

type Transport struct {
	conn net.UDPConn
	//sm   *Sender
}

func (self *Transport) SendMessage(m Message) error {
	wire, err := m.GetWire()
	if err != nil {
		return err
	}
	// TODO: Send the wire
	return nil
}

func (self *Transport) ReadMessageFrom() (Message, *net.UDPAddr, error) {
	wire := make([]byte, 65535) //TODO: should be optimized
	len, addr, err := self.conn.ReadFromUDP(wire)
	if err != nil {
		return nil, addr, err
	}
	m, err := ReadFrame(wire[:len])
	if err != nil {
		return nil, addr, err
	}

	return m, addr, nil
}
func (self *Transport) SendPublish(dup bool, qos uint8, retain bool, topic string, data string) error {
	// TODO: id should be considered
	pub := NewPublishMessage(dup, qos, retain, topic, 0, []uint8(data))
	err := self.SendMessage(pub)
	return err
}
func (self *Transport) SendPuback(packetID uint16) error {
	return nil
}

func (self *Transport) SendPubrec(packetID uint16) error {
	return nil
}

func (self *Transport) SendPubrel(packetID uint16) error {
	return nil
}

func (self *Transport) SendPubcomp(packetID uint16) error {
	return nil
}
<commit_msg>remove prefix 'Send'<commit_after>package MQTTg

import (
	"net"
)

type Transport struct {
	conn net.UDPConn
	//sm   *Sender
}

func (self *Transport) SendMessage(m Message) error {
	wire, err := m.GetWire()
	if err != nil {
		return err
	}
	// TODO: Send the wire
	return nil
}

func (self *Transport) ReadMessageFrom() (Message, *net.UDPAddr, error) {
	wire := make([]byte, 65535) //TODO: should be optimized
	len, addr, err := self.conn.ReadFromUDP(wire)
	if err != nil {
		return nil, addr, err
	}
	m, err := ReadFrame(wire[:len])
	if err != nil {
		return nil, addr, err
	}

	return m, addr, nil
}
func (self *Transport) Publish(dup bool, qos uint8, retain bool, topic string, data string) error {
	// TODO: id should be considered
	pub := NewPublishMessage(dup, qos, retain, topic, 0, []uint8(data))
	err := self.SendMessage(pub)
	return err
}
func (self *Transport) Puback(packetID uint16) error {
	return nil
}

func (self *Transport) Pubrec(packetID uint16) error {
	return nil
}

func (self *Transport) Pubrel(packetID uint16) error {
	return nil
}

func (self *Transport) Pubcomp(packetID uint16) error {
	return nil
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
)

// listParams - list object params used for list object map
type listParams struct {
	bucket    string
	recursive bool
	marker    string
	prefix    string
}

// Tree walk result carries results of tree walking.
type treeWalkResult struct {
	fileInfo FileInfo
	err      error
	end      bool
}

// Tree walk notify carries a channel which notifies tree walk
// results, additionally it also carries information if treeWalk
// should be timedOut.
type treeWalker struct {
	ch       <-chan treeWalkResult
	timedOut bool
}

// treeWalk walks FS directory tree recursively pushing fileInfo into the channel as and when it encounters files.
func treeWalk(layer ObjectLayer, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, send func(treeWalkResult) bool, count *int) bool {
	// Example:
	// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
	// called with prefixDir="one/two/three/four/" and marker="five.txt"

	var isXL bool
	var disk StorageAPI
	switch l := layer.(type) {
	case xlObjects:
		isXL = true
		disk = l.storage
	case fsObjects:
		disk = l.storage
	}

	// Convert entry to FileInfo
	entryToFileInfo := func(entry string) (fileInfo FileInfo, err error) {
		if strings.HasSuffix(entry, slashSeparator) {
			// Object name needs to be full path.
			fileInfo.Name = path.Join(prefixDir, entry)
			fileInfo.Name += slashSeparator
			fileInfo.Mode = os.ModeDir
			return
		}
		if isXL && strings.HasSuffix(entry, multipartSuffix) {
			// If the entry was detected as a multipart file we use
			// getMultipartObjectInfo() to fill the FileInfo structure.
			entry = strings.Trim(entry, multipartSuffix)
			var info MultipartObjectInfo
			info, err = getMultipartObjectInfo(disk, bucket, path.Join(prefixDir, entry))
			if err != nil {
				return
			}
			// Set the Mode to a "regular" file.
			fileInfo.Mode = 0
			// Trim the suffix that was temporarily added to indicate that this
			// is a multipart file.
			fileInfo.Name = path.Join(prefixDir, entry)
			fileInfo.Size = info.Size
			fileInfo.MD5Sum = info.MD5Sum
			fileInfo.ModTime = info.ModTime
			return
		}
		if fileInfo, err = disk.StatFile(bucket, path.Join(prefixDir, entry)); err != nil {
			return
		}
		// Object name needs to be full path.
		fileInfo.Name = path.Join(prefixDir, entry)
		return
	}

	var markerBase, markerDir string
	if marker != "" {
		// Ex: if marker="four/five.txt", markerDir="four/" markerBase="five.txt"
		markerSplit := strings.SplitN(marker, slashSeparator, 2)
		markerDir = markerSplit[0]
		if len(markerSplit) == 2 {
			markerDir += slashSeparator
			markerBase = markerSplit[1]
		}
	}
	entries, err := disk.ListDir(bucket, prefixDir)
	if err != nil {
		send(treeWalkResult{err: err})
		return false
	}

	if entryPrefixMatch != "" {
		for i, entry := range entries {
			if !strings.HasPrefix(entry, entryPrefixMatch) {
				entries[i] = ""
			}
			if hasReservedPrefix(entry) || hasReservedSuffix(entry) {
				entries[i] = ""
			}
		}
	}
	// For XL multipart files strip the trailing "/" and append ".minio.multipart" to the entry so that
	// entryToFileInfo() can call StatFile for regular files or getMultipartObjectInfo() for multipart files.
	for i, entry := range entries {
		if isXL && strings.HasSuffix(entry, slashSeparator) && isLeafDirectory(disk, bucket, path.Join(prefixDir, entry)) {
			entries[i] = strings.TrimSuffix(entry, slashSeparator) + multipartSuffix
		}
	}
	sort.Sort(byMultipartFiles(entries))
	// Skip the empty strings
	for len(entries) > 0 && entries[0] == "" {
		entries = entries[1:]
	}
	if len(entries) == 0 {
		return true
	}
	// example:
	// If markerDir="four/" Search() returns the index of "four/" in the sorted
	// entries list so we skip all the entries till "four/"
	idx := sort.Search(len(entries), func(i int) bool { return strings.TrimSuffix(entries[i], multipartSuffix) >= markerDir })
	entries = entries[idx:]
	*count += len(entries)
	for i, entry := range entries {
		if i == 0 && markerDir == entry {
			if !recursive {
				// Skip as the marker would already be listed in the previous listing.
				*count--
				continue
			}
			if recursive && !strings.HasSuffix(entry, slashSeparator) {
				// We should not skip for recursive listing and if markerDir is a directory
				// for ex. if marker is "four/five.txt" markerDir will be "four/" which
				// should not be skipped, instead it will need to be treeWalk()'ed into.

				// Skip if it is a file though as it would be listed in previous listing.
				*count--
				continue
			}
		}

		if recursive && strings.HasSuffix(entry, slashSeparator) {
			// If the entry is a directory, we will need recurse into it.
			markerArg := ""
			if entry == markerDir {
				// We need to pass "five.txt" as marker only if we are
				// recursing into "four/"
				markerArg = markerBase
			}
			*count--
			prefixMatch := "" // Valid only for first level treeWalk and empty for subdirectories.
			if !treeWalk(layer, bucket, path.Join(prefixDir, entry), prefixMatch, markerArg, recursive, send, count) {
				return false
			}
			continue
		}
		*count--
		fileInfo, err := entryToFileInfo(entry)
		if err != nil {
			// The file got deleted in the interim between ListDir() and StatFile()
			// Ignore error and continue.
			continue
		}
		if !send(treeWalkResult{fileInfo: fileInfo}) {
			return false
		}
	}
	return true
}

// Initiate a new treeWalk in a goroutine.
func startTreeWalk(layer ObjectLayer, bucket, prefix, marker string, recursive bool) *treeWalker {
	// Example 1
	// If prefix is "one/two/three/" and marker is "one/two/three/four/five.txt"
	// treeWalk is called with prefixDir="one/two/three/" and marker="four/five.txt"
	// and entryPrefixMatch=""

	// Example 2
	// if prefix is "one/two/th" and marker is "one/two/three/four/five.txt"
	// treeWalk is called with prefixDir="one/two/" and marker="three/four/five.txt"
	// and entryPrefixMatch="th"

	ch := make(chan treeWalkResult, maxObjectList)
	walkNotify := treeWalker{ch: ch}
	entryPrefixMatch := prefix
	prefixDir := ""
	lastIndex := strings.LastIndex(prefix, slashSeparator)
	if lastIndex != -1 {
		entryPrefixMatch = prefix[lastIndex+1:]
		prefixDir = prefix[:lastIndex+1]
	}
	count := 0
	marker = strings.TrimPrefix(marker, prefixDir)
	go func() {
		defer close(ch)
		send := func(walkResult treeWalkResult) bool {
			if count == 0 {
				walkResult.end = true
			}
			timer := time.After(time.Second * 60)
			select {
			case ch <- walkResult:
				return true
			case <-timer:
				walkNotify.timedOut = true
				return false
			}
		}
		treeWalk(layer, bucket, prefixDir, entryPrefixMatch, marker, recursive, send, &count)
	}()
	return &walkNotify
}

// Save the goroutine reference in the map
func saveTreeWalk(layer ObjectLayer, params listParams, walker *treeWalker) {
	var listObjectMap map[listParams][]*treeWalker
	var listObjectMapMutex *sync.Mutex
	switch l := layer.(type) {
	case xlObjects:
		listObjectMap = l.listObjectMap
		listObjectMapMutex = l.listObjectMapMutex
	case fsObjects:
		listObjectMap = l.listObjectMap
		listObjectMapMutex = l.listObjectMapMutex
	}
	listObjectMapMutex.Lock()
	defer listObjectMapMutex.Unlock()

	log.WithFields(logrus.Fields{
		"bucket":    params.bucket,
		"recursive": params.recursive,
		"marker":    params.marker,
		"prefix":    params.prefix,
	}).Debugf("saveTreeWalk has been invoked.")

	walkers, _ := listObjectMap[params]
	walkers = append(walkers, walker)

	listObjectMap[params] = walkers
	log.Debugf("Successfully saved in listObjectMap.")
}

// Lookup the goroutine reference from map
func lookupTreeWalk(layer ObjectLayer, params listParams) *treeWalker {
	var listObjectMap map[listParams][]*treeWalker
	var listObjectMapMutex *sync.Mutex
	switch l := layer.(type) {
	case xlObjects:
		listObjectMap = l.listObjectMap
		listObjectMapMutex = l.listObjectMapMutex
	case fsObjects:
		listObjectMap = l.listObjectMap
		listObjectMapMutex = l.listObjectMapMutex
	}
	listObjectMapMutex.Lock()
	defer listObjectMapMutex.Unlock()

	log.WithFields(logrus.Fields{
		"bucket":    params.bucket,
		"recursive": params.recursive,
		"marker":    params.marker,
		"prefix":    params.prefix,
	}).Debugf("lookupTreeWalk has been invoked.")
	if walkChs, ok := listObjectMap[params]; ok {
		for i, walkCh := range walkChs {
			if !walkCh.timedOut {
				newWalkChs := walkChs[i+1:]
				if len(newWalkChs) > 0 {
					listObjectMap[params] = newWalkChs
				} else {
					delete(listObjectMap, params)
				}
				log.WithFields(logrus.Fields{
					"bucket":    params.bucket,
					"recursive": params.recursive,
					"marker":    params.marker,
					"prefix":    params.prefix,
				}).Debugf("Found the previous saved listsObjects params.")
				return walkCh
			}
		}
		// As all channels are timed out, delete the map entry
		delete(listObjectMap, params)
	}
	return nil
}
<commit_msg>XL/ListObjects: use string.TrimSuffix instead of Trim. (#1498) (#1509)<commit_after>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
)

// listParams - list object params used for list object map
type listParams struct {
	bucket    string
	recursive bool
	marker    string
	prefix    string
}

// Tree walk result carries results of tree walking.
type treeWalkResult struct {
	fileInfo FileInfo
	err      error
	end      bool
}

// Tree walk notify carries a channel which notifies tree walk
// results, additionally it also carries information if treeWalk
// should be timedOut.
type treeWalker struct {
	ch       <-chan treeWalkResult
	timedOut bool
}

// treeWalk walks FS directory tree recursively pushing fileInfo into the channel as and when it encounters files.
func treeWalk(layer ObjectLayer, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, send func(treeWalkResult) bool, count *int) bool {
	// Example:
	// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
	// called with prefixDir="one/two/three/four/" and marker="five.txt"

	var isXL bool
	var disk StorageAPI
	switch l := layer.(type) {
	case xlObjects:
		isXL = true
		disk = l.storage
	case fsObjects:
		disk = l.storage
	}

	// Convert entry to FileInfo
	entryToFileInfo := func(entry string) (fileInfo FileInfo, err error) {
		if strings.HasSuffix(entry, slashSeparator) {
			// Object name needs to be full path.
			fileInfo.Name = path.Join(prefixDir, entry)
			fileInfo.Name += slashSeparator
			fileInfo.Mode = os.ModeDir
			return
		}
		if isXL && strings.HasSuffix(entry, multipartSuffix) {
			// If the entry was detected as a multipart file we use
			// getMultipartObjectInfo() to fill the FileInfo structure.
			entry = strings.TrimSuffix(entry, multipartSuffix)
			var info MultipartObjectInfo
			info, err = getMultipartObjectInfo(disk, bucket, path.Join(prefixDir, entry))
			if err != nil {
				return
			}
			// Set the Mode to a "regular" file.
			fileInfo.Mode = 0
			// Trim the suffix that was temporarily added to indicate that this
			// is a multipart file.
			fileInfo.Name = path.Join(prefixDir, entry)
			fileInfo.Size = info.Size
			fileInfo.MD5Sum = info.MD5Sum
			fileInfo.ModTime = info.ModTime
			return
		}
		if fileInfo, err = disk.StatFile(bucket, path.Join(prefixDir, entry)); err != nil {
			return
		}
		// Object name needs to be full path.
		fileInfo.Name = path.Join(prefixDir, entry)
		return
	}

	var markerBase, markerDir string
	if marker != "" {
		// Ex: if marker="four/five.txt", markerDir="four/" markerBase="five.txt"
		markerSplit := strings.SplitN(marker, slashSeparator, 2)
		markerDir = markerSplit[0]
		if len(markerSplit) == 2 {
			markerDir += slashSeparator
			markerBase = markerSplit[1]
		}
	}
	entries, err := disk.ListDir(bucket, prefixDir)
	if err != nil {
		send(treeWalkResult{err: err})
		return false
	}

	if entryPrefixMatch != "" {
		for i, entry := range entries {
			if !strings.HasPrefix(entry, entryPrefixMatch) {
				entries[i] = ""
			}
			if hasReservedPrefix(entry) || hasReservedSuffix(entry) {
				entries[i] = ""
			}
		}
	}
	// For XL multipart files strip the trailing "/" and append ".minio.multipart" to the entry so that
	// entryToFileInfo() can call StatFile for regular files or getMultipartObjectInfo() for multipart files.
	for i, entry := range entries {
		if isXL && strings.HasSuffix(entry, slashSeparator) && isLeafDirectory(disk, bucket, path.Join(prefixDir, entry)) {
			entries[i] = strings.TrimSuffix(entry, slashSeparator) + multipartSuffix
		}
	}
	sort.Sort(byMultipartFiles(entries))
	// Skip the empty strings
	for len(entries) > 0 && entries[0] == "" {
		entries = entries[1:]
	}
	if len(entries) == 0 {
		return true
	}
	// example:
	// If markerDir="four/" Search() returns the index of "four/" in the sorted
	// entries list so we skip all the entries till "four/"
	idx := sort.Search(len(entries), func(i int) bool { return strings.TrimSuffix(entries[i], multipartSuffix) >= markerDir })
	entries = entries[idx:]
	*count += len(entries)
	for i, entry := range entries {
		if i == 0 && markerDir == entry {
			if !recursive {
				// Skip as the marker would already be listed in the previous listing.
				*count--
				continue
			}
			if recursive && !strings.HasSuffix(entry, slashSeparator) {
				// We should not skip for recursive listing and if markerDir is a directory
				// for ex. if marker is "four/five.txt" markerDir will be "four/" which
				// should not be skipped, instead it will need to be treeWalk()'ed into.

				// Skip if it is a file though as it would be listed in previous listing.
				*count--
				continue
			}
		}

		if recursive && strings.HasSuffix(entry, slashSeparator) {
			// If the entry is a directory, we will need recurse into it.
			markerArg := ""
			if entry == markerDir {
				// We need to pass "five.txt" as marker only if we are
				// recursing into "four/"
				markerArg = markerBase
			}
			*count--
			prefixMatch := "" // Valid only for first level treeWalk and empty for subdirectories.
			if !treeWalk(layer, bucket, path.Join(prefixDir, entry), prefixMatch, markerArg, recursive, send, count) {
				return false
			}
			continue
		}
		*count--
		fileInfo, err := entryToFileInfo(entry)
		if err != nil {
			// The file got deleted in the interim between ListDir() and StatFile()
			// Ignore error and continue.
			continue
		}
		if !send(treeWalkResult{fileInfo: fileInfo}) {
			return false
		}
	}
	return true
}

// Initiate a new treeWalk in a goroutine.
func startTreeWalk(layer ObjectLayer, bucket, prefix, marker string, recursive bool) *treeWalker {
	// Example 1
	// If prefix is "one/two/three/" and marker is "one/two/three/four/five.txt"
	// treeWalk is called with prefixDir="one/two/three/" and marker="four/five.txt"
	// and entryPrefixMatch=""

	// Example 2
	// if prefix is "one/two/th" and marker is "one/two/three/four/five.txt"
	// treeWalk is called with prefixDir="one/two/" and marker="three/four/five.txt"
	// and entryPrefixMatch="th"

	ch := make(chan treeWalkResult, maxObjectList)
	walkNotify := treeWalker{ch: ch}
	entryPrefixMatch := prefix
	prefixDir := ""
	lastIndex := strings.LastIndex(prefix, slashSeparator)
	if lastIndex != -1 {
		entryPrefixMatch = prefix[lastIndex+1:]
		prefixDir = prefix[:lastIndex+1]
	}
	count := 0
	marker = strings.TrimPrefix(marker, prefixDir)
	go func() {
		defer close(ch)
		send := func(walkResult treeWalkResult) bool {
			if count == 0 {
				walkResult.end = true
			}
			timer := time.After(time.Second * 60)
			select {
			case ch <- walkResult:
				return true
			case <-timer:
				walkNotify.timedOut = true
				return false
			}
		}
		treeWalk(layer, bucket, prefixDir, entryPrefixMatch, marker, recursive, send, &count)
	}()
	return &walkNotify
}

// Save the goroutine reference in the map
func saveTreeWalk(layer ObjectLayer, params listParams, walker *treeWalker) {
	var listObjectMap map[listParams][]*treeWalker
	var listObjectMapMutex *sync.Mutex
	switch l := layer.(type) {
	case xlObjects:
		listObjectMap = l.listObjectMap
		listObjectMapMutex = l.listObjectMapMutex
	case fsObjects:
		listObjectMap = l.listObjectMap
		listObjectMapMutex = l.listObjectMapMutex
	}
	listObjectMapMutex.Lock()
	defer listObjectMapMutex.Unlock()

	log.WithFields(logrus.Fields{
		"bucket":    params.bucket,
		"recursive": params.recursive,
		"marker":    params.marker,
		"prefix":    params.prefix,
	}).Debugf("saveTreeWalk has been invoked.")

	walkers, _ := listObjectMap[params]
	walkers = append(walkers, walker)

	listObjectMap[params] = walkers
	log.Debugf("Successfully saved in listObjectMap.")
}

// Lookup the goroutine reference from map
func lookupTreeWalk(layer ObjectLayer, params listParams) *treeWalker {
	var listObjectMap map[listParams][]*treeWalker
	var listObjectMapMutex *sync.Mutex
	switch l := layer.(type) {
	case xlObjects:
		listObjectMap = l.listObjectMap
		listObjectMapMutex = l.listObjectMapMutex
	case fsObjects:
		listObjectMap = l.listObjectMap
		listObjectMapMutex = l.listObjectMapMutex
	}
	listObjectMapMutex.Lock()
	defer listObjectMapMutex.Unlock()

	log.WithFields(logrus.Fields{
		"bucket":    params.bucket,
		"recursive": params.recursive,
		"marker":    params.marker,
		"prefix":    params.prefix,
	}).Debugf("lookupTreeWalk has been invoked.")
	if walkChs, ok := listObjectMap[params]; ok {
		for i, walkCh := range walkChs {
			if !walkCh.timedOut {
				newWalkChs := walkChs[i+1:]
				if len(newWalkChs) > 0 {
					listObjectMap[params] = newWalkChs
				} else {
					delete(listObjectMap, params)
				}
				log.WithFields(logrus.Fields{
					"bucket":    params.bucket,
					"recursive": params.recursive,
					"marker":    params.marker,
					"prefix":    params.prefix,
				}).Debugf("Found the previous saved listsObjects params.")
				return walkCh
			}
		}
		// As all channels are timed out, delete the map entry
		delete(listObjectMap, params)
	}
	return nil
}
<|endoftext|>