<commit_before>// Package parser implements a parser and parse tree dumper for Dockerfiles.
package parser

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"github.com/docker/docker/builder/dockerfile/command"
)

// Node is a structure used to represent a parse tree.
//
// In the node there are three fields, Value, Next, and Children. Value is the
// current token's string value. Next is always the next non-child token, and
// children contains all the children. Here's an example:
//
// (value next (child child-next child-next-next) next-next)
//
// This data structure is frankly pretty lousy for handling complex languages,
// but lucky for us the Dockerfile isn't very complicated. This structure
// works a little more effectively than a "proper" parse tree for our needs.
//
type Node struct {
	Value      string          // actual content
	Next       *Node           // the next item in the current sexp
	Children   []*Node         // the children of this sexp
	Attributes map[string]bool // special attributes for this node
	Original   string          // original line used before parsing
	Flags      []string        // only top Node should have this set
	StartLine  int             // the line in the original dockerfile where the node begins
	EndLine    int             // the line in the original dockerfile where the node ends
}

// Dump dumps the AST defined by `node` as a list of sexps.
// Returns a string suitable for printing.
func (node *Node) Dump() string {
	str := ""
	str += node.Value

	if len(node.Flags) > 0 {
		str += fmt.Sprintf(" %q", node.Flags)
	}

	for _, n := range node.Children {
		str += "(" + n.Dump() + ")\n"
	}

	for n := node.Next; n != nil; n = n.Next {
		if len(n.Children) > 0 {
			str += " " + n.Dump()
		} else {
			str += " " + strconv.Quote(n.Value)
		}
	}

	return strings.TrimSpace(str)
}

// Directive is the structure used during a build run to hold the state of
// parsing directives.
type Directive struct {
	EscapeToken           rune           // Current escape token
	LineContinuationRegex *regexp.Regexp // Current line continuation regex
	LookingForDirectives  bool           // Whether we are currently looking for directives
	EscapeSeen            bool           // Whether the escape directive has been seen
}

var (
	dispatch           map[string]func(string, *Directive) (*Node, map[string]bool, error)
	tokenWhitespace    = regexp.MustCompile(`[\t\v\f\r ]+`)
	tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
	tokenComment       = regexp.MustCompile(`^#.*$`)
)

// DefaultEscapeToken is the default escape token
const DefaultEscapeToken = "\\"

// SetEscapeToken sets the default token for escaping characters in a Dockerfile.
func SetEscapeToken(s string, d *Directive) error {
	if s != "`" && s != "\\" {
		return fmt.Errorf("invalid ESCAPE '%s'. Must be ` or \\", s)
	}
	d.EscapeToken = rune(s[0])
	d.LineContinuationRegex = regexp.MustCompile(`\` + s + `[ \t]*$`)
	return nil
}

func init() {
	// Dispatch Table. see line_parsers.go for the parse functions.
	// The command is parsed and mapped to the line parser. The line parser
	// receives the arguments but not the command, and returns an AST after
	// reformulating the arguments according to the rules in the parser
	// functions. Errors are propagated up by Parse() and the resulting AST can
	// be incorporated directly into the existing AST as a next.
	dispatch = map[string]func(string, *Directive) (*Node, map[string]bool, error){
		command.Add:         parseMaybeJSONToList,
		command.Arg:         parseNameOrNameVal,
		command.Cmd:         parseMaybeJSON,
		command.Copy:        parseMaybeJSONToList,
		command.Entrypoint:  parseMaybeJSON,
		command.Env:         parseEnv,
		command.Expose:      parseStringsWhitespaceDelimited,
		command.From:        parseString,
		command.Healthcheck: parseHealthConfig,
		command.Label:       parseLabel,
		command.Maintainer:  parseString,
		command.Onbuild:     parseSubCommand,
		command.Run:         parseMaybeJSON,
		command.Shell:       parseMaybeJSON,
		command.StopSignal:  parseString,
		command.User:        parseString,
		command.Volume:      parseMaybeJSONToList,
		command.Workdir:     parseString,
	}
}

// ParseLine parses a line and returns the remainder.
func ParseLine(line string, d *Directive, ignoreCont bool) (string, *Node, error) {
	if escapeFound, err := handleParserDirective(line, d); err != nil || escapeFound {
		d.EscapeSeen = escapeFound
		return "", nil, err
	}

	d.LookingForDirectives = false

	if line = stripComments(line); line == "" {
		return "", nil, nil
	}

	if !ignoreCont && d.LineContinuationRegex.MatchString(line) {
		line = d.LineContinuationRegex.ReplaceAllString(line, "")
		return line, nil, nil
	}

	node, err := newNodeFromLine(line, d)
	return "", node, err
}

// newNodeFromLine splits the line into parts, and dispatches to a function
// based on the command and command arguments. A Node is created from the
// result of the dispatch.
func newNodeFromLine(line string, directive *Directive) (*Node, error) {
	cmd, flags, args, err := splitCommand(line)
	if err != nil {
		return nil, err
	}

	fn := dispatch[cmd]
	// Ignore invalid Dockerfile instructions
	if fn == nil {
		fn = parseIgnore
	}
	next, attrs, err := fn(args, directive)
	if err != nil {
		return nil, err
	}

	return &Node{
		Value:      cmd,
		Original:   line,
		Flags:      flags,
		Next:       next,
		Attributes: attrs,
	}, nil
}

// Handle the parser directive '# escape=<char>. Parser directives must precede
// any builder instruction or other comments, and cannot be repeated.
func handleParserDirective(line string, d *Directive) (bool, error) {
	if !d.LookingForDirectives {
		return false, nil
	}
	tecMatch := tokenEscapeCommand.FindStringSubmatch(strings.ToLower(line))
	if len(tecMatch) == 0 {
		return false, nil
	}
	if d.EscapeSeen == true {
		return false, fmt.Errorf("only one escape parser directive can be used")
	}
	for i, n := range tokenEscapeCommand.SubexpNames() {
		if n == "escapechar" {
			if err := SetEscapeToken(tecMatch[i], d); err != nil {
				return false, err
			}
			return true, nil
		}
	}
	return false, nil
}

// Parse is the main parse routine.
// It handles an io.ReadWriteCloser and returns the root of the AST.
func Parse(rwc io.Reader, d *Directive) (*Node, error) {
	currentLine := 0
	root := &Node{}
	root.StartLine = -1
	scanner := bufio.NewScanner(rwc)

	utf8bom := []byte{0xEF, 0xBB, 0xBF}
	for scanner.Scan() {
		scannedBytes := scanner.Bytes()
		// We trim UTF8 BOM
		if currentLine == 0 {
			scannedBytes = bytes.TrimPrefix(scannedBytes, utf8bom)
		}
		scannedLine := strings.TrimLeftFunc(string(scannedBytes), unicode.IsSpace)
		currentLine++
		line, child, err := ParseLine(scannedLine, d, false)
		if err != nil {
			return nil, err
		}
		startLine := currentLine

		if line != "" && child == nil {
			for scanner.Scan() {
				newline := scanner.Text()
				currentLine++

				if stripComments(strings.TrimSpace(newline)) == "" {
					continue
				}

				line, child, err = ParseLine(line+newline, d, false)
				if err != nil {
					return nil, err
				}

				if child != nil {
					break
				}
			}
			if child == nil && line != "" {
				// When we call ParseLine we'll pass in 'true' for
				// the ignoreCont param if we're at the EOF. This will
				// prevent the func from returning immediately w/o
				// parsing the line thinking that there's more input
				// to come.

				_, child, err = ParseLine(line, d, scanner.Err() == nil)
				if err != nil {
					return nil, err
				}
			}
		}

		if child != nil {
			// Update the line information for the current child.
			child.StartLine = startLine
			child.EndLine = currentLine
			// Update the line information for the root. The starting line of the root is always the
			// starting line of the first child and the ending line is the ending line of the last child.
			if root.StartLine < 0 {
				root.StartLine = currentLine
			}
			root.EndLine = currentLine
			root.Children = append(root.Children, child)
		}
	}

	return root, nil
}

// covers comments and empty lines. Lines should be trimmed before passing to
// this function.
func stripComments(line string) string {
	// string is already trimmed at this point
	if tokenComment.MatchString(line) {
		return tokenComment.ReplaceAllString(line, "")
	}

	return line
}
<commit_msg>Add named context support<commit_after>// Package parser implements a parser and parse tree dumper for Dockerfiles.
package parser

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"github.com/docker/docker/builder/dockerfile/command"
)

// Node is a structure used to represent a parse tree.
//
// In the node there are three fields, Value, Next, and Children. Value is the
// current token's string value. Next is always the next non-child token, and
// children contains all the children. Here's an example:
//
// (value next (child child-next child-next-next) next-next)
//
// This data structure is frankly pretty lousy for handling complex languages,
// but lucky for us the Dockerfile isn't very complicated. This structure
// works a little more effectively than a "proper" parse tree for our needs.
//
type Node struct {
	Value      string          // actual content
	Next       *Node           // the next item in the current sexp
	Children   []*Node         // the children of this sexp
	Attributes map[string]bool // special attributes for this node
	Original   string          // original line used before parsing
	Flags      []string        // only top Node should have this set
	StartLine  int             // the line in the original dockerfile where the node begins
	EndLine    int             // the line in the original dockerfile where the node ends
}

// Dump dumps the AST defined by `node` as a list of sexps.
// Returns a string suitable for printing.
func (node *Node) Dump() string {
	str := ""
	str += node.Value

	if len(node.Flags) > 0 {
		str += fmt.Sprintf(" %q", node.Flags)
	}

	for _, n := range node.Children {
		str += "(" + n.Dump() + ")\n"
	}

	for n := node.Next; n != nil; n = n.Next {
		if len(n.Children) > 0 {
			str += " " + n.Dump()
		} else {
			str += " " + strconv.Quote(n.Value)
		}
	}

	return strings.TrimSpace(str)
}

// Directive is the structure used during a build run to hold the state of
// parsing directives.
type Directive struct {
	EscapeToken           rune           // Current escape token
	LineContinuationRegex *regexp.Regexp // Current line continuation regex
	LookingForDirectives  bool           // Whether we are currently looking for directives
	EscapeSeen            bool           // Whether the escape directive has been seen
}

var (
	dispatch           map[string]func(string, *Directive) (*Node, map[string]bool, error)
	tokenWhitespace    = regexp.MustCompile(`[\t\v\f\r ]+`)
	tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
	tokenComment       = regexp.MustCompile(`^#.*$`)
)

// DefaultEscapeToken is the default escape token
const DefaultEscapeToken = "\\"

// SetEscapeToken sets the default token for escaping characters in a Dockerfile.
func SetEscapeToken(s string, d *Directive) error {
	if s != "`" && s != "\\" {
		return fmt.Errorf("invalid ESCAPE '%s'. Must be ` or \\", s)
	}
	d.EscapeToken = rune(s[0])
	d.LineContinuationRegex = regexp.MustCompile(`\` + s + `[ \t]*$`)
	return nil
}

func init() {
	// Dispatch Table. see line_parsers.go for the parse functions.
	// The command is parsed and mapped to the line parser. The line parser
	// receives the arguments but not the command, and returns an AST after
	// reformulating the arguments according to the rules in the parser
	// functions. Errors are propagated up by Parse() and the resulting AST can
	// be incorporated directly into the existing AST as a next.
	dispatch = map[string]func(string, *Directive) (*Node, map[string]bool, error){
		command.Add:         parseMaybeJSONToList,
		command.Arg:         parseNameOrNameVal,
		command.Cmd:         parseMaybeJSON,
		command.Copy:        parseMaybeJSONToList,
		command.Entrypoint:  parseMaybeJSON,
		command.Env:         parseEnv,
		command.Expose:      parseStringsWhitespaceDelimited,
		command.From:        parseStringsWhitespaceDelimited,
		command.Healthcheck: parseHealthConfig,
		command.Label:       parseLabel,
		command.Maintainer:  parseString,
		command.Onbuild:     parseSubCommand,
		command.Run:         parseMaybeJSON,
		command.Shell:       parseMaybeJSON,
		command.StopSignal:  parseString,
		command.User:        parseString,
		command.Volume:      parseMaybeJSONToList,
		command.Workdir:     parseString,
	}
}

// ParseLine parses a line and returns the remainder.
func ParseLine(line string, d *Directive, ignoreCont bool) (string, *Node, error) {
	if escapeFound, err := handleParserDirective(line, d); err != nil || escapeFound {
		d.EscapeSeen = escapeFound
		return "", nil, err
	}

	d.LookingForDirectives = false

	if line = stripComments(line); line == "" {
		return "", nil, nil
	}

	if !ignoreCont && d.LineContinuationRegex.MatchString(line) {
		line = d.LineContinuationRegex.ReplaceAllString(line, "")
		return line, nil, nil
	}

	node, err := newNodeFromLine(line, d)
	return "", node, err
}

// newNodeFromLine splits the line into parts, and dispatches to a function
// based on the command and command arguments. A Node is created from the
// result of the dispatch.
func newNodeFromLine(line string, directive *Directive) (*Node, error) {
	cmd, flags, args, err := splitCommand(line)
	if err != nil {
		return nil, err
	}

	fn := dispatch[cmd]
	// Ignore invalid Dockerfile instructions
	if fn == nil {
		fn = parseIgnore
	}
	next, attrs, err := fn(args, directive)
	if err != nil {
		return nil, err
	}

	return &Node{
		Value:      cmd,
		Original:   line,
		Flags:      flags,
		Next:       next,
		Attributes: attrs,
	}, nil
}

// Handle the parser directive '# escape=<char>. Parser directives must precede
// any builder instruction or other comments, and cannot be repeated.
func handleParserDirective(line string, d *Directive) (bool, error) {
	if !d.LookingForDirectives {
		return false, nil
	}
	tecMatch := tokenEscapeCommand.FindStringSubmatch(strings.ToLower(line))
	if len(tecMatch) == 0 {
		return false, nil
	}
	if d.EscapeSeen == true {
		return false, fmt.Errorf("only one escape parser directive can be used")
	}
	for i, n := range tokenEscapeCommand.SubexpNames() {
		if n == "escapechar" {
			if err := SetEscapeToken(tecMatch[i], d); err != nil {
				return false, err
			}
			return true, nil
		}
	}
	return false, nil
}

// Parse is the main parse routine.
// It handles an io.ReadWriteCloser and returns the root of the AST.
func Parse(rwc io.Reader, d *Directive) (*Node, error) {
	currentLine := 0
	root := &Node{}
	root.StartLine = -1
	scanner := bufio.NewScanner(rwc)

	utf8bom := []byte{0xEF, 0xBB, 0xBF}
	for scanner.Scan() {
		scannedBytes := scanner.Bytes()
		// We trim UTF8 BOM
		if currentLine == 0 {
			scannedBytes = bytes.TrimPrefix(scannedBytes, utf8bom)
		}
		scannedLine := strings.TrimLeftFunc(string(scannedBytes), unicode.IsSpace)
		currentLine++
		line, child, err := ParseLine(scannedLine, d, false)
		if err != nil {
			return nil, err
		}
		startLine := currentLine

		if line != "" && child == nil {
			for scanner.Scan() {
				newline := scanner.Text()
				currentLine++

				if stripComments(strings.TrimSpace(newline)) == "" {
					continue
				}

				line, child, err = ParseLine(line+newline, d, false)
				if err != nil {
					return nil, err
				}

				if child != nil {
					break
				}
			}
			if child == nil && line != "" {
				// When we call ParseLine we'll pass in 'true' for
				// the ignoreCont param if we're at the EOF. This will
				// prevent the func from returning immediately w/o
				// parsing the line thinking that there's more input
				// to come.

				_, child, err = ParseLine(line, d, scanner.Err() == nil)
				if err != nil {
					return nil, err
				}
			}
		}

		if child != nil {
			// Update the line information for the current child.
			child.StartLine = startLine
			child.EndLine = currentLine
			// Update the line information for the root. The starting line of the root is always the
			// starting line of the first child and the ending line is the ending line of the last child.
			if root.StartLine < 0 {
				root.StartLine = currentLine
			}
			root.EndLine = currentLine
			root.Children = append(root.Children, child)
		}
	}

	return root, nil
}

// covers comments and empty lines. Lines should be trimmed before passing to
// this function.
func stripComments(line string) string {
	// string is already trimmed at this point
	if tokenComment.MatchString(line) {
		return tokenComment.ReplaceAllString(line, "")
	}

	return line
}
<|endoftext|>


<commit_before>//  Copyright (c) 2015-2017 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package middleware

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/troubling/hummingbird/client"
	"github.com/troubling/hummingbird/common"
	"github.com/troubling/hummingbird/common/ring"
	"github.com/troubling/hummingbird/common/srv"
	"go.uber.org/zap"
)

var (
	autoCreateAccounts = true // TODO; figure out how to plumb this in as a config
	serverInfo         = make(map[string]interface{})
	sil                sync.Mutex
	excludeHeaders     = []string{
		"X-Account-Sysmeta-",
		"X-Container-Sysmeta-",
		"X-Object-Sysmeta-",
		"X-Object-Transient-Sysmeta-",
		"X-Backend-",
	}
)

func RegisterInfo(name string, data interface{}) {
	sil.Lock()
	defer sil.Unlock()
	serverInfo[name] = data
}

func serverInfoDump() ([]byte, error) {
	sil.Lock()
	defer sil.Unlock()
	data, err := json.Marshal(serverInfo)
	return data, err
}

type AccountInfo struct {
	ContainerCount int64
	ObjectCount    int64
	ObjectBytes    int64
	Metadata       map[string]string
	SysMetadata    map[string]string
}

type AuthorizeFunc func(r *http.Request) bool
type subrequestCopy func(dst, src *http.Request)

type ProxyContextMiddleware struct {
	next              http.Handler
	log               srv.LowLevelLogger
	Cache             ring.MemcacheRing
	proxyDirectClient *client.ProxyDirectClient
}

type ProxyContext struct {
	*ProxyContextMiddleware
	C                client.ProxyClient
	Authorize        AuthorizeFunc
	RemoteUser       string
	ResellerRequest  bool
	ACL              string
	subrequestCopy   subrequestCopy
	Logger           srv.LowLevelLogger
	TxId             string
	responseSent     bool
	status           int
	accountInfoCache map[string]*AccountInfo
	depth            int
	Source           string
}

func GetProxyContext(r *http.Request) *ProxyContext {
	if rv := r.Context().Value("proxycontext"); rv != nil {
		return rv.(*ProxyContext)
	}
	return nil
}

func (ctx *ProxyContext) Response() (bool, int) {
	return ctx.responseSent, ctx.status
}

func (ctx *ProxyContext) addSubrequestCopy(f subrequestCopy) {
	if ctx.subrequestCopy == nil {
		ctx.subrequestCopy = f
		return
	}
	ca := ctx.subrequestCopy
	ctx.subrequestCopy = func(dst, src *http.Request) {
		ca(dst, src)
		f(dst, src)
	}
}

func getPathParts(request *http.Request) (bool, string, string, string) {
	parts := strings.SplitN(request.URL.Path, "/", 5)
	apiRequest := len(parts) > 0 && parts[1] == "v1"
	if len(parts) == 5 {
		return apiRequest, parts[2], parts[3], parts[4]
	} else if len(parts) == 4 {
		return apiRequest, parts[2], parts[3], ""
	} else if len(parts) == 3 {
		return apiRequest, parts[2], "", ""
	}
	return apiRequest, "", "", ""
}

func (ctx *ProxyContext) GetAccountInfo(account string) *AccountInfo {
	key := fmt.Sprintf("account/%s", account)
	ai := ctx.accountInfoCache[key]
	if ai == nil {
		if err := ctx.Cache.GetStructured(key, &ai); err != nil {
			ai = nil
		}
	}
	if ai == nil {
		resp := ctx.C.HeadAccount(account, nil)
		if resp.StatusCode == http.StatusNotFound && autoCreateAccounts {
			resp.Body.Close()
			ctx.C.PutAccount(account, http.Header{"X-Timestamp": []string{common.GetTimestamp()}})
			resp = ctx.C.HeadAccount(account, nil)
		}
		resp.Body.Close()
		if resp.StatusCode/100 != 2 {
			return nil
		}
		ai = &AccountInfo{
			Metadata:    make(map[string]string),
			SysMetadata: make(map[string]string),
		}
		var err error
		if ai.ContainerCount, err = strconv.ParseInt(resp.Header.Get("X-Account-Container-Count"), 10, 64); err != nil {
			return nil
		}
		if ai.ObjectCount, err = strconv.ParseInt(resp.Header.Get("X-Account-Object-Count"), 10, 64); err != nil {
			return nil
		}
		if ai.ObjectBytes, err = strconv.ParseInt(resp.Header.Get("X-Account-Bytes-Used"), 10, 64); err != nil {
			return nil
		}
		for k := range resp.Header {
			if strings.HasPrefix(k, "X-Account-Meta-") {
				ai.Metadata[k[15:]] = resp.Header.Get(k)
			} else if strings.HasPrefix(k, "X-Account-Sysmeta-") {
				ai.SysMetadata[k[18:]] = resp.Header.Get(k)
			}
		}
		ctx.Cache.Set(key, ai, 30)
	}
	return ai
}

func (ctx *ProxyContext) InvalidateAccountInfo(account string) {
	key := fmt.Sprintf("account/%s", account)
	delete(ctx.accountInfoCache, key)
	ctx.Cache.Delete(key)
}

func (ctx *ProxyContext) newSubrequest(method, urlStr string, body io.Reader, req *http.Request, source string) (*http.Request, error) {
	subreq, err := http.NewRequest(method, urlStr, body)
	if err != nil {
		return nil, err
	}
	subctx := &ProxyContext{
		ProxyContextMiddleware: ctx.ProxyContextMiddleware,
		Authorize:              ctx.Authorize,
		RemoteUser:             ctx.RemoteUser,
		subrequestCopy:         ctx.subrequestCopy,
		Logger:                 ctx.Logger.With(zap.String("src", source)),
		C:                      ctx.C,
		TxId:                   ctx.TxId,
		accountInfoCache:       ctx.accountInfoCache,
		responseSent:           false,
		status:                 500,
		depth:                  ctx.depth + 1,
		Source:                 source,
	}
	subreq = subreq.WithContext(context.WithValue(req.Context(), "proxycontext", subctx))
	if subctx.subrequestCopy != nil {
		subctx.subrequestCopy(subreq, req)
	}
	subreq.Header.Set("X-Trans-Id", subctx.TxId)
	subreq.Header.Set("X-Timestamp", common.GetTimestamp())
	return subreq, nil
}

func (ctx *ProxyContext) serveHTTPSubrequest(writer http.ResponseWriter, subreq *http.Request) {
	subctx := GetProxyContext(subreq)
	// TODO: check subctx.depth
	subwriter := srv.NewCustomWriter(writer, func(w http.ResponseWriter, status int) int {
		subctx.responseSent = true
		subctx.status = status
		return status
	})
	subwriter.Header().Set("X-Trans-Id", subctx.TxId)
	ctx.next.ServeHTTP(subwriter, subreq)
}

func (m *ProxyContextMiddleware) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	if !srv.ValidateRequest(writer, request) {
		return
	}

	if request.URL.Path == "/info" {
		if request.Method == "GET" {
			if data, err := serverInfoDump(); err != nil {
				srv.StandardResponse(writer, 500)
			} else {
				writer.Header().Set("Content-Type", "application/json")
				writer.WriteHeader(200)
				writer.Write(data)
			}
			return
		} else if request.Method == "OPTIONS" {
			writer.Header().Set("Allow", "HEAD, GET, OPTIONS")
			writer.WriteHeader(200)
			return
		} else if request.Method == "HEAD" {
			if _, err := serverInfoDump(); err != nil {
				srv.StandardResponse(writer, 500)
			} else {
				writer.WriteHeader(200)
			}
			return
		}
	}

	for k := range request.Header {
		for _, ex := range excludeHeaders {
			if strings.HasPrefix(k, ex) || k == "X-Timestamp" {
				delete(request.Header, k)
			}
		}
	}

	transId := common.GetTransactionId()
	request.Header.Set("X-Trans-Id", transId)
	writer.Header().Set("X-Trans-Id", transId)
	writer.Header().Set("X-Openstack-Request-Id", transId)
	request.Header.Set("X-Timestamp", common.GetTimestamp())
	logr := m.log.With(zap.String("txn", transId))
	ctx := &ProxyContext{
		ProxyContextMiddleware: m,
		Authorize:              nil,
		Logger:                 logr,
		TxId:                   transId,
		responseSent:           false,
		status:                 500,
		accountInfoCache:       make(map[string]*AccountInfo),
		C:                      client.NewProxyClient(m.proxyDirectClient, m.Cache, make(map[string]*client.ContainerInfo)),
	}
	// we'll almost certainly need the AccountInfo and ContainerInfo for the current path, so pre-fetch them in parallel.
	apiRequest, account, container, _ := getPathParts(request)
	if apiRequest && account != "" {
		wg := &sync.WaitGroup{}
		wg.Add(1)
		go func() {
			defer wg.Done()
			if ctx.GetAccountInfo(account) == nil {
				hdr := http.Header{
					"X-Timestamp": []string{common.GetTimestamp()},
				}
				ctx.C.PutAccount(account, hdr) // Auto-create the account if we can't get its info
			}
		}()
		if container != "" {
			wg.Add(1)
			go func() {
				defer wg.Done()
				ctx.C.GetContainerInfo(account, container)
			}()
		}
		wg.Wait()
	}
	newWriter := srv.NewCustomWriter(writer, func(w http.ResponseWriter, status int) int {
		// strip out any bad headers before calling real WriteHeader
		for k := range w.Header() {
			if k == "X-Account-Sysmeta-Project-Domain-Id" {
				w.Header().Set("X-Account-Project-Domain-Id", w.Header().Get(k))
			}
			for _, ex := range excludeHeaders {
				if strings.HasPrefix(k, ex) {
					delete(w.Header(), k)
				}
			}
		}
		ctx.responseSent = true
		ctx.status = status
		return status
	})
	request = request.WithContext(context.WithValue(request.Context(), "proxycontext", ctx))
	m.next.ServeHTTP(newWriter, request)
}

func NewContext(mc ring.MemcacheRing, log srv.LowLevelLogger, proxyDirectClient *client.ProxyDirectClient) func(http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return &ProxyContextMiddleware{
			Cache:             mc,
			log:               log,
			next:              next,
			proxyDirectClient: proxyDirectClient,
		}
	}
}
<commit_msg>Add Referer to subrequests<commit_after>//  Copyright (c) 2015-2017 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package middleware

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/troubling/hummingbird/client"
	"github.com/troubling/hummingbird/common"
	"github.com/troubling/hummingbird/common/ring"
	"github.com/troubling/hummingbird/common/srv"
	"go.uber.org/zap"
)

var (
	autoCreateAccounts = true // TODO; figure out how to plumb this in as a config
	serverInfo         = make(map[string]interface{})
	sil                sync.Mutex
	excludeHeaders     = []string{
		"X-Account-Sysmeta-",
		"X-Container-Sysmeta-",
		"X-Object-Sysmeta-",
		"X-Object-Transient-Sysmeta-",
		"X-Backend-",
	}
)

func RegisterInfo(name string, data interface{}) {
	sil.Lock()
	defer sil.Unlock()
	serverInfo[name] = data
}

func serverInfoDump() ([]byte, error) {
	sil.Lock()
	defer sil.Unlock()
	data, err := json.Marshal(serverInfo)
	return data, err
}

type AccountInfo struct {
	ContainerCount int64
	ObjectCount    int64
	ObjectBytes    int64
	Metadata       map[string]string
	SysMetadata    map[string]string
}

type AuthorizeFunc func(r *http.Request) bool
type subrequestCopy func(dst, src *http.Request)

type ProxyContextMiddleware struct {
	next              http.Handler
	log               srv.LowLevelLogger
	Cache             ring.MemcacheRing
	proxyDirectClient *client.ProxyDirectClient
}

type ProxyContext struct {
	*ProxyContextMiddleware
	C                client.ProxyClient
	Authorize        AuthorizeFunc
	RemoteUser       string
	ResellerRequest  bool
	ACL              string
	subrequestCopy   subrequestCopy
	Logger           srv.LowLevelLogger
	TxId             string
	responseSent     bool
	status           int
	accountInfoCache map[string]*AccountInfo
	depth            int
	Source           string
}

func GetProxyContext(r *http.Request) *ProxyContext {
	if rv := r.Context().Value("proxycontext"); rv != nil {
		return rv.(*ProxyContext)
	}
	return nil
}

func (ctx *ProxyContext) Response() (bool, int) {
	return ctx.responseSent, ctx.status
}

func (ctx *ProxyContext) addSubrequestCopy(f subrequestCopy) {
	if ctx.subrequestCopy == nil {
		ctx.subrequestCopy = f
		return
	}
	ca := ctx.subrequestCopy
	ctx.subrequestCopy = func(dst, src *http.Request) {
		ca(dst, src)
		f(dst, src)
	}
}

func getPathParts(request *http.Request) (bool, string, string, string) {
	parts := strings.SplitN(request.URL.Path, "/", 5)
	apiRequest := len(parts) > 0 && parts[1] == "v1"
	if len(parts) == 5 {
		return apiRequest, parts[2], parts[3], parts[4]
	} else if len(parts) == 4 {
		return apiRequest, parts[2], parts[3], ""
	} else if len(parts) == 3 {
		return apiRequest, parts[2], "", ""
	}
	return apiRequest, "", "", ""
}

func (ctx *ProxyContext) GetAccountInfo(account string) *AccountInfo {
	key := fmt.Sprintf("account/%s", account)
	ai := ctx.accountInfoCache[key]
	if ai == nil {
		if err := ctx.Cache.GetStructured(key, &ai); err != nil {
			ai = nil
		}
	}
	if ai == nil {
		resp := ctx.C.HeadAccount(account, nil)
		if resp.StatusCode == http.StatusNotFound && autoCreateAccounts {
			resp.Body.Close()
			ctx.C.PutAccount(account, http.Header{"X-Timestamp": []string{common.GetTimestamp()}})
			resp = ctx.C.HeadAccount(account, nil)
		}
		resp.Body.Close()
		if resp.StatusCode/100 != 2 {
			return nil
		}
		ai = &AccountInfo{
			Metadata:    make(map[string]string),
			SysMetadata: make(map[string]string),
		}
		var err error
		if ai.ContainerCount, err = strconv.ParseInt(resp.Header.Get("X-Account-Container-Count"), 10, 64); err != nil {
			return nil
		}
		if ai.ObjectCount, err = strconv.ParseInt(resp.Header.Get("X-Account-Object-Count"), 10, 64); err != nil {
			return nil
		}
		if ai.ObjectBytes, err = strconv.ParseInt(resp.Header.Get("X-Account-Bytes-Used"), 10, 64); err != nil {
			return nil
		}
		for k := range resp.Header {
			if strings.HasPrefix(k, "X-Account-Meta-") {
				ai.Metadata[k[15:]] = resp.Header.Get(k)
			} else if strings.HasPrefix(k, "X-Account-Sysmeta-") {
				ai.SysMetadata[k[18:]] = resp.Header.Get(k)
			}
		}
		ctx.Cache.Set(key, ai, 30)
	}
	return ai
}

func (ctx *ProxyContext) InvalidateAccountInfo(account string) {
	key := fmt.Sprintf("account/%s", account)
	delete(ctx.accountInfoCache, key)
	ctx.Cache.Delete(key)
}

func (ctx *ProxyContext) newSubrequest(method, urlStr string, body io.Reader, req *http.Request, source string) (*http.Request, error) {
	subreq, err := http.NewRequest(method, urlStr, body)
	if err != nil {
		return nil, err
	}
	subctx := &ProxyContext{
		ProxyContextMiddleware: ctx.ProxyContextMiddleware,
		Authorize:              ctx.Authorize,
		RemoteUser:             ctx.RemoteUser,
		subrequestCopy:         ctx.subrequestCopy,
		Logger:                 ctx.Logger.With(zap.String("src", source)),
		C:                      ctx.C,
		TxId:                   ctx.TxId,
		accountInfoCache:       ctx.accountInfoCache,
		responseSent:           false,
		status:                 500,
		depth:                  ctx.depth + 1,
		Source:                 source,
	}
	subreq = subreq.WithContext(context.WithValue(req.Context(), "proxycontext", subctx))
	if subctx.subrequestCopy != nil {
		subctx.subrequestCopy(subreq, req)
	}
	if v := req.Header.Get("Referer"); v != "" {
		subreq.Header.Set("Referer", v)
	}
	subreq.Header.Set("X-Trans-Id", subctx.TxId)
	subreq.Header.Set("X-Timestamp", common.GetTimestamp())
	return subreq, nil
}

func (ctx *ProxyContext) serveHTTPSubrequest(writer http.ResponseWriter, subreq *http.Request) {
	subctx := GetProxyContext(subreq)
	// TODO: check subctx.depth
	subwriter := srv.NewCustomWriter(writer, func(w http.ResponseWriter, status int) int {
		subctx.responseSent = true
		subctx.status = status
		return status
	})
	subwriter.Header().Set("X-Trans-Id", subctx.TxId)
	ctx.next.ServeHTTP(subwriter, subreq)
}

func (m *ProxyContextMiddleware) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	if !srv.ValidateRequest(writer, request) {
		return
	}

	if request.URL.Path == "/info" {
		if request.Method == "GET" {
			if data, err := serverInfoDump(); err != nil {
				srv.StandardResponse(writer, 500)
			} else {
				writer.Header().Set("Content-Type", "application/json")
				writer.WriteHeader(200)
				writer.Write(data)
			}
			return
		} else if request.Method == "OPTIONS" {
			writer.Header().Set("Allow", "HEAD, GET, OPTIONS")
			writer.WriteHeader(200)
			return
		} else if request.Method == "HEAD" {
			if _, err := serverInfoDump(); err != nil {
				srv.StandardResponse(writer, 500)
			} else {
				writer.WriteHeader(200)
			}
			return
		}
	}

	for k := range request.Header {
		for _, ex := range excludeHeaders {
			if strings.HasPrefix(k, ex) || k == "X-Timestamp" {
				delete(request.Header, k)
			}
		}
	}

	transId := common.GetTransactionId()
	request.Header.Set("X-Trans-Id", transId)
	writer.Header().Set("X-Trans-Id", transId)
	writer.Header().Set("X-Openstack-Request-Id", transId)
	request.Header.Set("X-Timestamp", common.GetTimestamp())
	logr := m.log.With(zap.String("txn", transId))
	ctx := &ProxyContext{
		ProxyContextMiddleware: m,
		Authorize:              nil,
		Logger:                 logr,
		TxId:                   transId,
		responseSent:           false,
		status:                 500,
		accountInfoCache:       make(map[string]*AccountInfo),
		C:                      client.NewProxyClient(m.proxyDirectClient, m.Cache, make(map[string]*client.ContainerInfo)),
	}
	// we'll almost certainly need the AccountInfo and ContainerInfo for the current path, so pre-fetch them in parallel.
	apiRequest, account, container, _ := getPathParts(request)
	if apiRequest && account != "" {
		wg := &sync.WaitGroup{}
		wg.Add(1)
		go func() {
			defer wg.Done()
			if ctx.GetAccountInfo(account) == nil {
				hdr := http.Header{
					"X-Timestamp": []string{common.GetTimestamp()},
				}
				ctx.C.PutAccount(account, hdr) // Auto-create the account if we can't get its info
			}
		}()
		if container != "" {
			wg.Add(1)
			go func() {
				defer wg.Done()
				ctx.C.GetContainerInfo(account, container)
			}()
		}
		wg.Wait()
	}
	newWriter := srv.NewCustomWriter(writer, func(w http.ResponseWriter, status int) int {
		// strip out any bad headers before calling real WriteHeader
		for k := range w.Header() {
			if k == "X-Account-Sysmeta-Project-Domain-Id" {
				w.Header().Set("X-Account-Project-Domain-Id", w.Header().Get(k))
			}
			for _, ex := range excludeHeaders {
				if strings.HasPrefix(k, ex) {
					delete(w.Header(), k)
				}
			}
		}
		ctx.responseSent = true
		ctx.status = status
		return status
	})
	request = request.WithContext(context.WithValue(request.Context(), "proxycontext", ctx))
	m.next.ServeHTTP(newWriter, request)
}

func NewContext(mc ring.MemcacheRing, log srv.LowLevelLogger, proxyDirectClient *client.ProxyDirectClient) func(http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return &ProxyContextMiddleware{
			Cache:             mc,
			log:               log,
			next:              next,
			proxyDirectClient: proxyDirectClient,
		}
	}
}
<|endoftext|>


<commit_before>package runtime

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/taskcluster/taskcluster-base-go/jsontest"
	"github.com/xeipuuv/gojsonschema"
)

// ErrConflictingSchemas is returned if two schema entries are conflicting.
var ErrConflictingSchemas = errors.New("Two schema entries are conflicting!")

// CompositeSchema hides one or more composed JSON schemas.
type CompositeSchema interface {
	Parse(data map[string]json.RawMessage) (interface{}, error)
	visit(visitor func(*schemaEntry))
}

type emptySchema struct{}

type schemaEntry struct {
	property   string
	schema     string
	required   bool
	makeTarget func() interface{}
	validator  *gojsonschema.Schema
}

type composedSchema []CompositeSchema

// NewEmptyCompositeSchema returns a CompositeSchema schema that is empty.
// The resulting value from Parse is nil, and the schema does no validation.
func NewEmptyCompositeSchema() CompositeSchema {
	return &emptySchema{}
}

func (*emptySchema) Parse(map[string]json.RawMessage) (interface{}, error) {
	return nil, nil
}

func (*emptySchema) visit(func(*schemaEntry)) {}

// NewCompositeSchema creates a CompositeSchema from the description of a single
// property and a function to produce unmarshalling targets with.
//
// Schema will only validate the 'property' against the JSON schema passed as
// 'schema'. If the 'required' is true the property must be present.
//
// When parsing the makeTarget will be used as factory to create objects
// which the payload will unmarshalled to.
func NewCompositeSchema(
	property string,
	schema string,
	required bool,
	makeTarget func() interface{},
) (CompositeSchema, error) {
	validator, err := gojsonschema.NewSchema(gojsonschema.NewStringLoader(schema))
	if err != nil {
		return nil, err
	}
	return &schemaEntry{
		property:   property,
		schema:     schema,
		required:   required,
		makeTarget: makeTarget,
		validator:  validator,
	}, nil
}

// MergeCompositeSchemas will merge two or more CompositeSchema
//
// When CompositeSchema.Parse is called it will return an array of the results
// from the schemas that were merged. Hence, the order in which the schemas is
// given is important and will be preserved.
//
// This function may return ErrConflictingSchemas, if two of the schemas merged
// have conflicting definitions.
func MergeCompositeSchemas(schemas ...CompositeSchema) (CompositeSchema, error) {
	hasConflict := false
	for i, schema := range schemas {
		schema.visit(func(entry *schemaEntry) {
			for _, s := range schemas[i:] {
				s.visit(func(e *schemaEntry) {
					if entry.property == e.property {
						if schemasMatch, _, _, _ := jsontest.JsonEqual([]byte(entry.schema), []byte(e.schema)); !schemasMatch {
							// TODO: We probably should make an error with a custom message
							hasConflict = true
						}
					}
				})
			}
		})
	}
	if hasConflict {
		return nil, ErrConflictingSchemas
	}
	return composedSchema(schemas), nil
}

// Parse will validate and parse data.
//
// This method will return an object returned from makeTarget (or )
func (s *schemaEntry) Parse(data map[string]json.RawMessage) (interface{}, error) {
	// TODO: Validate property against schema
	value := data[s.property]
	if value == nil {
		if s.required {
			return nil, errors.New("Property is missing")
		}
		return nil, nil
	}

	// Validate value against json schema
	result, err := s.validator.Validate(gojsonschema.NewStringLoader(string(value)))
	if err != nil {
		return nil, err
	}
	// Check for validation errors
	if !result.Valid() {
		message := "JSON schema validation failed:\n"
		for _, err := range result.Errors() {
			// Err implements the ResultError interface
			message += err.Description() + "\n"
		}
		return nil, errors.New(message)
	}

	// Unmarshal value to target
	target := s.makeTarget()
	err = json.Unmarshal(value, target)
	if err != nil {
		return nil, err
	}
	return target, nil
}

func (s *schemaEntry) visit(visitor func(entry *schemaEntry)) {
	visitor(s)
}

func (s composedSchema) Parse(data map[string]json.RawMessage) (interface{}, error) {
	results := []interface{}{}
	for _, entry := range s {
		result, err := entry.Parse(data)
		if err != nil {
			return nil, err
		}
		results = append(results, result)
	}
	return results, nil
}

func (s composedSchema) visit(visitor func(*schemaEntry)) {
	for _, entry := range s {
		entry.visit(visitor)
	}
}

// Schema returns the composed JSON schema from a CompositeSchema object
func Schema(schema CompositeSchema, id, title, description string) string {
	properties := make(map[string]*json.RawMessage)
	required := []string{}

	schema.visit(func(entry *schemaEntry) {
		s := json.RawMessage(entry.schema)
		properties[entry.property] = &s

		// We panic if there is two entries for the same property with different
		// schemas... This shouldn't be possible as MergeCompositeSchemas checks it.
		if properties[entry.property] != nil {
			schemasMatch, _, _, _ := jsontest.JsonEqual(s, []byte(*properties[entry.property]))
			if !schemasMatch {
				panic(fmt.Sprint(
					"It shouldn't be possible to construct a CompositeSchema with two ",
					"different schemas for the same property. This happend for ",
					"property: ", entry.property, " where: ", entry.schema, " is ",
					"different from: ", string(*properties[entry.property]),
				))
			}
		}

		// If property is required and not already in the list we append it
		if entry.required {
			contains := false
			for _, property := range required {
				contains = contains || property == entry.property
			}
			if !contains {
				required = append(required, entry.property)
			}
		}
	})

	// Marshal to JSON, as this is generated for export we will intent it, just
	// in case some human decides to read it.
	json, err := json.MarshalIndent(struct {
		ID                   string                      `json:"id"`
		Schema               string                      `json:"$schema"`
		Title                string                      `json:"title"`
		Description          string                      `json:"description"`
		Type                 string                      `json:"type"`
		Properties           map[string]*json.RawMessage `json:"properties,omitempty"`
		Required             []string                    `json:"required,omitempty"`
		AdditionalProperties bool                        `json:"additionalProperties"`
	}{
		ID:                   id,
		Schema:               "http://json-schema.org/draft-04/schema#",
		Title:                title,
		Description:          description,
		Type:                 "object",
		Properties:           properties,
		Required:             required,
		AdditionalProperties: false,
	}, "", "  ")
	// All properties should be JSON, this should always render to valid JSON.
	// Errors can't happen here, they should happen in NewCompositeSchema
	if err != nil {
		panic(fmt.Sprint("Rendering composite schema to JSON failed: ", err))
	}

	return string(json)
}
<commit_msg>Tell which field is missing when schema parsing fails.<commit_after>package runtime

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/taskcluster/taskcluster-base-go/jsontest"
	"github.com/xeipuuv/gojsonschema"
)

// ErrConflictingSchemas is returned if two schema entries are conflicting.
var ErrConflictingSchemas = errors.New("Two schema entries are conflicting!")

// CompositeSchema hides one or more composed JSON schemas.
type CompositeSchema interface {
	Parse(data map[string]json.RawMessage) (interface{}, error)
	visit(visitor func(*schemaEntry))
}

type emptySchema struct{}

type schemaEntry struct {
	property   string
	schema     string
	required   bool
	makeTarget func() interface{}
	validator  *gojsonschema.Schema
}

type composedSchema []CompositeSchema

// NewEmptyCompositeSchema returns a CompositeSchema schema that is empty.
// The resulting value from Parse is nil, and the schema does no validation.
func NewEmptyCompositeSchema() CompositeSchema {
	return &emptySchema{}
}

func (*emptySchema) Parse(map[string]json.RawMessage) (interface{}, error) {
	return nil, nil
}

func (*emptySchema) visit(func(*schemaEntry)) {}

// NewCompositeSchema creates a CompositeSchema from the description of a single
// property and a function to produce unmarshalling targets with.
//
// Schema will only validate the 'property' against the JSON schema passed as
// 'schema'. If the 'required' is true the property must be present.
//
// When parsing the makeTarget will be used as factory to create objects
// which the payload will unmarshalled to.
func NewCompositeSchema(
	property string,
	schema string,
	required bool,
	makeTarget func() interface{},
) (CompositeSchema, error) {
	validator, err := gojsonschema.NewSchema(gojsonschema.NewStringLoader(schema))
	if err != nil {
		return nil, err
	}
	return &schemaEntry{
		property:   property,
		schema:     schema,
		required:   required,
		makeTarget: makeTarget,
		validator:  validator,
	}, nil
}

// MergeCompositeSchemas will merge two or more CompositeSchema
//
// When CompositeSchema.Parse is called it will return an array of the results
// from the schemas that were merged. Hence, the order in which the schemas is
// given is important and will be preserved.
//
// This function may return ErrConflictingSchemas, if two of the schemas merged
// have conflicting definitions.
func MergeCompositeSchemas(schemas ...CompositeSchema) (CompositeSchema, error) {
	hasConflict := false
	for i, schema := range schemas {
		schema.visit(func(entry *schemaEntry) {
			for _, s := range schemas[i:] {
				s.visit(func(e *schemaEntry) {
					if entry.property == e.property {
						if schemasMatch, _, _, _ := jsontest.JsonEqual([]byte(entry.schema), []byte(e.schema)); !schemasMatch {
							// TODO: We probably should make an error with a custom message
							hasConflict = true
						}
					}
				})
			}
		})
	}
	if hasConflict {
		return nil, ErrConflictingSchemas
	}
	return composedSchema(schemas), nil
}

// Parse will validate and parse data.
//
// This method will return an object returned from makeTarget (or )
func (s *schemaEntry) Parse(data map[string]json.RawMessage) (interface{}, error) {
	// TODO: Validate property against schema
	value := data[s.property]
	if value == nil {
		if s.required {
			return nil, errors.New("Property \"" + s.property + "\" is missing")
		}
		return nil, nil
	}

	// Validate value against json schema
	result, err := s.validator.Validate(gojsonschema.NewStringLoader(string(value)))
	if err != nil {
		return nil, err
	}
	// Check for validation errors
	if !result.Valid() {
		message := "JSON schema validation failed:\n"
		for _, err := range result.Errors() {
			// Err implements the ResultError interface
			message += err.Description() + "\n"
		}
		return nil, errors.New(message)
	}

	// Unmarshal value to target
	target := s.makeTarget()
	err = json.Unmarshal(value, target)
	if err != nil {
		return nil, err
	}
	return target, nil
}

func (s *schemaEntry) visit(visitor func(entry *schemaEntry)) {
	visitor(s)
}

func (s composedSchema) Parse(data map[string]json.RawMessage) (interface{}, error) {
	results := []interface{}{}
	for _, entry := range s {
		result, err := entry.Parse(data)
		if err != nil {
			return nil, err
		}
		results = append(results, result)
	}
	return results, nil
}

func (s composedSchema) visit(visitor func(*schemaEntry)) {
	for _, entry := range s {
		entry.visit(visitor)
	}
}

// Schema returns the composed JSON schema from a CompositeSchema object
func Schema(schema CompositeSchema, id, title, description string) string {
	properties := make(map[string]*json.RawMessage)
	required := []string{}

	schema.visit(func(entry *schemaEntry) {
		s := json.RawMessage(entry.schema)
		properties[entry.property] = &s

		// We panic if there is two entries for the same property with different
		// schemas... This shouldn't be possible as MergeCompositeSchemas checks it.
		if properties[entry.property] != nil {
			schemasMatch, _, _, _ := jsontest.JsonEqual(s, []byte(*properties[entry.property]))
			if !schemasMatch {
				panic(fmt.Sprint(
					"It shouldn't be possible to construct a CompositeSchema with two ",
					"different schemas for the same property. This happend for ",
					"property: ", entry.property, " where: ", entry.schema, " is ",
					"different from: ", string(*properties[entry.property]),
				))
			}
		}

		// If property is required and not already in the list we append it
		if entry.required {
			contains := false
			for _, property := range required {
				contains = contains || property == entry.property
			}
			if !contains {
				required = append(required, entry.property)
			}
		}
	})

	// Marshal to JSON, as this is generated for export we will intent it, just
	// in case some human decides to read it.
	json, err := json.MarshalIndent(struct {
		ID                   string                      `json:"id"`
		Schema               string                      `json:"$schema"`
		Title                string                      `json:"title"`
		Description          string                      `json:"description"`
		Type                 string                      `json:"type"`
		Properties           map[string]*json.RawMessage `json:"properties,omitempty"`
		Required             []string                    `json:"required,omitempty"`
		AdditionalProperties bool                        `json:"additionalProperties"`
	}{
		ID:                   id,
		Schema:               "http://json-schema.org/draft-04/schema#",
		Title:                title,
		Description:          description,
		Type:                 "object",
		Properties:           properties,
		Required:             required,
		AdditionalProperties: false,
	}, "", "  ")
	// All properties should be JSON, this should always render to valid JSON.
	// Errors can't happen here, they should happen in NewCompositeSchema
	if err != nil {
		panic(fmt.Sprint("Rendering composite schema to JSON failed: ", err))
	}

	return string(json)
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/fatih/color"
	"github.com/hashicorp/hcl2/hcl"
	"github.com/hashicorp/hcl2/hclparse"
	"golang.org/x/crypto/ssh/terminal"
)

// Create a logger with the given prefix
func CreateLogger(prefix string) *log.Logger {
	return CreateLoggerWithWriter(os.Stderr, prefix)
}

// CreateLoggerWithWriter Create a logger around the given output stream and prefix
func CreateLoggerWithWriter(writer io.Writer, prefix string) *log.Logger {
	if prefix != "" {
		prefix = fmt.Sprintf("[%s] ", prefix)
	}
	return log.New(writer, fmt.Sprintf("[terragrunt] %s", prefix), log.LstdFlags)
}

// MAINTAINER'S NOTE: This is a temporary solution for logging levels in terragrunt. This is not a permanent debug
// logging solution.
// Debugf will only print out terragrunt logs if the TG_LOG environment variable is set to DEBUG.
func Debugf(logger *log.Logger, fmtString string, fmtArgs ...interface{}) {
	if strings.ToLower(os.Getenv("TG_LOG")) == "debug" {
		logger.Printf(fmtString, fmtArgs...)
	}
}

// ColorLogf
func ColorLogf(logger *log.Logger, colorCode *color.Color, fmtString string, fmtArgs ...interface{}) {
	logOut := fmt.Sprintf(fmtString, fmtArgs...)

	loggerIsStderr := logger.Writer() == os.Stderr
	allowColor := terminal.IsTerminal(int(os.Stderr.Fd()))
	if loggerIsStderr && allowColor {
		logOut = colorCode.SprintFunc()(logOut)
	}
	logger.Println(logOut)
}

// GetDiagnosticsWriter returns a hcl2 parsing diagnostics emitter for the current terminal.
func GetDiagnosticsWriter(parser *hclparse.Parser) hcl.DiagnosticWriter {
	termColor := terminal.IsTerminal(int(os.Stderr.Fd()))
	termWidth, _, err := terminal.GetSize(int(os.Stdout.Fd()))
	if err != nil {
		termWidth = 80
	}
	return hcl.NewDiagnosticTextWriter(os.Stderr, parser.Files(), uint(termWidth), termColor)
}
<commit_msg>logger.Writer is not available until go1.12<commit_after>package util

import (
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/fatih/color"
	"github.com/hashicorp/hcl2/hcl"
	"github.com/hashicorp/hcl2/hclparse"
	"golang.org/x/crypto/ssh/terminal"
)

// Create a logger with the given prefix
func CreateLogger(prefix string) *log.Logger {
	return CreateLoggerWithWriter(os.Stderr, prefix)
}

// CreateLoggerWithWriter Create a logger around the given output stream and prefix
func CreateLoggerWithWriter(writer io.Writer, prefix string) *log.Logger {
	if prefix != "" {
		prefix = fmt.Sprintf("[%s] ", prefix)
	}
	return log.New(writer, fmt.Sprintf("[terragrunt] %s", prefix), log.LstdFlags)
}

// MAINTAINER'S NOTE: This is a temporary solution for logging levels in terragrunt. This is not a permanent debug
// logging solution.
// Debugf will only print out terragrunt logs if the TG_LOG environment variable is set to DEBUG.
func Debugf(logger *log.Logger, fmtString string, fmtArgs ...interface{}) {
	if strings.ToLower(os.Getenv("TG_LOG")) == "debug" {
		logger.Printf(fmtString, fmtArgs...)
	}
}

// ColorLogf
func ColorLogf(logger *log.Logger, colorCode *color.Color, fmtString string, fmtArgs ...interface{}) {
	logOut := fmt.Sprintf(fmtString, fmtArgs...)

	allowColor := terminal.IsTerminal(int(os.Stderr.Fd()))
	if allowColor {
		logOut = colorCode.SprintFunc()(logOut)
	}
	logger.Println(logOut)
}

// GetDiagnosticsWriter returns a hcl2 parsing diagnostics emitter for the current terminal.
func GetDiagnosticsWriter(parser *hclparse.Parser) hcl.DiagnosticWriter {
	termColor := terminal.IsTerminal(int(os.Stderr.Fd()))
	termWidth, _, err := terminal.GetSize(int(os.Stdout.Fd()))
	if err != nil {
		termWidth = 80
	}
	return hcl.NewDiagnosticTextWriter(os.Stderr, parser.Files(), uint(termWidth), termColor)
}
<|endoftext|>


<commit_before>package bagins

/*

"Oft the unbidden guest proves the best company."

- Eomer

*/

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// TAG FIELD

/*
 Represents a tag field as referenced in the standard BagIt tag file and used
 in bag-info.txt.  It represents a standard key value pair with the label with corresponding
 value.  For more information see
 http://tools.ietf.org/html/draft-kunze-bagit-09#section-2.2.2
*/
type TagField struct {
	label string // Name of the tag field
	value string // Value of the tag field
}

// Creates and returns a pointer to a new TagField
func NewTagField(label string, value string) *TagField {
	return &TagField{label, value}
}

// Returns the label string for the tag field.
func (f *TagField) Label() string {
	return f.label
}

// Sets the label string for the tag field.
func (f *TagField) SetLabel(l string) {
	f.label = l
}

// Returns the value string for the tag field.
func (f *TagField) Value() string {
	return f.value
}

// Sets the value string for the tag file.
func (f *TagField) SetValue(v string) {
	f.value = v
}

// TAG FIELD LIST

/*
 Represents an ordered list of tag fields as specified for use with bag-info.txt
 in the bag it standard.  It supports ordered, repeatable fields.
 http://tools.ietf.org/html/draft-kunze-bagit-09#section-2.2.2
*/
type TagFieldList struct {
	fields []TagField // Some useful manipulations in https://code.google.com/p/go-wiki/wiki/SliceTricks
}

// Returns a pointer to a new TagFieldList.
func NewTagFieldList() *TagFieldList {
	return new(TagFieldList)
}

// Returns a slice copy of the current tag fields.
func (fl *TagFieldList) Fields() []TagField {
	return fl.fields
}

// Sets the tag field slice to use for the tag field list.
func (fl *TagFieldList) SetFields(fields []TagField) {
	fl.fields = fields
}

// Adds a Field to the end of the tag field list.
func (fl *TagFieldList) AddField(field TagField) {
	fl.fields = append(fl.Fields(), field)
}

/*
 Removes a field from the tag field list at the specified index.  Returns an error if
 index out of bounds.
*/
func (fl *TagFieldList) RemoveField(i int) error {
	if i+1 > len(fl.Fields()) || i < 0 {
		return errors.New("Invalid index for TagField")
	}
	if len(fl.fields) == i {
		fl.fields = fl.Fields()[:i]
		return nil
	}
	fl.fields = append(fl.Fields()[:i], fl.Fields()[i+1:]...)
	return nil
}

// TAG FILES

// Represents a tag file object in the bag with its related fields.
type TagFile struct {
	name string        // Filepath for tag file.
	Data *TagFieldList // key value pairs of data for the tagfile.
}

/*
 Creates a new tagfile object and returns it or returns an error if improperly formatted.
 The name argument represents the filepath of the tagfile, which must end in txt
*/
func NewTagFile(name string) (tf *TagFile, err error) {
	err = validateTagFileName(name)
	tf = new(TagFile)
	tf.name = filepath.Clean(name)
	tf.Data = new(TagFieldList)
	return tf, err
}

/*
 Reads a tagfile, parsing the contents as tagfile field data and returning the TagFile object.
 name is the filepath to the tag file.  It throws an error if contents cannot be properly parsed.
*/
func ReadTagFile(name string) (*TagFile, []error) {
	var errs []error

	file, err := os.Open(name)
	if err != nil {
		return nil, append(errs, err)
	}
	defer file.Close()

	tf, err := NewTagFile(name)
	if err != nil {
		return nil, append(errs, err)
	}

	data, errs := parseTagFields(file)
	tf.Data.SetFields(data)

	return tf, errs
}

// Returns the named filepath of the tagfile.
func (tf *TagFile) Name() string {
	return tf.name
}

/*
 Creates the named tagfile and writes key value pairs to it, with indented
 formatting as indicated in the BagIt spec.
*/
func (tf *TagFile) Create() error {
	// Create directory if needed.
	if err := os.MkdirAll(filepath.Dir(tf.name), 0777); err != nil {
		return err
	}

	// Create the tagfile.
	fileOut, err := os.Create(tf.Name())
	if err != nil {
		return err
	}
	defer fileOut.Close()

	// Write fields and data to the file.
	for _, f := range tf.Data.Fields() {
		field, err := formatField(f.Label(), f.Value())
		if err != nil {
			return err
		}
		_, err = fmt.Fprintln(fileOut, field)
		if err != nil {
			return err
		}
	}

	return nil
}

/*
Takes a tag field key and data and wraps lines at 79 with indented spaces as
per recommendation in spec.
*/
func formatField(key string, data string) (string, error) {
	delimeter := "\n   "
	var buff bytes.Buffer

	// Initiate it by writing the proper key.
	writeLen, err := buff.WriteString(fmt.Sprintf("%s: ", key))
	if err != nil {
		return "", err
	}
	splitCounter := writeLen

	words := strings.Split(data, " ")

	for word := range words {
		if splitCounter+len(words[word]) > 79 {
			splitCounter, err = buff.WriteString(delimeter)
			if err != nil {
				return "", err
			}
		}
		writeLen, err = buff.WriteString(strings.Join([]string{" ", words[word]}, ""))
		if err != nil {
			return "", err
		}
		splitCounter += writeLen

	}

	return buff.String(), nil
}

// Some private convenence methods for manipulating tag files.
func validateTagFileName(name string) (err error) {
	_, err = os.Stat(filepath.Dir(name))
	re, _ := regexp.Compile(`.*\.txt`)
	if !re.MatchString(filepath.Base(name)) {
		err = errors.New(fmt.Sprint("Tagfiles must end in .txt and contain at least 1 letter.  Provided: ", filepath.Base(name)))
	}
	return err
}

/*
 Reads the contents of file and parses tagfile fields from the contents or returns an error if
 it contains unparsable data.
*/
func parseTagFields(file *os.File) ([]TagField, []error) {
	var errors []error
	re, err := regexp.Compile(`^(\S*):?(\s.*)?$`)
	if err != nil {
		errors = append(errors, err)
		return nil, errors
	}

	scanner := bufio.NewScanner(file)
	var fields []TagField
	var field TagField

	// Parse the remaining lines.
	for scanner.Scan() {
		line := scanner.Text()
		// See http://play.golang.org/p/zLqvg2qo1D for some testing on the field match.
		if re.MatchString(line) {
			data := re.FindStringSubmatch(line)
			if data[1] != "" {
				fields = append(fields, field)
				field = *NewTagField(data[1], strings.Trim(data[2], " "))
				continue
			}
			value := strings.Trim(data[2], " ")
			field.SetValue(strings.Join([]string{field.Value(), value}, " "))

		} else {
			err := fmt.Errorf("Unable to parse tag data from line: %s", line)
			errors = append(errors, err)
		}
	}
	if scanner.Err() != nil {
		errors = append(errors, scanner.Err())
	}

	// See http://play.golang.org/p/nsw9zsAEPF for some testing on the field match.
	return fields, errors
}
<commit_msg>Reverted regex change. Strip colon after capture<commit_after>package bagins

/*

"Oft the unbidden guest proves the best company."

- Eomer

*/

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// TAG FIELD

/*
 Represents a tag field as referenced in the standard BagIt tag file and used
 in bag-info.txt.  It represents a standard key value pair with the label with corresponding
 value.  For more information see
 http://tools.ietf.org/html/draft-kunze-bagit-09#section-2.2.2
*/
type TagField struct {
	label string // Name of the tag field
	value string // Value of the tag field
}

// Creates and returns a pointer to a new TagField
func NewTagField(label string, value string) *TagField {
	return &TagField{label, value}
}

// Returns the label string for the tag field.
func (f *TagField) Label() string {
	return f.label
}

// Sets the label string for the tag field.
func (f *TagField) SetLabel(l string) {
	f.label = l
}

// Returns the value string for the tag field.
func (f *TagField) Value() string {
	return f.value
}

// Sets the value string for the tag file.
func (f *TagField) SetValue(v string) {
	f.value = v
}

// TAG FIELD LIST

/*
 Represents an ordered list of tag fields as specified for use with bag-info.txt
 in the bag it standard.  It supports ordered, repeatable fields.
 http://tools.ietf.org/html/draft-kunze-bagit-09#section-2.2.2
*/
type TagFieldList struct {
	fields []TagField // Some useful manipulations in https://code.google.com/p/go-wiki/wiki/SliceTricks
}

// Returns a pointer to a new TagFieldList.
func NewTagFieldList() *TagFieldList {
	return new(TagFieldList)
}

// Returns a slice copy of the current tag fields.
func (fl *TagFieldList) Fields() []TagField {
	return fl.fields
}

// Sets the tag field slice to use for the tag field list.
func (fl *TagFieldList) SetFields(fields []TagField) {
	fl.fields = fields
}

// Adds a Field to the end of the tag field list.
func (fl *TagFieldList) AddField(field TagField) {
	fl.fields = append(fl.Fields(), field)
}

/*
 Removes a field from the tag field list at the specified index.  Returns an error if
 index out of bounds.
*/
func (fl *TagFieldList) RemoveField(i int) error {
	if i+1 > len(fl.Fields()) || i < 0 {
		return errors.New("Invalid index for TagField")
	}
	if len(fl.fields) == i {
		fl.fields = fl.Fields()[:i]
		return nil
	}
	fl.fields = append(fl.Fields()[:i], fl.Fields()[i+1:]...)
	return nil
}

// TAG FILES

// Represents a tag file object in the bag with its related fields.
type TagFile struct {
	name string        // Filepath for tag file.
	Data *TagFieldList // key value pairs of data for the tagfile.
}

/*
 Creates a new tagfile object and returns it or returns an error if improperly formatted.
 The name argument represents the filepath of the tagfile, which must end in txt
*/
func NewTagFile(name string) (tf *TagFile, err error) {
	err = validateTagFileName(name)
	tf = new(TagFile)
	tf.name = filepath.Clean(name)
	tf.Data = new(TagFieldList)
	return tf, err
}

/*
 Reads a tagfile, parsing the contents as tagfile field data and returning the TagFile object.
 name is the filepath to the tag file.  It throws an error if contents cannot be properly parsed.
*/
func ReadTagFile(name string) (*TagFile, []error) {
	var errs []error

	file, err := os.Open(name)
	if err != nil {
		return nil, append(errs, err)
	}
	defer file.Close()

	tf, err := NewTagFile(name)
	if err != nil {
		return nil, append(errs, err)
	}

	data, errs := parseTagFields(file)
	tf.Data.SetFields(data)

	return tf, errs
}

// Returns the named filepath of the tagfile.
func (tf *TagFile) Name() string {
	return tf.name
}

/*
 Creates the named tagfile and writes key value pairs to it, with indented
 formatting as indicated in the BagIt spec.
*/
func (tf *TagFile) Create() error {
	// Create directory if needed.
	if err := os.MkdirAll(filepath.Dir(tf.name), 0777); err != nil {
		return err
	}

	// Create the tagfile.
	fileOut, err := os.Create(tf.Name())
	if err != nil {
		return err
	}
	defer fileOut.Close()

	// Write fields and data to the file.
	for _, f := range tf.Data.Fields() {
		field, err := formatField(f.Label(), f.Value())
		if err != nil {
			return err
		}
		_, err = fmt.Fprintln(fileOut, field)
		if err != nil {
			return err
		}
	}

	return nil
}

/*
Takes a tag field key and data and wraps lines at 79 with indented spaces as
per recommendation in spec.
*/
func formatField(key string, data string) (string, error) {
	delimeter := "\n   "
	var buff bytes.Buffer

	// Initiate it by writing the proper key.
	writeLen, err := buff.WriteString(fmt.Sprintf("%s: ", key))
	if err != nil {
		return "", err
	}
	splitCounter := writeLen

	words := strings.Split(data, " ")

	for word := range words {
		if splitCounter+len(words[word]) > 79 {
			splitCounter, err = buff.WriteString(delimeter)
			if err != nil {
				return "", err
			}
		}
		writeLen, err = buff.WriteString(strings.Join([]string{" ", words[word]}, ""))
		if err != nil {
			return "", err
		}
		splitCounter += writeLen

	}

	return buff.String(), nil
}

// Some private convenence methods for manipulating tag files.
func validateTagFileName(name string) (err error) {
	_, err = os.Stat(filepath.Dir(name))
	re, _ := regexp.Compile(`.*\.txt`)
	if !re.MatchString(filepath.Base(name)) {
		err = errors.New(fmt.Sprint("Tagfiles must end in .txt and contain at least 1 letter.  Provided: ", filepath.Base(name)))
	}
	return err
}

/*
 Reads the contents of file and parses tagfile fields from the contents or returns an error if
 it contains unparsable data.
*/
func parseTagFields(file *os.File) ([]TagField, []error) {
	var errors []error
	re, err := regexp.Compile(`^(\S*\:)?(\s.*)?$`)
	if err != nil {
		errors = append(errors, err)
		return nil, errors
	}

	scanner := bufio.NewScanner(file)
	var fields []TagField
	var field TagField

	// Parse the remaining lines.
	for scanner.Scan() {
		line := scanner.Text()
		// See http://play.golang.org/p/zLqvg2qo1D for some testing on the field match.
		if re.MatchString(line) {
			data := re.FindStringSubmatch(line)
			data[1] = strings.Replace(data[1], ":", "", 1)
			if data[1] != "" {
				fields = append(fields, field)
				field = *NewTagField(data[1], strings.Trim(data[2], " "))
				continue
			}
			value := strings.Trim(data[2], " ")
			field.SetValue(strings.Join([]string{field.Value(), value}, " "))

		} else {
			err := fmt.Errorf("Unable to parse tag data from line: %s", line)
			errors = append(errors, err)
		}
	}
	if scanner.Err() != nil {
		errors = append(errors, scanner.Err())
	}

	// See http://play.golang.org/p/nsw9zsAEPF for some testing on the field match.
	return fields, errors
}
<|endoftext|>


<commit_before>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dbtester

import (
	"github.com/coreos/etcd/clientv3"
	"golang.org/x/net/context"
)

type request struct {
	etcdv2Op etcdv2Op
	etcdv3Op clientv3.Op
	zkOp     zkOp
	consulOp consulOp
}

// ReqHandler wraps request handler.
type ReqHandler func(ctx context.Context, req *request) error
<commit_msg>stress: remove etcdv2Op<commit_after>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dbtester

import (
	"github.com/coreos/etcd/clientv3"
	"golang.org/x/net/context"
)

type request struct {
	etcdv3Op clientv3.Op
	zkOp     zkOp
	consulOp consulOp
}

// ReqHandler wraps request handler.
type ReqHandler func(ctx context.Context, req *request) error
<|endoftext|>


<commit_before>package structs

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
)

// APIMatch : raw data returned from Riot's API. Converted to Match using ToMatch function
type APIMatch struct {
	GameID       RiotID `json:"gameId"`
	SeasonID     int    `json:"seasonId"`
	GameCreation int64  `json:"gameCreation"`
	GameDuration int    `json:"gameDuration"`

	Participants []struct {
		TeamID     int    `json:"teamId"`
		ChampionID RiotID `json:"championId"`
		Stats      struct {
			Win bool `json:"win"`
		} `json:"stats"`
	} `json:"participants"`

	ParticipantIdentities []struct {
		Player struct {
			AccountID    RiotID `json:"accountId"`
			SummonerName string `json:"summonerName"`
			SummonerID   RiotID `json:"summonerId"`
			ProfileIcon  int    `json:"profileIcon"`
		} `json:"player"`
	}

	Teams []struct {
		Bans []struct {
			ChampionID RiotID `json:"championId"`
		} `json:"bans"`
	}

	GameMode string `json:"gameMode"`
	MapID    int    `json:"mapId"`
	GameType string `json:"gameType"`
}

type RiotID int64

func (r RiotID) Bytes() []byte {
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.BigEndian, r)

	return buf.Bytes()
}

type Match struct {
	GameID       RiotID `json:"gameId"`
	SeasonID     int    `json:"seasonId"`
	GameCreation int64  `json:"gameCreation"`
	GameDuration int    `json:"gameDuration"`

	Participants []Participant
	Bans         []RiotID

	GameMode string `json:"gameMode"`
	MapID    int    `json:"mapId"`
	GameType string `json:"gameType"`
}

func (m *Match) Banned(id RiotID) bool {
	for _, ban := range m.Bans {
		if ban == id {
			return true
		}
	}

	return false
}

// Picked : Returns a boolean indicating whether the specified champion played in this game.
func (m *Match) Picked(id RiotID) bool {
	for _, p := range m.Participants {
		if id == p.ChampionID {
			return true
		}
	}

	return false
}

// Won : Returns a boolean indicating whether the specified champion won the game.
func (m *Match) Won(id RiotID) bool {
	for _, p := range m.Participants {
		if id == p.ChampionID && p.Winner {
			return true
		}
	}

	return false
}

func (m Match) Bytes() []byte {
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)

	enc.Encode(m)
	return buf.Bytes()
}

type Participant struct {
	SummonerName string `json:"summonerName"`
	AccountID    RiotID `json:"accountId"`
	ProfileIcon  int    `json:"profileIcon"`
	SummonerID   RiotID `json:"summonerId"`
	ChampionID   RiotID `json:"championId"`
	TeamID       int    `json:"teamId"`

	Winner bool `json:"winner"`
}

// ToMatch : Convert raw API data to a Match object
func ToMatch(raw APIMatch) Match {
	var match Match

	match.GameID = raw.GameID
	match.SeasonID = raw.SeasonID
	match.GameCreation = raw.GameCreation
	match.GameDuration = raw.GameDuration

	match.GameMode = raw.GameMode
	match.MapID = raw.MapID
	match.GameType = raw.GameType

	match.Participants = make([]Participant, len(raw.Participants))

	for i := 0; i < len(raw.Participants); i++ {
		match.Participants[i] = Participant{
			TeamID:     raw.Participants[i].TeamID,
			ChampionID: raw.Participants[i].ChampionID,
			AccountID:  raw.ParticipantIdentities[i].Player.AccountID,
			SummonerID: raw.ParticipantIdentities[i].Player.SummonerID,

			ProfileIcon:  raw.ParticipantIdentities[i].Player.ProfileIcon,
			SummonerName: raw.ParticipantIdentities[i].Player.SummonerName,
			Winner:       raw.Participants[i].Stats.Win,
		}
	}

	match.Bans = make([]RiotID, 0)
	for _, team := range raw.Teams {
		for _, ban := range team.Bans {
			match.Bans = append(match.Bans, ban.ChampionID)
		}
	}

	return match
}
<commit_msg>Added match.When() to convert timestamp to time.Time object.<commit_after>package structs

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"time"
)

// APIMatch : raw data returned from Riot's API. Converted to Match using ToMatch function
type APIMatch struct {
	GameID       RiotID `json:"gameId"`
	SeasonID     int    `json:"seasonId"`
	GameCreation int64  `json:"gameCreation"`
	GameDuration int    `json:"gameDuration"`

	Participants []struct {
		TeamID     int    `json:"teamId"`
		ChampionID RiotID `json:"championId"`
		Stats      struct {
			Win bool `json:"win"`
		} `json:"stats"`
	} `json:"participants"`

	ParticipantIdentities []struct {
		Player struct {
			AccountID    RiotID `json:"accountId"`
			SummonerName string `json:"summonerName"`
			SummonerID   RiotID `json:"summonerId"`
			ProfileIcon  int    `json:"profileIcon"`
		} `json:"player"`
	}

	Teams []struct {
		Bans []struct {
			ChampionID RiotID `json:"championId"`
		} `json:"bans"`
	}

	GameMode string `json:"gameMode"`
	MapID    int    `json:"mapId"`
	GameType string `json:"gameType"`
}

type RiotID int64

func (r RiotID) Bytes() []byte {
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.BigEndian, r)

	return buf.Bytes()
}

type Match struct {
	GameID       RiotID `json:"gameId"`
	SeasonID     int    `json:"seasonId"`
	GameCreation int64  `json:"gameCreation"`
	GameDuration int    `json:"gameDuration"`

	Participants []Participant
	Bans         []RiotID

	GameMode string `json:"gameMode"`
	MapID    int    `json:"mapId"`
	GameType string `json:"gameType"`
}

func (m *Match) When() time.Time {
	return time.Unix(m.GameCreation/1000, 0)
}

func (m *Match) Banned(id RiotID) bool {
	for _, ban := range m.Bans {
		if ban == id {
			return true
		}
	}

	return false
}

// Picked : Returns a boolean indicating whether the specified champion played in this game.
func (m *Match) Picked(id RiotID) bool {
	for _, p := range m.Participants {
		if id == p.ChampionID {
			return true
		}
	}

	return false
}

// Won : Returns a boolean indicating whether the specified champion won the game.
func (m *Match) Won(id RiotID) bool {
	for _, p := range m.Participants {
		if id == p.ChampionID && p.Winner {
			return true
		}
	}

	return false
}

func (m Match) Bytes() []byte {
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)

	enc.Encode(m)
	return buf.Bytes()
}

type Participant struct {
	SummonerName string `json:"summonerName"`
	AccountID    RiotID `json:"accountId"`
	ProfileIcon  int    `json:"profileIcon"`
	SummonerID   RiotID `json:"summonerId"`
	ChampionID   RiotID `json:"championId"`
	TeamID       int    `json:"teamId"`

	Winner bool `json:"winner"`
}

// ToMatch : Convert raw API data to a Match object
func ToMatch(raw APIMatch) Match {
	var match Match

	match.GameID = raw.GameID
	match.SeasonID = raw.SeasonID
	match.GameCreation = raw.GameCreation
	match.GameDuration = raw.GameDuration

	match.GameMode = raw.GameMode
	match.MapID = raw.MapID
	match.GameType = raw.GameType

	match.Participants = make([]Participant, len(raw.Participants))

	for i := 0; i < len(raw.Participants); i++ {
		match.Participants[i] = Participant{
			TeamID:     raw.Participants[i].TeamID,
			ChampionID: raw.Participants[i].ChampionID,
			AccountID:  raw.ParticipantIdentities[i].Player.AccountID,
			SummonerID: raw.ParticipantIdentities[i].Player.SummonerID,

			ProfileIcon:  raw.ParticipantIdentities[i].Player.ProfileIcon,
			SummonerName: raw.ParticipantIdentities[i].Player.SummonerName,
			Winner:       raw.Participants[i].Stats.Win,
		}
	}

	match.Bans = make([]RiotID, 0)
	for _, team := range raw.Teams {
		for _, ban := range team.Bans {
			match.Bans = append(match.Bans, ban.ChampionID)
		}
	}

	return match
}
<|endoftext|>


<commit_before>package eulerian_cycle

func EulerianCycle(g Graph) []int {
	if !g.Directed {
		panic("This algorithm only supports directed graphs")
	}

	// A directed graph contains an Eulerian cycle if and only if:
	// - It is strongly connected;
	// - And each vertex has the same in-degree as out-degree.
	if !stronglyConnected(g) || !sameInDegreesAsOutDegrees(g) {
		return nil
	}

	return findEulerianCycle(g.Edges)
}

func stronglyConnected(g Graph) bool {
	n := 0
	NewAdjacencyList(g).DepthFirstSearch(0, func(x int) {
		n++
	})
	return n == g.VertexCount
}

func sameInDegreesAsOutDegrees(g Graph) bool {
	degrees := make([]int, g.VertexCount)
	for _, edge := range g.Edges {
		degrees[edge.X]--
		degrees[edge.Y]++
	}
	for _, degree := range degrees {
		if degree != 0 {
			return false
		}
	}
	return true
}

func findEulerianCycle(edges []Edge) []int {
	eulerianCycle := []int{}
	visited := make([]bool, len(edges))

	// The visitEdges call below will return a cycle, starting and ending at the vertex 0.
	// Explanation from http://en.wikipedia.org/wiki/Eulerian_path#Hierholzer.27s_algorithm:
	// "It is not possible to get stuck at any vertex other than [0], because the even degree of all
	// vertices ensures that, when the trail enters another vertex w there must be an unused edge
	// leaving w. The tour formed in this way is a closed tour, but may not cover all the vertices
	// and edges of the initial graph."
	path := visitEdges(edges, visited, 0)

	for len(path) > 0 {
		// Move the last edge of the path into the eulerian cycle. This builds the eulerian cycle in
		// reverse order, so to get the correct final order we prepend the edge instead of appending
		// it.
		popped := path[len(path)-1]
		path = path[:len(path)-1]
		eulerianCycle = prepend(eulerianCycle, popped.X)

		// The visitEdges call below will again return a cycle, starting and ending at popped.X.
		// That's because each vertex in the graph of unvisited edges still maintains the Eulerian
		// property of in-degree == out-degree, so the same reasoning as above still applies.
		//
		// Note that if path is not empty, popped.X is equal to path[len(path)-1].Y, so the path
		// will still be a valid continuous path even after being extended by the new cycle.
		path = append(path, visitEdges(edges, visited, popped.X)...)
	}

	return eulerianCycle
}

// visitEdges builds a path of unvisited edges, starting at the vertex start, and marking the edges
// of the path as visited. When the path cannot be extended with unvisited edges anymore, visitEdges
// returns the path.
func visitEdges(edges []Edge, visited []bool, start int) []Edge {
	for i, edge := range edges {
		if !visited[i] && edge.X == start {
			visited[i] = true
			return append([]Edge{edge}, visitEdges(edges, visited, edge.Y)...)
		}
	}
	return nil
}

func prepend(a []int, v int) []int {
	return append([]int{v}, a...)
}
<commit_msg>[eulerian_cycle/go] Update comments for clarity<commit_after>package eulerian_cycle

func EulerianCycle(g Graph) []int {
	if !g.Directed {
		panic("This algorithm only supports directed graphs")
	}

	// A directed graph contains an Eulerian cycle if and only if:
	// - It is strongly connected;
	// - And each vertex has the same in-degree as out-degree.
	if !stronglyConnected(g) || !sameInDegreesAsOutDegrees(g) {
		return nil
	}

	return findEulerianCycle(g.Edges)
}

func stronglyConnected(g Graph) bool {
	n := 0
	NewAdjacencyList(g).DepthFirstSearch(0, func(x int) {
		n++
	})
	return n == g.VertexCount
}

func sameInDegreesAsOutDegrees(g Graph) bool {
	degrees := make([]int, g.VertexCount)
	for _, edge := range g.Edges {
		degrees[edge.X]--
		degrees[edge.Y]++
	}
	for _, degree := range degrees {
		if degree != 0 {
			return false
		}
	}
	return true
}

func findEulerianCycle(edges []Edge) []int {
	eulerianCycle := []int{}
	visited := make([]bool, len(edges))

	// The visitEdges call below will return a cycle, starting and ending at the vertex 0.
	// Explanation from http://en.wikipedia.org/wiki/Eulerian_path#Hierholzer.27s_algorithm:
	// "It is not possible to get stuck at any vertex other than [0], because the even degree of all
	// vertices ensures that, when the trail enters another vertex w there must be an unused edge
	// leaving w. The tour formed in this way is a closed tour, but may not cover all the vertices
	// and edges of the initial graph."
	path := visitEdges(edges, visited, 0)

	for len(path) > 0 {
		// Move the last edge of path into the eulerian cycle. This builds the eulerian cycle in
		// reverse order, so to get the correct final order we need to prepend the edge instead of
		// appending it.
		popped := path[len(path)-1]
		path = path[:len(path)-1]
		eulerianCycle = prepend(eulerianCycle, popped.X)

		// The visitEdges call below will again return a cycle, starting and ending at popped.X.
		// That's because each vertex in the graph of unvisited edges still maintains the Eulerian
		// property of in-degree == out-degree, so the same reasoning as above still applies.
		//
		// Note that if path is not empty, popped.X is equal to path[len(path)-1].Y, so path will
		// still be a valid continuous path even after being extended by the new cycle.
		path = append(path, visitEdges(edges, visited, popped.X)...)
	}

	return eulerianCycle
}

// visitEdges builds a path of unvisited edges, starting at the vertex start, and marking the edges
// of the path as visited. When the path cannot be extended with unvisited edges anymore, visitEdges
// returns the path.
func visitEdges(edges []Edge, visited []bool, start int) []Edge {
	for i, edge := range edges {
		if !visited[i] && edge.X == start {
			visited[i] = true
			return append([]Edge{edge}, visitEdges(edges, visited, edge.Y)...)
		}
	}
	return nil
}

func prepend(a []int, v int) []int {
	return append([]int{v}, a...)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel

import (
	"github.com/pkg/errors"
	"github.com/uber/tchannel-go"
	"github.com/uber/tchannel-go/thrift"
	"golang.org/x/net/context"
)

// client implements TChanClient and makes outgoing Thrift calls.
type Client struct {
	ch          *tchannel.Channel
	sc          *tchannel.SubChannel
	serviceName string
}

// NewClient returns a Client that makes calls over the given tchannel to the given Hyperbahn service.
func NewClient(ch *tchannel.Channel, serviceName string) TChanClient {
	client := &Client{
		ch:          ch,
		sc:          ch.GetSubChannel(serviceName),
		serviceName: serviceName,
	}
	return client
}

func (c *Client) writeArgs(call *tchannel.OutboundCall, headers map[string]string, req RWTStruct) error {
	writer, err := call.Arg2Writer()
	if err != nil {
		return errors.Wrapf(err, "could not create arg2writer for client outbound call %s", c.serviceName)
	}
	headers = tchannel.InjectOutboundSpan(call.Response(), headers)
	if err := WriteHeaders(writer, headers); err != nil {
		return errors.Wrapf(err, "could not write headers for client outbound call %s", c.serviceName)
	}
	if err := writer.Close(); err != nil {
		return errors.Wrapf(err, "could not close arg2 writer for client outbound call %s", c.serviceName)
	}

	writer, err = call.Arg3Writer()
	if err != nil {
		return errors.Wrapf(err, "could not create arg3writer for client outbound call %s", c.serviceName)
	}

	if err := WriteStruct(writer, req); err != nil {
		return errors.Wrapf(err, "could not write request for client outbound call %s", c.serviceName)
	}

	return writer.Close()
}

// readResponse reads the response struct into resp, and returns:
// (response headers, whether there was an application error, unexpected error).
func readResponse(response *tchannel.OutboundCallResponse, resp RWTStruct) (map[string]string, bool, error) {
	reader, err := response.Arg2Reader()
	if err != nil {
		return nil, false, err
	}

	headers, err := ReadHeaders(reader)
	if err != nil {
		return nil, false, err
	}

	if err := EnsureEmpty(reader, "reading response headers"); err != nil {
		return nil, false, err
	}

	if err := reader.Close(); err != nil {
		return nil, false, err
	}

	success := !response.ApplicationError()
	reader, err = response.Arg3Reader()
	if err != nil {
		return headers, success, err
	}

	if err := ReadStruct(reader, resp); err != nil {
		return headers, success, err
	}

	if err := EnsureEmpty(reader, "reading response body"); err != nil {
		return nil, false, err
	}

	return headers, success, reader.Close()
}

func (c *Client) Call(ctx thrift.Context, thriftService, methodName string, req, resp RWTStruct) (bool, error) {
	var (
		headers = ctx.Headers()

		respHeaders map[string]string
		isOK        bool
	)

	err := c.ch.RunWithRetry(ctx, func(ctx context.Context, rs *tchannel.RequestState) error {
		respHeaders, isOK = nil, false

		call, err := c.sc.BeginCall(ctx, thriftService+"::"+methodName, &tchannel.CallOptions{
			Format:       tchannel.Thrift,
			RequestState: rs,
		})
		if err != nil {
			return err
		}

		if err := c.writeArgs(call, headers, req); err != nil {
			return err
		}

		respHeaders, isOK, err = readResponse(call.Response(), resp)
		return err
	})
	if err != nil {
		return false, err
	}

	ctx.SetResponseHeaders(respHeaders)
	return isOK, nil
}
<commit_msg>Concatenate arg1 only once<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel

import (
	"github.com/pkg/errors"
	"github.com/uber/tchannel-go"
	"github.com/uber/tchannel-go/thrift"
	"golang.org/x/net/context"
)

// client implements TChanClient and makes outgoing Thrift calls.
type Client struct {
	ch          *tchannel.Channel
	sc          *tchannel.SubChannel
	serviceName string
}

// NewClient returns a Client that makes calls over the given tchannel to the given Hyperbahn service.
func NewClient(ch *tchannel.Channel, serviceName string) TChanClient {
	client := &Client{
		ch:          ch,
		sc:          ch.GetSubChannel(serviceName),
		serviceName: serviceName,
	}
	return client
}

func (c *Client) writeArgs(call *tchannel.OutboundCall, headers map[string]string, req RWTStruct) error {
	writer, err := call.Arg2Writer()
	if err != nil {
		return errors.Wrapf(err, "could not create arg2writer for client outbound call %s", c.serviceName)
	}
	headers = tchannel.InjectOutboundSpan(call.Response(), headers)
	if err := WriteHeaders(writer, headers); err != nil {
		return errors.Wrapf(err, "could not write headers for client outbound call %s", c.serviceName)
	}
	if err := writer.Close(); err != nil {
		return errors.Wrapf(err, "could not close arg2 writer for client outbound call %s", c.serviceName)
	}

	writer, err = call.Arg3Writer()
	if err != nil {
		return errors.Wrapf(err, "could not create arg3writer for client outbound call %s", c.serviceName)
	}

	if err := WriteStruct(writer, req); err != nil {
		return errors.Wrapf(err, "could not write request for client outbound call %s", c.serviceName)
	}

	return writer.Close()
}

// readResponse reads the response struct into resp, and returns:
// (response headers, whether there was an application error, unexpected error).
func readResponse(response *tchannel.OutboundCallResponse, resp RWTStruct) (map[string]string, bool, error) {
	reader, err := response.Arg2Reader()
	if err != nil {
		return nil, false, err
	}

	headers, err := ReadHeaders(reader)
	if err != nil {
		return nil, false, err
	}

	if err := EnsureEmpty(reader, "reading response headers"); err != nil {
		return nil, false, err
	}

	if err := reader.Close(); err != nil {
		return nil, false, err
	}

	success := !response.ApplicationError()
	reader, err = response.Arg3Reader()
	if err != nil {
		return headers, success, err
	}

	if err := ReadStruct(reader, resp); err != nil {
		return headers, success, err
	}

	if err := EnsureEmpty(reader, "reading response body"); err != nil {
		return nil, false, err
	}

	return headers, success, reader.Close()
}

func (c *Client) Call(ctx thrift.Context, thriftService, methodName string, req, resp RWTStruct) (bool, error) {
	var (
		headers = ctx.Headers()

		respHeaders map[string]string
		isOK        bool
	)

	arg1 := thriftService + "::" + methodName
	err := c.ch.RunWithRetry(ctx, func(ctx context.Context, rs *tchannel.RequestState) error {
		respHeaders, isOK = nil, false

		call, err := c.sc.BeginCall(ctx, arg1, &tchannel.CallOptions{
			Format:       tchannel.Thrift,
			RequestState: rs,
		})
		if err != nil {
			return err
		}

		if err := c.writeArgs(call, headers, req); err != nil {
			return err
		}

		respHeaders, isOK, err = readResponse(call.Response(), resp)
		return err
	})
	if err != nil {
		return false, err
	}

	ctx.SetResponseHeaders(respHeaders)
	return isOK, nil
}
<|endoftext|>