<commit_before>package main

import (
	"fmt"
	"http"
	"os"
	"sort"
	"strconv"
	"strings"
)

const maxEnumerate = 100000

type blobInfo struct {
	*BlobRef
	*os.FileInfo
	*os.Error
}

func readBlobs(ch chan *blobInfo, blobPrefix, diskRoot, after string, remain *uint) {
	dirFullPath := *flagStorageRoot + "/" + diskRoot
	dir, err := os.Open(dirFullPath, os.O_RDONLY, 0)
	if err != nil {
		fmt.Println("Error opening directory: ", err)
		ch <- &blobInfo{Error: &err}
		return
	}
	names, err := dir.Readdirnames(32768)
	if err != nil {
		fmt.Println("Error reading dirnames: ", err)
		ch <- &blobInfo{Error: &err}
		return
	}
	sort.SortStrings(names)
	for _, name := range names {
		if *remain == 0 {
			ch <- &blobInfo{Error: &os.ENOSPC}
			return
		}

		fullPath := dirFullPath + "/" + name
		fi, err := os.Stat(fullPath)
		if err != nil {
			bi := &blobInfo{Error: &err}
			ch <- bi
			return
		}

		if fi.IsDirectory() {
			var newBlobPrefix string
			if blobPrefix == "" {
				newBlobPrefix = name + "-"
			} else {
				newBlobPrefix = blobPrefix + name
			}
			if len(after) > 0 {
				compareLen := len(newBlobPrefix)
				if len(after) < compareLen {
					compareLen = len(after)
				}
				if newBlobPrefix[0:compareLen] < after[0:compareLen] {
					continue
				}
			}
			readBlobs(ch, newBlobPrefix, diskRoot + "/" + name, after, remain)
			continue
		}

		if fi.IsRegular() && strings.HasSuffix(name, ".dat") {
			blobName := name[0:len(name)-4]
			if blobName <= after {
				continue
			}
			blobRef := ParseBlobRef(blobName)
			if blobRef != nil {
				bi := &blobInfo{BlobRef: blobRef, FileInfo: fi}
				ch <- bi
				(*remain)--
			}
			continue
		}
	}

	if diskRoot == "" {
		ch <- nil
	}
}

func handleEnumerateBlobs(conn *http.Conn, req *http.Request) {
	req.ParseForm()
	ch := make(chan *blobInfo, 100)

	limit, err := strconv.Atoui(req.FormValue("limit"))
	if err != nil || limit > maxEnumerate {
		limit = maxEnumerate
	}

	conn.SetHeader("Content-Type", "text/javascript; charset=utf-8")
	fmt.Fprintf(conn, "{\n  \"blobs\": [\n")

	var after string
	go readBlobs(ch, "", "", req.FormValue("after"), &limit);
	for bi := range ch {
		if bi == nil {
			after = ""
			break
		}
		if bi.Error != nil {
			break
		}
		blobName := bi.BlobRef.String()
		fmt.Fprintf(conn, "    {\"blobref\": \"%s\", \"size\": %d},\n",
			blobName, bi.FileInfo.Size)
		after = blobName
	}
	fmt.Fprintf(conn, "  ]")
	if after != "" {
		fmt.Fprintf(conn, ",\n  \"after\": \"%s\"", after)
	}
	fmt.Fprintf(conn, "\n}\n")
}

<commit_msg>fix case in JSON: blobref -> blobRef<commit_after>package main

import (
	"fmt"
	"http"
	"os"
	"sort"
	"strconv"
	"strings"
)

const maxEnumerate = 100000

type blobInfo struct {
	*BlobRef
	*os.FileInfo
	*os.Error
}

func readBlobs(ch chan *blobInfo, blobPrefix, diskRoot, after string, remain *uint) {
	dirFullPath := *flagStorageRoot + "/" + diskRoot
	dir, err := os.Open(dirFullPath, os.O_RDONLY, 0)
	if err != nil {
		fmt.Println("Error opening directory: ", err)
		ch <- &blobInfo{Error: &err}
		return
	}
	names, err := dir.Readdirnames(32768)
	if err != nil {
		fmt.Println("Error reading dirnames: ", err)
		ch <- &blobInfo{Error: &err}
		return
	}
	sort.SortStrings(names)
	for _, name := range names {
		if *remain == 0 {
			ch <- &blobInfo{Error: &os.ENOSPC}
			return
		}

		fullPath := dirFullPath + "/" + name
		fi, err := os.Stat(fullPath)
		if err != nil {
			bi := &blobInfo{Error: &err}
			ch <- bi
			return
		}

		if fi.IsDirectory() {
			var newBlobPrefix string
			if blobPrefix == "" {
				newBlobPrefix = name + "-"
			} else {
				newBlobPrefix = blobPrefix + name
			}
			if len(after) > 0 {
				compareLen := len(newBlobPrefix)
				if len(after) < compareLen {
					compareLen = len(after)
				}
				if newBlobPrefix[0:compareLen] < after[0:compareLen] {
					continue
				}
			}
			readBlobs(ch, newBlobPrefix, diskRoot + "/" + name, after, remain)
			continue
		}

		if fi.IsRegular() && strings.HasSuffix(name, ".dat") {
			blobName := name[0:len(name)-4]
			if blobName <= after {
				continue
			}
			blobRef := ParseBlobRef(blobName)
			if blobRef != nil {
				bi := &blobInfo{BlobRef: blobRef, FileInfo: fi}
				ch <- bi
				(*remain)--
			}
			continue
		}
	}

	if diskRoot == "" {
		ch <- nil
	}
}

func handleEnumerateBlobs(conn *http.Conn, req *http.Request) {
	req.ParseForm()
	ch := make(chan *blobInfo, 100)

	limit, err := strconv.Atoui(req.FormValue("limit"))
	if err != nil || limit > maxEnumerate {
		limit = maxEnumerate
	}

	conn.SetHeader("Content-Type", "text/javascript; charset=utf-8")
	fmt.Fprintf(conn, "{\n  \"blobs\": [\n")

	var after string
	go readBlobs(ch, "", "", req.FormValue("after"), &limit);
	for bi := range ch {
		if bi == nil {
			after = ""
			break
		}
		if bi.Error != nil {
			break
		}
		blobName := bi.BlobRef.String()
		fmt.Fprintf(conn, "    {\"blobRef\": \"%s\", \"size\": %d},\n",
			blobName, bi.FileInfo.Size)
		after = blobName
	}
	fmt.Fprintf(conn, "  ]")
	if after != "" {
		fmt.Fprintf(conn, ",\n  \"after\": \"%s\"", after)
	}
	fmt.Fprintf(conn, "\n}\n")
}

<|endoftext|>


<commit_before>package smtpapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"unicode/utf16"
)

// Version represents the current version of the smtpapi-go library
const Version = "0.6.7"

// SMTPAPIHeader will be used to set up X-SMTPAPI params
type SMTPAPIHeader struct {
	To          []string               `json:"to,omitempty"`
	Sub         map[string][]string    `json:"sub,omitempty"`
	Section     map[string]string      `json:"section,omitempty"`
	Category    []string               `json:"category,omitempty"`
	UniqueArgs  map[string]string      `json:"unique_args,omitempty"`
	Filters     map[string]Filter      `json:"filters,omitempty"`
	ASMGroupID  int                    `json:"asm_group_id,omitempty"`
	ASMGroups   []int                  `json:"asm_groups_to_display,omitempty"`
	SendAt      int64                  `json:"send_at,omitempty"`
	SendEachAt  []int64                `json:"send_each_at,omitempty"`
	IpPool      string                 `json:"ip_pool,omitempty"`
	BatchID     string                 `json:"batch_id,omitempty"`
	DynamicData map[string]interface{} `json:"dynamic_template_data,omitempty"`
}

// Filter represents an App/Filter and its settings
type Filter struct {
	Settings map[string]interface{} `json:"settings,omitempty"`
}

// NewSMTPAPIHeader creates a new header struct
func NewSMTPAPIHeader() *SMTPAPIHeader {
	return &SMTPAPIHeader{}
}

// AddTo appends a single email to the To header
func (h *SMTPAPIHeader) AddTo(email string) {
	h.To = append(h.To, email)
}

// AddTos appends multiple emails to the To header
func (h *SMTPAPIHeader) AddTos(emails []string) {
	for i := 0; i < len(emails); i++ {
		h.AddTo(emails[i])
	}
}

// SetTos sets the value of the To header
func (h *SMTPAPIHeader) SetTos(emails []string) {
	h.To = emails
}

// AddSubstitution adds a new substitution to a specific key
func (h *SMTPAPIHeader) AddSubstitution(key, sub string) {
	if h.Sub == nil {
		h.Sub = make(map[string][]string)
	}
	h.Sub[key] = append(h.Sub[key], sub)
}

// AddSubstitutions adds a multiple substitutions to a specific key
func (h *SMTPAPIHeader) AddSubstitutions(key string, subs []string) {
	for i := 0; i < len(subs); i++ {
		h.AddSubstitution(key, subs[i])
	}
}

// SetSubstitutions sets the value of the substitutions on the Sub header
func (h *SMTPAPIHeader) SetSubstitutions(sub map[string][]string) {
	h.Sub = sub
}

// AddSection sets the value for a specific section
func (h *SMTPAPIHeader) AddSection(section, value string) {
	if h.Section == nil {
		h.Section = make(map[string]string)
	}
	h.Section[section] = value
}

// SetSections sets the value for the Section header
func (h *SMTPAPIHeader) SetSections(sections map[string]string) {
	h.Section = sections
}

// AddCategory adds a new category to the Category header
func (h *SMTPAPIHeader) AddCategory(category string) {
	h.Category = append(h.Category, category)
}

// AddCategories adds multiple categories to the Category header
func (h *SMTPAPIHeader) AddCategories(categories []string) {
	for i := 0; i < len(categories); i++ {
		h.AddCategory(categories[i])
	}
}

// SetCategories will set the value of the Categories field
func (h *SMTPAPIHeader) SetCategories(categories []string) {
	h.Category = categories
}

// SetASMGroupID will set the value of the ASMGroupID field
func (h *SMTPAPIHeader) SetASMGroupID(groupID int) {
	h.ASMGroupID = groupID
}

// AddASMGroupToDisplay adds a new ASM group ID to be displayed
func (h *SMTPAPIHeader) AddASMGroupToDisplay(groupID int) {
	h.ASMGroups = append(h.ASMGroups, groupID)
}

// AddASMGroupsToDisplay adds multiple ASM group IDs to be displayed
func (h *SMTPAPIHeader) AddASMGroupsToDisplay(groupIDs []int) {
	for i := 0; i < len(groupIDs); i++ {
		h.AddASMGroupToDisplay(groupIDs[i])
	}
}

// SetASMGroupsToDisplay will set the value of the ASMGroups field
func (h *SMTPAPIHeader) SetASMGroupsToDisplay(groups []int) {
	h.ASMGroups = groups
}

// AddUniqueArg will set the value of a specific argument
func (h *SMTPAPIHeader) AddUniqueArg(arg, value string) {
	if h.UniqueArgs == nil {
		h.UniqueArgs = make(map[string]string)
	}
	h.UniqueArgs[arg] = value
}

// SetUniqueArgs will set the value of the Unique_args header
func (h *SMTPAPIHeader) SetUniqueArgs(args map[string]string) {
	h.UniqueArgs = args
}

// AddFilter will set the specific setting for a filter
func (h *SMTPAPIHeader) AddFilter(filter, setting string, value interface{}) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	if _, ok := h.Filters[filter]; !ok {
		h.Filters[filter] = Filter{
			Settings: make(map[string]interface{}),
		}
	}
	h.Filters[filter].Settings[setting] = value
}

// SetFilter takes in a Filter struct with predetermined settings and sets it for such Filter key
func (h *SMTPAPIHeader) SetFilter(filter string, value *Filter) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	h.Filters[filter] = *value
}

// SetSendAt takes in a timestamp which determines when the email will be sent
func (h *SMTPAPIHeader) SetSendAt(sendAt int64) {
	h.SendAt = sendAt
}

// AddSendEachAt takes in a timestamp and pushes it into a list Must match length of To emails
func (h *SMTPAPIHeader) AddSendEachAt(sendEachAt int64) {
	h.SendEachAt = append(h.SendEachAt, sendEachAt)
}

// SetSendEachAt takes an array of timestamps. Must match length of To emails
func (h *SMTPAPIHeader) SetSendEachAt(sendEachAt []int64) {
	h.SendEachAt = sendEachAt
}

// SetIpPool takes a strings and sets the IpPool field
func (h *SMTPAPIHeader) SetIpPool(ipPool string) {
	h.IpPool = ipPool
}

// Unicode escape
func escapeUnicode(input string) string {
	//var buffer bytes.Buffer
	buffer := bytes.NewBufferString("")
	for _, r := range input {
		if r > 65535 {
			// surrogate pair
			var r1, r2 = utf16.EncodeRune(r)
			var s = fmt.Sprintf("\\u%x\\u%x", r1, r2)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		} else if r > 127 {
			var s = fmt.Sprintf("\\u%04x", r)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		} else {
			var s = fmt.Sprintf("%c", r)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		}
	}
	return buffer.String()
}

// JSONString returns the representation of the Header
func (h *SMTPAPIHeader) JSONString() (string, error) {
	headers, e := json.Marshal(h)
	return escapeUnicode(string(headers)), e
}

// Load allows you to load a pre-formed x-smtpapi header
func (h *SMTPAPIHeader) Load(b []byte) error {
	return json.Unmarshal(b, h)
}
<commit_msg>Release v0.6.8<commit_after>package smtpapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"unicode/utf16"
)

// Version represents the current version of the smtpapi-go library
const Version = "0.6.8"

// SMTPAPIHeader will be used to set up X-SMTPAPI params
type SMTPAPIHeader struct {
	To          []string               `json:"to,omitempty"`
	Sub         map[string][]string    `json:"sub,omitempty"`
	Section     map[string]string      `json:"section,omitempty"`
	Category    []string               `json:"category,omitempty"`
	UniqueArgs  map[string]string      `json:"unique_args,omitempty"`
	Filters     map[string]Filter      `json:"filters,omitempty"`
	ASMGroupID  int                    `json:"asm_group_id,omitempty"`
	ASMGroups   []int                  `json:"asm_groups_to_display,omitempty"`
	SendAt      int64                  `json:"send_at,omitempty"`
	SendEachAt  []int64                `json:"send_each_at,omitempty"`
	IpPool      string                 `json:"ip_pool,omitempty"`
	BatchID     string                 `json:"batch_id,omitempty"`
	DynamicData map[string]interface{} `json:"dynamic_template_data,omitempty"`
}

// Filter represents an App/Filter and its settings
type Filter struct {
	Settings map[string]interface{} `json:"settings,omitempty"`
}

// NewSMTPAPIHeader creates a new header struct
func NewSMTPAPIHeader() *SMTPAPIHeader {
	return &SMTPAPIHeader{}
}

// AddTo appends a single email to the To header
func (h *SMTPAPIHeader) AddTo(email string) {
	h.To = append(h.To, email)
}

// AddTos appends multiple emails to the To header
func (h *SMTPAPIHeader) AddTos(emails []string) {
	for i := 0; i < len(emails); i++ {
		h.AddTo(emails[i])
	}
}

// SetTos sets the value of the To header
func (h *SMTPAPIHeader) SetTos(emails []string) {
	h.To = emails
}

// AddSubstitution adds a new substitution to a specific key
func (h *SMTPAPIHeader) AddSubstitution(key, sub string) {
	if h.Sub == nil {
		h.Sub = make(map[string][]string)
	}
	h.Sub[key] = append(h.Sub[key], sub)
}

// AddSubstitutions adds a multiple substitutions to a specific key
func (h *SMTPAPIHeader) AddSubstitutions(key string, subs []string) {
	for i := 0; i < len(subs); i++ {
		h.AddSubstitution(key, subs[i])
	}
}

// SetSubstitutions sets the value of the substitutions on the Sub header
func (h *SMTPAPIHeader) SetSubstitutions(sub map[string][]string) {
	h.Sub = sub
}

// AddSection sets the value for a specific section
func (h *SMTPAPIHeader) AddSection(section, value string) {
	if h.Section == nil {
		h.Section = make(map[string]string)
	}
	h.Section[section] = value
}

// SetSections sets the value for the Section header
func (h *SMTPAPIHeader) SetSections(sections map[string]string) {
	h.Section = sections
}

// AddCategory adds a new category to the Category header
func (h *SMTPAPIHeader) AddCategory(category string) {
	h.Category = append(h.Category, category)
}

// AddCategories adds multiple categories to the Category header
func (h *SMTPAPIHeader) AddCategories(categories []string) {
	for i := 0; i < len(categories); i++ {
		h.AddCategory(categories[i])
	}
}

// SetCategories will set the value of the Categories field
func (h *SMTPAPIHeader) SetCategories(categories []string) {
	h.Category = categories
}

// SetASMGroupID will set the value of the ASMGroupID field
func (h *SMTPAPIHeader) SetASMGroupID(groupID int) {
	h.ASMGroupID = groupID
}

// AddASMGroupToDisplay adds a new ASM group ID to be displayed
func (h *SMTPAPIHeader) AddASMGroupToDisplay(groupID int) {
	h.ASMGroups = append(h.ASMGroups, groupID)
}

// AddASMGroupsToDisplay adds multiple ASM group IDs to be displayed
func (h *SMTPAPIHeader) AddASMGroupsToDisplay(groupIDs []int) {
	for i := 0; i < len(groupIDs); i++ {
		h.AddASMGroupToDisplay(groupIDs[i])
	}
}

// SetASMGroupsToDisplay will set the value of the ASMGroups field
func (h *SMTPAPIHeader) SetASMGroupsToDisplay(groups []int) {
	h.ASMGroups = groups
}

// AddUniqueArg will set the value of a specific argument
func (h *SMTPAPIHeader) AddUniqueArg(arg, value string) {
	if h.UniqueArgs == nil {
		h.UniqueArgs = make(map[string]string)
	}
	h.UniqueArgs[arg] = value
}

// SetUniqueArgs will set the value of the Unique_args header
func (h *SMTPAPIHeader) SetUniqueArgs(args map[string]string) {
	h.UniqueArgs = args
}

// AddFilter will set the specific setting for a filter
func (h *SMTPAPIHeader) AddFilter(filter, setting string, value interface{}) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	if _, ok := h.Filters[filter]; !ok {
		h.Filters[filter] = Filter{
			Settings: make(map[string]interface{}),
		}
	}
	h.Filters[filter].Settings[setting] = value
}

// SetFilter takes in a Filter struct with predetermined settings and sets it for such Filter key
func (h *SMTPAPIHeader) SetFilter(filter string, value *Filter) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	h.Filters[filter] = *value
}

// SetSendAt takes in a timestamp which determines when the email will be sent
func (h *SMTPAPIHeader) SetSendAt(sendAt int64) {
	h.SendAt = sendAt
}

// AddSendEachAt takes in a timestamp and pushes it into a list Must match length of To emails
func (h *SMTPAPIHeader) AddSendEachAt(sendEachAt int64) {
	h.SendEachAt = append(h.SendEachAt, sendEachAt)
}

// SetSendEachAt takes an array of timestamps. Must match length of To emails
func (h *SMTPAPIHeader) SetSendEachAt(sendEachAt []int64) {
	h.SendEachAt = sendEachAt
}

// SetIpPool takes a strings and sets the IpPool field
func (h *SMTPAPIHeader) SetIpPool(ipPool string) {
	h.IpPool = ipPool
}

// Unicode escape
func escapeUnicode(input string) string {
	//var buffer bytes.Buffer
	buffer := bytes.NewBufferString("")
	for _, r := range input {
		if r > 65535 {
			// surrogate pair
			var r1, r2 = utf16.EncodeRune(r)
			var s = fmt.Sprintf("\\u%x\\u%x", r1, r2)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		} else if r > 127 {
			var s = fmt.Sprintf("\\u%04x", r)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		} else {
			var s = fmt.Sprintf("%c", r)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		}
	}
	return buffer.String()
}

// JSONString returns the representation of the Header
func (h *SMTPAPIHeader) JSONString() (string, error) {
	headers, e := json.Marshal(h)
	return escapeUnicode(string(headers)), e
}

// Load allows you to load a pre-formed x-smtpapi header
func (h *SMTPAPIHeader) Load(b []byte) error {
	return json.Unmarshal(b, h)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package hex implements hexadecimal encoding and decoding.
package hex

import (
	"bytes"
	"errors"
	"fmt"
	"io"
)

const hextable = "0123456789abcdef"

// EncodedLen returns the length of an encoding of n source bytes.
func EncodedLen(n int) int { return n * 2 }

// Encode encodes src into EncodedLen(len(src))
// bytes of dst.  As a convenience, it returns the number
// of bytes written to dst, but this value is always EncodedLen(len(src)).
// Encode implements hexadecimal encoding.
func Encode(dst, src []byte) int {
	for i, v := range src {
		dst[i*2] = hextable[v>>4]
		dst[i*2+1] = hextable[v&0x0f]
	}

	return len(src) * 2
}

// ErrLength results from decoding an odd length slice.
var ErrLength = errors.New("encoding/hex: odd length hex string")

// InvalidByteError values describe errors resulting from an invalid byte in a hex string.
type InvalidByteError byte

func (e InvalidByteError) Error() string {
	return fmt.Sprintf("encoding/hex: invalid byte: %#U", rune(e))
}

func DecodedLen(x int) int { return x / 2 }

// Decode decodes src into DecodedLen(len(src)) bytes, returning the actual
// number of bytes written to dst.
//
// If Decode encounters invalid input, it returns an error describing the failure.
func Decode(dst, src []byte) (int, error) {
	if len(src)%2 == 1 {
		return 0, ErrLength
	}

	for i := 0; i < len(src)/2; i++ {
		a, ok := fromHexChar(src[i*2])
		if !ok {
			return 0, InvalidByteError(src[i*2])
		}
		b, ok := fromHexChar(src[i*2+1])
		if !ok {
			return 0, InvalidByteError(src[i*2+1])
		}
		dst[i] = (a << 4) | b
	}

	return len(src) / 2, nil
}

// fromHexChar converts a hex character into its value and a success flag.
func fromHexChar(c byte) (byte, bool) {
	switch {
	case '0' <= c && c <= '9':
		return c - '0', true
	case 'a' <= c && c <= 'f':
		return c - 'a' + 10, true
	case 'A' <= c && c <= 'F':
		return c - 'A' + 10, true
	}

	return 0, false
}

// EncodeToString returns the hexadecimal encoding of src.
func EncodeToString(src []byte) string {
	dst := make([]byte, EncodedLen(len(src)))
	Encode(dst, src)
	return string(dst)
}

// DecodeString returns the bytes represented by the hexadecimal string s.
func DecodeString(s string) ([]byte, error) {
	src := []byte(s)
	dst := make([]byte, DecodedLen(len(src)))
	_, err := Decode(dst, src)
	if err != nil {
		return nil, err
	}
	return dst, nil
}

// Dump returns a string that contains a hex dump of the given data. The format
// of the hex dump matches the output of `hexdump -C` on the command line.
func Dump(data []byte) string {
	var buf bytes.Buffer
	dumper := Dumper(&buf)
	dumper.Write(data)
	dumper.Close()
	return string(buf.Bytes())
}

// Dumper returns a WriteCloser that writes a hex dump of all written data to
// w. The format of the dump matches the output of `hexdump -C` on the command
// line.
func Dumper(w io.Writer) io.WriteCloser {
	return &dumper{w: w}
}

type dumper struct {
	w          io.Writer
	rightChars [18]byte
	buf        [14]byte
	used       int  // number of bytes in the current line
	n          uint // number of bytes, total
}

func toChar(b byte) byte {
	if b < 32 || b > 126 {
		return '.'
	}
	return b
}

func (h *dumper) Write(data []byte) (n int, err error) {
	// Output lines look like:
	// 00000010  2e 2f 30 31 32 33 34 35  36 37 38 39 3a 3b 3c 3d  |./0123456789:;<=|
	// ^ offset                          ^ extra space              ^ ASCII of line.
	for i := range data {
		if h.used == 0 {
			// At the beginning of a line we print the current
			// offset in hex.
			h.buf[0] = byte(h.n >> 24)
			h.buf[1] = byte(h.n >> 16)
			h.buf[2] = byte(h.n >> 8)
			h.buf[3] = byte(h.n)
			Encode(h.buf[4:], h.buf[:4])
			h.buf[12] = ' '
			h.buf[13] = ' '
			_, err = h.w.Write(h.buf[4:])
			if err != nil {
				return
			}
		}
		Encode(h.buf[:], data[i:i+1])
		h.buf[2] = ' '
		l := 3
		if h.used == 7 {
			// There's an additional space after the 8th byte.
			h.buf[3] = ' '
			l = 4
		} else if h.used == 15 {
			// At the end of the line there's an extra space and
			// the bar for the right column.
			h.buf[3] = ' '
			h.buf[4] = '|'
			l = 5
		}
		_, err = h.w.Write(h.buf[:l])
		if err != nil {
			return
		}
		n++
		h.rightChars[h.used] = toChar(data[i])
		h.used++
		h.n++
		if h.used == 16 {
			h.rightChars[16] = '|'
			h.rightChars[17] = '\n'
			_, err = h.w.Write(h.rightChars[:])
			if err != nil {
				return
			}
			h.used = 0
		}
	}
	return
}

func (h *dumper) Close() (err error) {
	// See the comments in Write() for the details of this format.
	if h.used == 0 {
		return
	}
	h.buf[0] = ' '
	h.buf[1] = ' '
	h.buf[2] = ' '
	h.buf[3] = ' '
	h.buf[4] = '|'
	nBytes := h.used
	for h.used < 16 {
		l := 3
		if h.used == 7 {
			l = 4
		} else if h.used == 15 {
			l = 5
		}
		_, err = h.w.Write(h.buf[:l])
		if err != nil {
			return
		}
		h.used++
	}
	h.rightChars[nBytes] = '|'
	h.rightChars[nBytes+1] = '\n'
	_, err = h.w.Write(h.rightChars[:nBytes+2])
	return
}
<commit_msg>encoding/hex: minor cleanup<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package hex implements hexadecimal encoding and decoding.
package hex

import (
	"bytes"
	"errors"
	"fmt"
	"io"
)

const hextable = "0123456789abcdef"

// EncodedLen returns the length of an encoding of n source bytes.
func EncodedLen(n int) int { return n * 2 }

// Encode encodes src into EncodedLen(len(src))
// bytes of dst.  As a convenience, it returns the number
// of bytes written to dst, but this value is always EncodedLen(len(src)).
// Encode implements hexadecimal encoding.
func Encode(dst, src []byte) int {
	for i, v := range src {
		dst[i*2] = hextable[v>>4]
		dst[i*2+1] = hextable[v&0x0f]
	}

	return len(src) * 2
}

// ErrLength results from decoding an odd length slice.
var ErrLength = errors.New("encoding/hex: odd length hex string")

// InvalidByteError values describe errors resulting from an invalid byte in a hex string.
type InvalidByteError byte

func (e InvalidByteError) Error() string {
	return fmt.Sprintf("encoding/hex: invalid byte: %#U", rune(e))
}

func DecodedLen(x int) int { return x / 2 }

// Decode decodes src into DecodedLen(len(src)) bytes, returning the actual
// number of bytes written to dst.
//
// If Decode encounters invalid input, it returns an error describing the failure.
func Decode(dst, src []byte) (int, error) {
	if len(src)%2 == 1 {
		return 0, ErrLength
	}

	for i := 0; i < len(src)/2; i++ {
		a, ok := fromHexChar(src[i*2])
		if !ok {
			return 0, InvalidByteError(src[i*2])
		}
		b, ok := fromHexChar(src[i*2+1])
		if !ok {
			return 0, InvalidByteError(src[i*2+1])
		}
		dst[i] = (a << 4) | b
	}

	return len(src) / 2, nil
}

// fromHexChar converts a hex character into its value and a success flag.
func fromHexChar(c byte) (byte, bool) {
	switch {
	case '0' <= c && c <= '9':
		return c - '0', true
	case 'a' <= c && c <= 'f':
		return c - 'a' + 10, true
	case 'A' <= c && c <= 'F':
		return c - 'A' + 10, true
	}

	return 0, false
}

// EncodeToString returns the hexadecimal encoding of src.
func EncodeToString(src []byte) string {
	dst := make([]byte, EncodedLen(len(src)))
	Encode(dst, src)
	return string(dst)
}

// DecodeString returns the bytes represented by the hexadecimal string s.
func DecodeString(s string) ([]byte, error) {
	src := []byte(s)
	dst := make([]byte, DecodedLen(len(src)))
	_, err := Decode(dst, src)
	if err != nil {
		return nil, err
	}
	return dst, nil
}

// Dump returns a string that contains a hex dump of the given data. The format
// of the hex dump matches the output of `hexdump -C` on the command line.
func Dump(data []byte) string {
	var buf bytes.Buffer
	dumper := Dumper(&buf)
	dumper.Write(data)
	dumper.Close()
	return buf.String()
}

// Dumper returns a WriteCloser that writes a hex dump of all written data to
// w. The format of the dump matches the output of `hexdump -C` on the command
// line.
func Dumper(w io.Writer) io.WriteCloser {
	return &dumper{w: w}
}

type dumper struct {
	w          io.Writer
	rightChars [18]byte
	buf        [14]byte
	used       int  // number of bytes in the current line
	n          uint // number of bytes, total
}

func toChar(b byte) byte {
	if b < 32 || b > 126 {
		return '.'
	}
	return b
}

func (h *dumper) Write(data []byte) (n int, err error) {
	// Output lines look like:
	// 00000010  2e 2f 30 31 32 33 34 35  36 37 38 39 3a 3b 3c 3d  |./0123456789:;<=|
	// ^ offset                          ^ extra space              ^ ASCII of line.
	for i := range data {
		if h.used == 0 {
			// At the beginning of a line we print the current
			// offset in hex.
			h.buf[0] = byte(h.n >> 24)
			h.buf[1] = byte(h.n >> 16)
			h.buf[2] = byte(h.n >> 8)
			h.buf[3] = byte(h.n)
			Encode(h.buf[4:], h.buf[:4])
			h.buf[12] = ' '
			h.buf[13] = ' '
			_, err = h.w.Write(h.buf[4:])
			if err != nil {
				return
			}
		}
		Encode(h.buf[:], data[i:i+1])
		h.buf[2] = ' '
		l := 3
		if h.used == 7 {
			// There's an additional space after the 8th byte.
			h.buf[3] = ' '
			l = 4
		} else if h.used == 15 {
			// At the end of the line there's an extra space and
			// the bar for the right column.
			h.buf[3] = ' '
			h.buf[4] = '|'
			l = 5
		}
		_, err = h.w.Write(h.buf[:l])
		if err != nil {
			return
		}
		n++
		h.rightChars[h.used] = toChar(data[i])
		h.used++
		h.n++
		if h.used == 16 {
			h.rightChars[16] = '|'
			h.rightChars[17] = '\n'
			_, err = h.w.Write(h.rightChars[:])
			if err != nil {
				return
			}
			h.used = 0
		}
	}
	return
}

func (h *dumper) Close() (err error) {
	// See the comments in Write() for the details of this format.
	if h.used == 0 {
		return
	}
	h.buf[0] = ' '
	h.buf[1] = ' '
	h.buf[2] = ' '
	h.buf[3] = ' '
	h.buf[4] = '|'
	nBytes := h.used
	for h.used < 16 {
		l := 3
		if h.used == 7 {
			l = 4
		} else if h.used == 15 {
			l = 5
		}
		_, err = h.w.Write(h.buf[:l])
		if err != nil {
			return
		}
		h.used++
	}
	h.rightChars[nBytes] = '|'
	h.rightChars[nBytes+1] = '\n'
	_, err = h.w.Write(h.rightChars[:nBytes+2])
	return
}
<|endoftext|>


<commit_before>package bootstrap

import (
	"context"
	"fmt"
	"os"
	"strconv"

	"github.com/buildkite/agent/v3/agent"
	"github.com/buildkite/agent/v3/env"
	"github.com/buildkite/agent/v3/experiments"
	"github.com/buildkite/agent/v3/tracetools"
	"github.com/opentracing/opentracing-go"
	"go.opentelemetry.io/contrib/propagators/aws/xray"
	"go.opentelemetry.io/contrib/propagators/b3"
	"go.opentelemetry.io/contrib/propagators/jaeger"
	"go.opentelemetry.io/contrib/propagators/ot"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
	"go.opentelemetry.io/otel/propagation"
	"go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
	"go.opentelemetry.io/otel/trace"
	"golang.org/x/exp/slices"
	ddext "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext"
	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer"
	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
)

// stopper lets us abstract the tracer wrap up code so we can plug in different tracing
// library implementations that are opentracing compatible. Opentracing itself
// doesn't have a Stop function on its Tracer interface.
type stopper func()

func noopStopper() {}

func (b *Bootstrap) startTracing(ctx context.Context) (tracetools.Span, context.Context, stopper) {
	switch b.Config.TracingBackend {
	case tracetools.BackendDatadog:
		// Newer versions of the tracing libs print out diagnostic info which spams the
		// Buildkite agent logs. Disable it by default unless it's been explicitly set.
		if _, has := os.LookupEnv("DD_TRACE_STARTUP_LOGS"); !has {
			os.Setenv("DD_TRACE_STARTUP_LOGS", "false")
		}

		return b.startTracingDatadog(ctx)

	case tracetools.BackendOpenTelemetry:
		if !experiments.IsEnabled("opentelemetry-tracing") {
			b.shell.Warningf("You've used the OpenTelemetry tracing backend, but the `opentelemetry-tracing` experiment isn't enabled. No tracing will occur.")
			b.shell.Warningf("To enable OpenTelemetry tracing, use the `opentelemetry` tracing backend, as well as the `opentelemetry-tracing` experiment.")
			return &tracetools.NoopSpan{}, ctx, noopStopper
		}

		return b.startTracingOpenTelemetry(ctx)

	case tracetools.BackendNone:
		return &tracetools.NoopSpan{}, ctx, noopStopper

	default:
		b.shell.Commentf("An invalid tracing backend was provided: %q. Tracing will not occur.", b.Config.TracingBackend)
		b.Config.TracingBackend = tracetools.BackendNone // Ensure that we don't do any tracing after this, some of the stuff in tracetools uses the bootstrap's tracking backend
		return &tracetools.NoopSpan{}, ctx, noopStopper
	}
}

func (b *Bootstrap) tracingResourceName() string {
	label, ok := b.shell.Env.Get("BUILDKITE_LABEL")
	if !ok {
		label = "job"
	}

	return b.OrganizationSlug + "/" + b.PipelineSlug + "/" + label
}

// startTracingDatadog sets up tracing based on the config values. It uses opentracing as an
// abstraction so the agent can support multiple libraries if needbe.
func (b *Bootstrap) startTracingDatadog(ctx context.Context) (tracetools.Span, context.Context, stopper) {
	opts := []tracer.StartOption{
		tracer.WithServiceName("buildkite_agent"),
		tracer.WithSampler(tracer.NewAllSampler()),
		tracer.WithAnalytics(true),
	}

	tags := Merge(GenericTracingExtras(b, b.shell.Env), DDTracingExtras())
	opts = slices.Grow(opts, len(tags))
	for k, v := range tags {
		opts = append(opts, tracer.WithGlobalTag(k, v))
	}

	opentracing.SetGlobalTracer(opentracer.New(opts...))

	wireContext := b.extractDDTraceCtx()

	span := opentracing.StartSpan("job.run",
		opentracing.ChildOf(wireContext),
		opentracing.Tag{Key: ddext.ResourceName, Value: b.tracingResourceName()},
	)
	ctx = opentracing.ContextWithSpan(ctx, span)

	return tracetools.NewOpenTracingSpan(span), ctx, tracer.Stop
}

// extractTraceCtx pulls encoded distributed tracing information from the env vars.
// Note: This should match the injectTraceCtx code in shell.
func (b *Bootstrap) extractDDTraceCtx() opentracing.SpanContext {
	sctx, err := tracetools.DecodeTraceContext(b.shell.Env)
	if err != nil {
		// Return nil so a new span will be created
		return nil
	}
	return sctx
}

func (b *Bootstrap) startTracingOpenTelemetry(ctx context.Context) (tracetools.Span, context.Context, stopper) {
	client := otlptracegrpc.NewClient()
	exporter, err := otlptrace.New(ctx, client)
	if err != nil {
		b.shell.Errorf("Error creating OTLP trace exporter %s. Disabling tracing.", err)
		return &tracetools.NoopSpan{}, ctx, noopStopper
	}

	attributes := []attribute.KeyValue{
		semconv.ServiceNameKey.String("buildkite_agent"),
		semconv.ServiceVersionKey.String(agent.Version()),
		semconv.DeploymentEnvironmentKey.String("ci"),
	}

	extras, warnings := toOpenTelemetryAttributes(GenericTracingExtras(b, b.shell.Env))
	for k, v := range warnings {
		b.shell.Warningf("Unknown attribute type (key: %v, value: %v (%T)) passed when initialising OpenTelemetry. This is a bug, submit this error message at https://github.com/buildkite/agent/issues", k, v, v)
		b.shell.Warningf("OpenTelemetry will still work, but the attribute %v and its value above will not be included", v)
	}

	attributes = append(attributes, extras...)

	resources := resource.NewWithAttributes(semconv.SchemaURL, attributes...)
	tracerProvider := sdktrace.NewTracerProvider(
		sdktrace.WithBatcher(exporter),
		sdktrace.WithResource(resources),
	)

	otel.SetTracerProvider(tracerProvider)
	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
		propagation.TraceContext{},
		propagation.Baggage{},
		b3.New(),
		&jaeger.Jaeger{},
		&ot.OT{},
		&xray.Propagator{},
	))

	tracer := tracerProvider.Tracer(
		"buildkite_agent",
		trace.WithInstrumentationVersion(agent.Version()),
		trace.WithSchemaURL(semconv.SchemaURL),
	)

	ctx, span := tracer.Start(ctx, "job.run")

	span.SetAttributes(
		attribute.String("resource.name", b.tracingResourceName()),
		attribute.String("analytics.event", "true"),
	)

	stop := func() {
		ctx := context.Background()
		_ = tracerProvider.ForceFlush(ctx)
		_ = tracerProvider.Shutdown(ctx)
	}

	return tracetools.NewOpenTelemetrySpan(span), ctx, stop
}

func GenericTracingExtras(b *Bootstrap, env env.Environment) map[string]any {
	buildID, _ := env.Get("BUILDKITE_BUILD_ID")
	buildNumber, _ := env.Get("BUILDKITE_BUILD_NUMBER")
	buildURL, _ := env.Get("BUILDKITE_BUILD_URL")
	jobURL := fmt.Sprintf("%s#%s", buildURL, b.JobID)
	source, _ := env.Get("BUILDKITE_SOURCE")

	retry := 0
	if attemptStr, has := env.Get("BUILDKITE_RETRY_COUNT"); has {
		if parsedRetry, err := strconv.Atoi(attemptStr); err == nil {
			retry = parsedRetry
		}
	}

	parallel := 0
	if parallelStr, has := env.Get("BUILDKITE_PARALLEL_JOB"); has {
		if parsedParallel, err := strconv.Atoi(parallelStr); err == nil {
			parallel = parsedParallel
		}
	}

	rebuiltFromID, has := env.Get("BUILDKITE_REBUILT_FROM_BUILD_NUMBER")
	if !has || rebuiltFromID == "" {
		rebuiltFromID = "n/a"
	}

	triggeredFromID, has := env.Get("BUILDKITE_TRIGGERED_FROM_BUILD_ID")
	if !has || triggeredFromID == "" {
		triggeredFromID = "n/a"
	}

	return map[string]any{
		"buildkite.agent":             b.AgentName,
		"buildkite.version":           agent.Version(),
		"buildkite.queue":             b.Queue,
		"buildkite.org":               b.OrganizationSlug,
		"buildkite.pipeline":          b.PipelineSlug,
		"buildkite.branch":            b.Branch,
		"buildkite.job_id":            b.JobID,
		"buildkite.job_url":           jobURL,
		"buildkite.build_id":          buildID,
		"buildkite.build_number":      buildNumber,
		"buildkite.build_url":         buildURL,
		"buildkite.source":            source,
		"buildkite.retry":             retry,
		"buildkite.parallel":          parallel,
		"buildkite.rebuilt_from_id":   rebuiltFromID,
		"buildkite.triggered_from_id": triggeredFromID,
	}
}

func DDTracingExtras() map[string]any {
	return map[string]any{
		ddext.AnalyticsEvent:   true,
		ddext.SamplingPriority: ddext.PriorityUserKeep,
	}
}

func Merge(maps ...map[string]any) map[string]any {
	fullCap := 0
	for _, m := range maps {
		fullCap += len(m)
	}

	merged := make(map[string]any, fullCap)
	for _, m := range maps {
		for key, val := range m {
			merged[key] = val
		}
	}

	return merged
}

func toOpenTelemetryAttributes(extras map[string]any) ([]attribute.KeyValue, map[string]any) {
	attrs := make([]attribute.KeyValue, 0, len(extras))
	unknownAttrTypes := make(map[string]any, len(extras))
	for k, v := range extras {
		switch v := v.(type) {
		case string:
			attrs = append(attrs, attribute.String(k, v))
		case int:
			attrs = append(attrs, attribute.Int(k, v))
		case bool:
			attrs = append(attrs, attribute.Bool(k, v))
		default:
			unknownAttrTypes[k] = v
		}
	}

	return attrs, unknownAttrTypes
}
<commit_msg>Name OpenTel root span after the job being run<commit_after>package bootstrap

import (
	"context"
	"fmt"
	"os"
	"strconv"

	"github.com/buildkite/agent/v3/agent"
	"github.com/buildkite/agent/v3/env"
	"github.com/buildkite/agent/v3/experiments"
	"github.com/buildkite/agent/v3/tracetools"
	"github.com/opentracing/opentracing-go"
	"go.opentelemetry.io/contrib/propagators/aws/xray"
	"go.opentelemetry.io/contrib/propagators/b3"
	"go.opentelemetry.io/contrib/propagators/jaeger"
	"go.opentelemetry.io/contrib/propagators/ot"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
	"go.opentelemetry.io/otel/propagation"
	"go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
	"go.opentelemetry.io/otel/trace"
	"golang.org/x/exp/slices"
	ddext "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext"
	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer"
	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
)

// stopper lets us abstract the tracer wrap up code so we can plug in different tracing
// library implementations that are opentracing compatible. Opentracing itself
// doesn't have a Stop function on its Tracer interface.
type stopper func()

func noopStopper() {}

func (b *Bootstrap) startTracing(ctx context.Context) (tracetools.Span, context.Context, stopper) {
	switch b.Config.TracingBackend {
	case tracetools.BackendDatadog:
		// Newer versions of the tracing libs print out diagnostic info which spams the
		// Buildkite agent logs. Disable it by default unless it's been explicitly set.
		if _, has := os.LookupEnv("DD_TRACE_STARTUP_LOGS"); !has {
			os.Setenv("DD_TRACE_STARTUP_LOGS", "false")
		}

		return b.startTracingDatadog(ctx)

	case tracetools.BackendOpenTelemetry:
		if !experiments.IsEnabled("opentelemetry-tracing") {
			b.shell.Warningf("You've used the OpenTelemetry tracing backend, but the `opentelemetry-tracing` experiment isn't enabled. No tracing will occur.")
			b.shell.Warningf("To enable OpenTelemetry tracing, use the `opentelemetry` tracing backend, as well as the `opentelemetry-tracing` experiment.")
			return &tracetools.NoopSpan{}, ctx, noopStopper
		}

		return b.startTracingOpenTelemetry(ctx)

	case tracetools.BackendNone:
		return &tracetools.NoopSpan{}, ctx, noopStopper

	default:
		b.shell.Commentf("An invalid tracing backend was provided: %q. Tracing will not occur.", b.Config.TracingBackend)
		b.Config.TracingBackend = tracetools.BackendNone // Ensure that we don't do any tracing after this, some of the stuff in tracetools uses the bootstrap's tracking backend
		return &tracetools.NoopSpan{}, ctx, noopStopper
	}
}

func (b *Bootstrap) ddResourceName() string {
	label, ok := b.shell.Env.Get("BUILDKITE_LABEL")
	if !ok {
		label = "job"
	}

	return b.OrganizationSlug + "/" + b.PipelineSlug + "/" + label
}

// startTracingDatadog sets up tracing based on the config values. It uses opentracing as an
// abstraction so the agent can support multiple libraries if needbe.
func (b *Bootstrap) startTracingDatadog(ctx context.Context) (tracetools.Span, context.Context, stopper) {
	opts := []tracer.StartOption{
		tracer.WithServiceName("buildkite_agent"),
		tracer.WithSampler(tracer.NewAllSampler()),
		tracer.WithAnalytics(true),
	}

	tags := Merge(GenericTracingExtras(b, b.shell.Env), DDTracingExtras())
	opts = slices.Grow(opts, len(tags))
	for k, v := range tags {
		opts = append(opts, tracer.WithGlobalTag(k, v))
	}

	opentracing.SetGlobalTracer(opentracer.New(opts...))

	wireContext := b.extractDDTraceCtx()

	span := opentracing.StartSpan("job.run",
		opentracing.ChildOf(wireContext),
		opentracing.Tag{Key: ddext.ResourceName, Value: b.ddResourceName()},
	)
	ctx = opentracing.ContextWithSpan(ctx, span)

	return tracetools.NewOpenTracingSpan(span), ctx, tracer.Stop
}

// extractTraceCtx pulls encoded distributed tracing information from the env vars.
// Note: This should match the injectTraceCtx code in shell.
func (b *Bootstrap) extractDDTraceCtx() opentracing.SpanContext {
	sctx, err := tracetools.DecodeTraceContext(b.shell.Env)
	if err != nil {
		// Return nil so a new span will be created
		return nil
	}
	return sctx
}

func (b *Bootstrap) otRootSpanName() string {
	base := b.OrganizationSlug + "/" + b.PipelineSlug + "/"
	key, ok := b.shell.Env.Get("BUILDKITE_STEP_KEY")
	if ok && key != "" {
		return base + key
	}

	label, ok := b.shell.Env.Get("BUILDKITE_LABEL")
	if ok && label != "" {
		return base + label
	}

	return base + "job"
}

func (b *Bootstrap) startTracingOpenTelemetry(ctx context.Context) (tracetools.Span, context.Context, stopper) {
	client := otlptracegrpc.NewClient()
	exporter, err := otlptrace.New(ctx, client)
	if err != nil {
		b.shell.Errorf("Error creating OTLP trace exporter %s. Disabling tracing.", err)
		return &tracetools.NoopSpan{}, ctx, noopStopper
	}

	attributes := []attribute.KeyValue{
		semconv.ServiceNameKey.String("buildkite_agent"),
		semconv.ServiceVersionKey.String(agent.Version()),
		semconv.DeploymentEnvironmentKey.String("ci"),
	}

	extras, warnings := toOpenTelemetryAttributes(GenericTracingExtras(b, b.shell.Env))
	for k, v := range warnings {
		b.shell.Warningf("Unknown attribute type (key: %v, value: %v (%T)) passed when initialising OpenTelemetry. This is a bug, submit this error message at https://github.com/buildkite/agent/issues", k, v, v)
		b.shell.Warningf("OpenTelemetry will still work, but the attribute %v and its value above will not be included", v)
	}

	attributes = append(attributes, extras...)

	resources := resource.NewWithAttributes(semconv.SchemaURL, attributes...)
	tracerProvider := sdktrace.NewTracerProvider(
		sdktrace.WithBatcher(exporter),
		sdktrace.WithResource(resources),
	)

	otel.SetTracerProvider(tracerProvider)
	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
		propagation.TraceContext{},
		propagation.Baggage{},
		b3.New(),
		&jaeger.Jaeger{},
		&ot.OT{},
		&xray.Propagator{},
	))

	tracer := tracerProvider.Tracer(
		"buildkite_agent",
		trace.WithInstrumentationVersion(agent.Version()),
		trace.WithSchemaURL(semconv.SchemaURL),
	)

	ctx, span := tracer.Start(ctx, b.otRootSpanName(),
		trace.WithAttributes(
			attribute.String("analytics.event", "true"),
		),
	)

	stop := func() {
		ctx := context.Background()
		_ = tracerProvider.ForceFlush(ctx)
		_ = tracerProvider.Shutdown(ctx)
	}

	return tracetools.NewOpenTelemetrySpan(span), ctx, stop
}

func GenericTracingExtras(b *Bootstrap, env env.Environment) map[string]any {
	buildID, _ := env.Get("BUILDKITE_BUILD_ID")
	buildNumber, _ := env.Get("BUILDKITE_BUILD_NUMBER")
	buildURL, _ := env.Get("BUILDKITE_BUILD_URL")
	jobURL := fmt.Sprintf("%s#%s", buildURL, b.JobID)
	source, _ := env.Get("BUILDKITE_SOURCE")

	retry := 0
	if attemptStr, has := env.Get("BUILDKITE_RETRY_COUNT"); has {
		if parsedRetry, err := strconv.Atoi(attemptStr); err == nil {
			retry = parsedRetry
		}
	}

	parallel := 0
	if parallelStr, has := env.Get("BUILDKITE_PARALLEL_JOB"); has {
		if parsedParallel, err := strconv.Atoi(parallelStr); err == nil {
			parallel = parsedParallel
		}
	}

	rebuiltFromID, has := env.Get("BUILDKITE_REBUILT_FROM_BUILD_NUMBER")
	if !has || rebuiltFromID == "" {
		rebuiltFromID = "n/a"
	}

	triggeredFromID, has := env.Get("BUILDKITE_TRIGGERED_FROM_BUILD_ID")
	if !has || triggeredFromID == "" {
		triggeredFromID = "n/a"
	}

	return map[string]any{
		"buildkite.agent":             b.AgentName,
		"buildkite.version":           agent.Version(),
		"buildkite.queue":             b.Queue,
		"buildkite.org":               b.OrganizationSlug,
		"buildkite.pipeline":          b.PipelineSlug,
		"buildkite.branch":            b.Branch,
		"buildkite.job_id":            b.JobID,
		"buildkite.job_url":           jobURL,
		"buildkite.build_id":          buildID,
		"buildkite.build_number":      buildNumber,
		"buildkite.build_url":         buildURL,
		"buildkite.source":            source,
		"buildkite.retry":             retry,
		"buildkite.parallel":          parallel,
		"buildkite.rebuilt_from_id":   rebuiltFromID,
		"buildkite.triggered_from_id": triggeredFromID,
	}
}

func DDTracingExtras() map[string]any {
	return map[string]any{
		ddext.AnalyticsEvent:   true,
		ddext.SamplingPriority: ddext.PriorityUserKeep,
	}
}

func Merge(maps ...map[string]any) map[string]any {
	fullCap := 0
	for _, m := range maps {
		fullCap += len(m)
	}

	merged := make(map[string]any, fullCap)
	for _, m := range maps {
		for key, val := range m {
			merged[key] = val
		}
	}

	return merged
}

func toOpenTelemetryAttributes(extras map[string]any) ([]attribute.KeyValue, map[string]any) {
	attrs := make([]attribute.KeyValue, 0, len(extras))
	unknownAttrTypes := make(map[string]any, len(extras))
	for k, v := range extras {
		switch v := v.(type) {
		case string:
			attrs = append(attrs, attribute.String(k, v))
		case int:
			attrs = append(attrs, attribute.Int(k, v))
		case bool:
			attrs = append(attrs, attribute.Bool(k, v))
		default:
			unknownAttrTypes[k] = v
		}
	}

	return attrs, unknownAttrTypes
}
<|endoftext|>


<commit_before>package cl11

import (
	"fmt"
	clw "github.com/rdwilliamson/clw11"
	"strings"
	"unsafe"
)

type Device struct {
	ID                       DeviceID
	Available                bool
	CompilerAvailable        bool
	LittleEndian             bool
	ErrorCorrectionSupport   bool
	ImageSupport             bool
	UnifiedHostMemory        bool
	AddressBits              uint32
	GlobalMemCachelineSize   uint32
	MaxClockFrequency        uint32
	MaxComputeUnits          uint32
	MaxConstantArgs          uint32
	MaxReadImageArgs         uint32
	MaxSamplers              uint32
	MaxWorkItemDimensions    uint32
	MaxWriteImageArgs        uint32
	MemBaseAddrAlign         uint32
	MinDataTypeAlignSize     uint32
	VendorID                 uint32
	PreferredVectorWidths    VectorWidths
	NativeVectorWidths       VectorWidths
	Extensions               string
	Name                     string
	Profile                  string
	Vendor                   string
	Version                  string
	OpenclCVersion           string
	DriverVersion            string
	GlobalMemCacheSize       uint64
	GlobalMemSize            uint64
	LocalMemSize             uint64
	MaxConstantBufferSize    uint64
	MaxMemAllocSize          uint64
	Image2dMaxHeight         uint
	Image2dMaxWidth          uint
	Image3dMaxDepth          uint
	Image3dMaxHeight         uint
	Image3dMaxWidth          uint
	MaxParameterSize         uint
	MaxWorkGroupSize         uint
	ProfilingTimerResolution uint

	MaxWorkItemSizes []uint

	SingleFpConfig FPConfig
	DoubleFpConfig FPConfig
}

type DeviceID clw.DeviceID

// Bitfield.
const (
	FPDenorm = iota
	FPFma
	FPInfNan
	FPRoundToInf
	FPRoundToNearest
	FPRoundToZero
	FPBits
)

type VectorWidths struct {
	Char   uint8
	Short  uint8
	Int    uint8
	Long   uint8
	Float  uint8
	Double uint8
	Half   uint8
}

type FPConfig uint8

func (v FPConfig) String() string {
	var configStrings []string
	if v&FPDenorm != 0 {
		configStrings = append(configStrings, "CL_FP_DENORM")
	}
	if v&FPFma != 0 {
		configStrings = append(configStrings, "CL_FP_FMA")
	}
	if v&FPInfNan != 0 {
		configStrings = append(configStrings, "CL_FP_INF_NAN")
	}
	if v&FPRoundToInf != 0 {
		configStrings = append(configStrings, "CL_FP_ROUND_TO_INF")
	}
	if v&FPRoundToNearest != 0 {
		configStrings = append(configStrings, "CL_FP_ROUND_TO_NEAREST")
	}
	if v&FPRoundToZero != 0 {
		configStrings = append(configStrings, "CL_FP_ROUND_TO_ZERO")
	}
	return "(" + strings.Join(configStrings, "|") + ")"
}

type MemCache uint8

type LocalMem uint8

type ExecCapabilities uint8

func (p *Platform) GetDevices() ([]Device, error) {

	if p.Devices != nil {
		return p.Devices, nil
	}

	var numEntries clw.Uint
	err := clw.GetDeviceIDs(clw.PlatformID(p.ID), clw.DeviceTypeAll, 0, nil, &numEntries)
	if err != nil {
		return nil, err
	}

	deviceIDs := make([]clw.DeviceID, numEntries)
	err = clw.GetDeviceIDs(clw.PlatformID(p.ID), clw.DeviceTypeAll, numEntries, &deviceIDs[0], nil)
	if err != nil {
		return nil, err
	}

	p.Devices = make([]Device, len(deviceIDs))
	for i := range p.Devices {

		p.Devices[i].ID = DeviceID(deviceIDs[i])

		err = p.Devices[i].getAllInfo()
		if err != nil {
			return nil, err
		}
	}

	return p.Devices, nil
}

func (d *Device) getAllInfo() (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	d.Available = d.getBool(clw.DeviceAvailable)
	d.CompilerAvailable = d.getBool(clw.DeviceCompilerAvailable)
	d.LittleEndian = d.getBool(clw.DeviceEndianLittle)
	d.ErrorCorrectionSupport = d.getBool(clw.DeviceErrorCorrectionSupport)
	d.ImageSupport = d.getBool(clw.DeviceImageSupport)
	d.UnifiedHostMemory = d.getBool(clw.DeviceHostUnifiedMemory)

	d.AddressBits = d.getUint(clw.DeviceAddressBits)
	d.GlobalMemCachelineSize = d.getUint(clw.DeviceGlobalMemCachelineSize)
	d.MaxClockFrequency = d.getUint(clw.DeviceMaxClockFrequency)
	d.MaxComputeUnits = d.getUint(clw.DeviceMaxComputeUnits)
	d.MaxConstantArgs = d.getUint(clw.DeviceMaxConstantArgs)
	d.MaxReadImageArgs = d.getUint(clw.DeviceMaxReadImageArgs)
	d.MaxSamplers = d.getUint(clw.DeviceMaxSamplers)
	d.MaxWorkItemDimensions = d.getUint(clw.DeviceMaxWorkItemDimensions)
	d.MaxWriteImageArgs = d.getUint(clw.DeviceMaxWriteImageArgs)
	d.MemBaseAddrAlign = d.getUint(clw.DeviceMemBaseAddrAlign)
	d.MinDataTypeAlignSize = d.getUint(clw.DeviceMinDataTypeAlignSize)
	d.VendorID = d.getUint(clw.DeviceVendorID)

	d.PreferredVectorWidths.Char = uint8(d.getUint(clw.DevicePreferredVectorWidthChar))
	d.PreferredVectorWidths.Short = uint8(d.getUint(clw.DevicePreferredVectorWidthShort))
	d.PreferredVectorWidths.Int = uint8(d.getUint(clw.DevicePreferredVectorWidthInt))
	d.PreferredVectorWidths.Long = uint8(d.getUint(clw.DevicePreferredVectorWidthLong))
	d.PreferredVectorWidths.Float = uint8(d.getUint(clw.DevicePreferredVectorWidthFloat))
	d.PreferredVectorWidths.Double = uint8(d.getUint(clw.DevicePreferredVectorWidthDouble))
	d.PreferredVectorWidths.Half = uint8(d.getUint(clw.DevicePreferredVectorWidthHalf))
	d.NativeVectorWidths.Char = uint8(d.getUint(clw.DeviceNativeVectorWidthChar))
	d.NativeVectorWidths.Short = uint8(d.getUint(clw.DeviceNativeVectorWidthShort))
	d.NativeVectorWidths.Int = uint8(d.getUint(clw.DeviceNativeVectorWidthInt))
	d.NativeVectorWidths.Long = uint8(d.getUint(clw.DeviceNativeVectorWidthLong))
	d.NativeVectorWidths.Float = uint8(d.getUint(clw.DeviceNativeVectorWidthFloat))
	d.NativeVectorWidths.Double = uint8(d.getUint(clw.DeviceNativeVectorWidthDouble))
	d.NativeVectorWidths.Half = uint8(d.getUint(clw.DeviceNativeVectorWidthHalf))

	d.Extensions = d.getString(clw.DeviceExtensions)
	d.Name = d.getString(clw.DeviceName)
	d.Profile = d.getString(clw.DeviceProfile)
	d.Vendor = d.getString(clw.DeviceVendor)
	d.Version = d.getString(clw.DeviceVersion)
	d.OpenclCVersion = d.getString(clw.DeviceOpenclCVersion)
	d.DriverVersion = d.getString(clw.DriverVersion)

	d.GlobalMemCacheSize = d.getUlong(clw.DeviceGlobalMemCacheSize)
	d.GlobalMemSize = d.getUlong(clw.DeviceGlobalMemSize)
	d.LocalMemSize = d.getUlong(clw.DeviceLocalMemSize)
	d.MaxConstantBufferSize = d.getUlong(clw.DeviceMaxConstantBufferSize)
	d.MaxMemAllocSize = d.getUlong(clw.DeviceMaxMemAllocSize)

	d.Image2dMaxHeight = d.getSize(clw.DeviceImage2dMaxHeight)
	d.Image2dMaxWidth = d.getSize(clw.DeviceImage2dMaxWidth)
	d.Image3dMaxDepth = d.getSize(clw.DeviceImage3dMaxDepth)
	d.Image3dMaxHeight = d.getSize(clw.DeviceImage3dMaxHeight)
	d.Image3dMaxWidth = d.getSize(clw.DeviceImage3dMaxWidth)
	d.MaxParameterSize = d.getSize(clw.DeviceMaxParameterSize)
	d.MaxWorkGroupSize = d.getSize(clw.DeviceMaxWorkGroupSize)
	d.ProfilingTimerResolution = d.getSize(clw.DeviceProfilingTimerResolution)

	d.MaxWorkItemSizes = d.getSizeArray(clw.DeviceMaxWorkItemSizes)

	d.SingleFpConfig = d.getFpConfig(clw.DeviceSingleFpConfig)
	d.DoubleFpConfig = d.getFpConfig(clw.DeviceDoubleFpConfig)

	return
}

func (d *Device) getInfo(paramName clw.DeviceInfo) (interface{}, error) {

	switch paramName {

	// fp_config
	case clw.DeviceSingleFpConfig,
		clw.DeviceDoubleFpConfig:

	// exec_capabilities
	case clw.DeviceExecutionCapabilities:

	// mem_cache_type
	case clw.DeviceGlobalMemCacheType:

	// size_t
	case clw.DeviceImage2dMaxHeight,
		clw.DeviceImage2dMaxWidth,
		clw.DeviceImage3dMaxDepth,
		clw.DeviceImage3dMaxHeight,
		clw.DeviceImage3dMaxWidth,
		clw.DeviceMaxParameterSize,
		clw.DeviceMaxWorkGroupSize,
		clw.DeviceMaxWorkItemSizes,
		clw.DeviceProfilingTimerResolution:

	// device_type
	case clw.DeviceTypeInfo:

	// command_queue_properties
	case clw.DeviceQueueProperties:

	// local_mem_type
	case clw.DeviceLocalMemTypeInfo:

	// platform_id
	case clw.DevicePlatform:
	}

	return nil, nil
}

func (d *Device) getBool(paramName clw.DeviceInfo) bool {
	var paramValue clw.Bool
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, clw.Size(unsafe.Sizeof(paramValue)),
		unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return clw.ToGoBool(paramValue)
}

func (d *Device) getUint(paramName clw.DeviceInfo) uint32 {
	var paramValue clw.Uint
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, clw.Size(unsafe.Sizeof(paramValue)),
		unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint32(paramValue)
}

func (d *Device) getString(paramName clw.DeviceInfo) string {
	var paramValueSize clw.Size
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, 0, nil, &paramValueSize)
	if err != nil {
		panic(err)
	}

	buffer := make([]byte, paramValueSize)
	err = clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, paramValueSize, unsafe.Pointer(&buffer[0]), nil)
	if err != nil {
		panic(err)
	}

	// Trim space and trailing \0.
	return strings.TrimSpace(string(buffer[:len(buffer)-1]))
}

func (d *Device) getUlong(paramName clw.DeviceInfo) uint64 {
	var paramValue clw.Ulong
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, clw.Size(unsafe.Sizeof(paramValue)),
		unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint64(paramValue)
}

func (d *Device) getSize(paramName clw.DeviceInfo) uint {
	var paramValue clw.Size
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, clw.Size(unsafe.Sizeof(paramValue)),
		unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint(paramValue)
}

func (d *Device) getSizeArray(paramName clw.DeviceInfo) []uint {
	var paramValueSize clw.Size
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, 0, nil, &paramValueSize)
	if err != nil {
		panic(err)
	}

	var a clw.Size
	buffer := make([]clw.Size, paramValueSize/clw.Size(unsafe.Sizeof(a)))
	fmt.Println(paramValueSize, len(buffer))
	err = clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, paramValueSize, unsafe.Pointer(&buffer[0]), nil)
	if err != nil {
		panic(err)
	}

	results := make([]uint, len(buffer))
	for i := range results {
		results[i] = uint(buffer[i])
	}

	return results
}

func (d *Device) getFpConfig(paramName clw.DeviceInfo) FPConfig {
	var paramValue clw.DeviceFPConfig
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, clw.Size(unsafe.Sizeof(paramValue)),
		unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}

	var result FPConfig
	if paramValue&clw.FPDenorm != 0 {
		result |= FPDenorm
	}
	if paramValue&clw.FPFma != 0 {
		result |= FPFma
	}
	if paramValue&clw.FPInfNan != 0 {
		result |= FPInfNan
	}
	if paramValue&clw.FPRoundToInf != 0 {
		result |= FPRoundToInf
	}
	if paramValue&clw.FPRoundToNearest != 0 {
		result |= FPRoundToNearest
	}
	if paramValue&clw.FPRoundToZero != 0 {
		result |= FPRoundToZero
	}
	return result
}
<commit_msg>Removed converted items.<commit_after>package cl11

import (
	"fmt"
	clw "github.com/rdwilliamson/clw11"
	"strings"
	"unsafe"
)

type Device struct {
	ID                       DeviceID
	Available                bool
	CompilerAvailable        bool
	LittleEndian             bool
	ErrorCorrectionSupport   bool
	ImageSupport             bool
	UnifiedHostMemory        bool
	AddressBits              uint32
	GlobalMemCachelineSize   uint32
	MaxClockFrequency        uint32
	MaxComputeUnits          uint32
	MaxConstantArgs          uint32
	MaxReadImageArgs         uint32
	MaxSamplers              uint32
	MaxWorkItemDimensions    uint32
	MaxWriteImageArgs        uint32
	MemBaseAddrAlign         uint32
	MinDataTypeAlignSize     uint32
	VendorID                 uint32
	PreferredVectorWidths    VectorWidths
	NativeVectorWidths       VectorWidths
	Extensions               string
	Name                     string
	Profile                  string
	Vendor                   string
	Version                  string
	OpenclCVersion           string
	DriverVersion            string
	GlobalMemCacheSize       uint64
	GlobalMemSize            uint64
	LocalMemSize             uint64
	MaxConstantBufferSize    uint64
	MaxMemAllocSize          uint64
	Image2dMaxHeight         uint
	Image2dMaxWidth          uint
	Image3dMaxDepth          uint
	Image3dMaxHeight         uint
	Image3dMaxWidth          uint
	MaxParameterSize         uint
	MaxWorkGroupSize         uint
	ProfilingTimerResolution uint

	MaxWorkItemSizes []uint

	SingleFpConfig FPConfig
	DoubleFpConfig FPConfig
}

type DeviceID clw.DeviceID

// Bitfield.
const (
	FPDenorm = iota
	FPFma
	FPInfNan
	FPRoundToInf
	FPRoundToNearest
	FPRoundToZero
	FPBits
)

type VectorWidths struct {
	Char   uint8
	Short  uint8
	Int    uint8
	Long   uint8
	Float  uint8
	Double uint8
	Half   uint8
}

type FPConfig uint8

func (v FPConfig) String() string {
	var configStrings []string
	if v&FPDenorm != 0 {
		configStrings = append(configStrings, "CL_FP_DENORM")
	}
	if v&FPFma != 0 {
		configStrings = append(configStrings, "CL_FP_FMA")
	}
	if v&FPInfNan != 0 {
		configStrings = append(configStrings, "CL_FP_INF_NAN")
	}
	if v&FPRoundToInf != 0 {
		configStrings = append(configStrings, "CL_FP_ROUND_TO_INF")
	}
	if v&FPRoundToNearest != 0 {
		configStrings = append(configStrings, "CL_FP_ROUND_TO_NEAREST")
	}
	if v&FPRoundToZero != 0 {
		configStrings = append(configStrings, "CL_FP_ROUND_TO_ZERO")
	}
	return "(" + strings.Join(configStrings, "|") + ")"
}

type MemCache uint8

type LocalMem uint8

type ExecCapabilities uint8

func (p *Platform) GetDevices() ([]Device, error) {

	if p.Devices != nil {
		return p.Devices, nil
	}

	var numEntries clw.Uint
	err := clw.GetDeviceIDs(clw.PlatformID(p.ID), clw.DeviceTypeAll, 0, nil, &numEntries)
	if err != nil {
		return nil, err
	}

	deviceIDs := make([]clw.DeviceID, numEntries)
	err = clw.GetDeviceIDs(clw.PlatformID(p.ID), clw.DeviceTypeAll, numEntries, &deviceIDs[0], nil)
	if err != nil {
		return nil, err
	}

	p.Devices = make([]Device, len(deviceIDs))
	for i := range p.Devices {

		p.Devices[i].ID = DeviceID(deviceIDs[i])

		err = p.Devices[i].getAllInfo()
		if err != nil {
			return nil, err
		}
	}

	return p.Devices, nil
}

func (d *Device) getAllInfo() (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	d.Available = d.getBool(clw.DeviceAvailable)
	d.CompilerAvailable = d.getBool(clw.DeviceCompilerAvailable)
	d.LittleEndian = d.getBool(clw.DeviceEndianLittle)
	d.ErrorCorrectionSupport = d.getBool(clw.DeviceErrorCorrectionSupport)
	d.ImageSupport = d.getBool(clw.DeviceImageSupport)
	d.UnifiedHostMemory = d.getBool(clw.DeviceHostUnifiedMemory)

	d.AddressBits = d.getUint(clw.DeviceAddressBits)
	d.GlobalMemCachelineSize = d.getUint(clw.DeviceGlobalMemCachelineSize)
	d.MaxClockFrequency = d.getUint(clw.DeviceMaxClockFrequency)
	d.MaxComputeUnits = d.getUint(clw.DeviceMaxComputeUnits)
	d.MaxConstantArgs = d.getUint(clw.DeviceMaxConstantArgs)
	d.MaxReadImageArgs = d.getUint(clw.DeviceMaxReadImageArgs)
	d.MaxSamplers = d.getUint(clw.DeviceMaxSamplers)
	d.MaxWorkItemDimensions = d.getUint(clw.DeviceMaxWorkItemDimensions)
	d.MaxWriteImageArgs = d.getUint(clw.DeviceMaxWriteImageArgs)
	d.MemBaseAddrAlign = d.getUint(clw.DeviceMemBaseAddrAlign)
	d.MinDataTypeAlignSize = d.getUint(clw.DeviceMinDataTypeAlignSize)
	d.VendorID = d.getUint(clw.DeviceVendorID)

	d.PreferredVectorWidths.Char = uint8(d.getUint(clw.DevicePreferredVectorWidthChar))
	d.PreferredVectorWidths.Short = uint8(d.getUint(clw.DevicePreferredVectorWidthShort))
	d.PreferredVectorWidths.Int = uint8(d.getUint(clw.DevicePreferredVectorWidthInt))
	d.PreferredVectorWidths.Long = uint8(d.getUint(clw.DevicePreferredVectorWidthLong))
	d.PreferredVectorWidths.Float = uint8(d.getUint(clw.DevicePreferredVectorWidthFloat))
	d.PreferredVectorWidths.Double = uint8(d.getUint(clw.DevicePreferredVectorWidthDouble))
	d.PreferredVectorWidths.Half = uint8(d.getUint(clw.DevicePreferredVectorWidthHalf))
	d.NativeVectorWidths.Char = uint8(d.getUint(clw.DeviceNativeVectorWidthChar))
	d.NativeVectorWidths.Short = uint8(d.getUint(clw.DeviceNativeVectorWidthShort))
	d.NativeVectorWidths.Int = uint8(d.getUint(clw.DeviceNativeVectorWidthInt))
	d.NativeVectorWidths.Long = uint8(d.getUint(clw.DeviceNativeVectorWidthLong))
	d.NativeVectorWidths.Float = uint8(d.getUint(clw.DeviceNativeVectorWidthFloat))
	d.NativeVectorWidths.Double = uint8(d.getUint(clw.DeviceNativeVectorWidthDouble))
	d.NativeVectorWidths.Half = uint8(d.getUint(clw.DeviceNativeVectorWidthHalf))

	d.Extensions = d.getString(clw.DeviceExtensions)
	d.Name = d.getString(clw.DeviceName)
	d.Profile = d.getString(clw.DeviceProfile)
	d.Vendor = d.getString(clw.DeviceVendor)
	d.Version = d.getString(clw.DeviceVersion)
	d.OpenclCVersion = d.getString(clw.DeviceOpenclCVersion)
	d.DriverVersion = d.getString(clw.DriverVersion)

	d.GlobalMemCacheSize = d.getUlong(clw.DeviceGlobalMemCacheSize)
	d.GlobalMemSize = d.getUlong(clw.DeviceGlobalMemSize)
	d.LocalMemSize = d.getUlong(clw.DeviceLocalMemSize)
	d.MaxConstantBufferSize = d.getUlong(clw.DeviceMaxConstantBufferSize)
	d.MaxMemAllocSize = d.getUlong(clw.DeviceMaxMemAllocSize)

	d.Image2dMaxHeight = d.getSize(clw.DeviceImage2dMaxHeight)
	d.Image2dMaxWidth = d.getSize(clw.DeviceImage2dMaxWidth)
	d.Image3dMaxDepth = d.getSize(clw.DeviceImage3dMaxDepth)
	d.Image3dMaxHeight = d.getSize(clw.DeviceImage3dMaxHeight)
	d.Image3dMaxWidth = d.getSize(clw.DeviceImage3dMaxWidth)
	d.MaxParameterSize = d.getSize(clw.DeviceMaxParameterSize)
	d.MaxWorkGroupSize = d.getSize(clw.DeviceMaxWorkGroupSize)
	d.ProfilingTimerResolution = d.getSize(clw.DeviceProfilingTimerResolution)

	d.MaxWorkItemSizes = d.getSizeArray(clw.DeviceMaxWorkItemSizes)

	d.SingleFpConfig = d.getFpConfig(clw.DeviceSingleFpConfig)
	d.DoubleFpConfig = d.getFpConfig(clw.DeviceDoubleFpConfig)

	return
}

func (d *Device) getInfo(paramName clw.DeviceInfo) (interface{}, error) {

	switch paramName {

	// exec_capabilities
	case clw.DeviceExecutionCapabilities:

	// mem_cache_type
	case clw.DeviceGlobalMemCacheType:

	// device_type
	case clw.DeviceTypeInfo:

	// command_queue_properties
	case clw.DeviceQueueProperties:

	// local_mem_type
	case clw.DeviceLocalMemTypeInfo:
	}

	return nil, nil
}

func (d *Device) getBool(paramName clw.DeviceInfo) bool {
	var paramValue clw.Bool
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, clw.Size(unsafe.Sizeof(paramValue)),
		unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return clw.ToGoBool(paramValue)
}

func (d *Device) getUint(paramName clw.DeviceInfo) uint32 {
	var paramValue clw.Uint
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, clw.Size(unsafe.Sizeof(paramValue)),
		unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint32(paramValue)
}

func (d *Device) getString(paramName clw.DeviceInfo) string {
	var paramValueSize clw.Size
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, 0, nil, &paramValueSize)
	if err != nil {
		panic(err)
	}

	buffer := make([]byte, paramValueSize)
	err = clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, paramValueSize, unsafe.Pointer(&buffer[0]), nil)
	if err != nil {
		panic(err)
	}

	// Trim space and trailing \0.
	return strings.TrimSpace(string(buffer[:len(buffer)-1]))
}

func (d *Device) getUlong(paramName clw.DeviceInfo) uint64 {
	var paramValue clw.Ulong
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, clw.Size(unsafe.Sizeof(paramValue)),
		unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint64(paramValue)
}

func (d *Device) getSize(paramName clw.DeviceInfo) uint {
	var paramValue clw.Size
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, clw.Size(unsafe.Sizeof(paramValue)),
		unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint(paramValue)
}

func (d *Device) getSizeArray(paramName clw.DeviceInfo) []uint {
	var paramValueSize clw.Size
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, 0, nil, &paramValueSize)
	if err != nil {
		panic(err)
	}

	var a clw.Size
	buffer := make([]clw.Size, paramValueSize/clw.Size(unsafe.Sizeof(a)))
	fmt.Println(paramValueSize, len(buffer))
	err = clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, paramValueSize, unsafe.Pointer(&buffer[0]), nil)
	if err != nil {
		panic(err)
	}

	results := make([]uint, len(buffer))
	for i := range results {
		results[i] = uint(buffer[i])
	}

	return results
}

func (d *Device) getFpConfig(paramName clw.DeviceInfo) FPConfig {
	var paramValue clw.DeviceFPConfig
	err := clw.GetDeviceInfo(clw.DeviceID(d.ID), paramName, clw.Size(unsafe.Sizeof(paramValue)),
		unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}

	var result FPConfig
	if paramValue&clw.FPDenorm != 0 {
		result |= FPDenorm
	}
	if paramValue&clw.FPFma != 0 {
		result |= FPFma
	}
	if paramValue&clw.FPInfNan != 0 {
		result |= FPInfNan
	}
	if paramValue&clw.FPRoundToInf != 0 {
		result |= FPRoundToInf
	}
	if paramValue&clw.FPRoundToNearest != 0 {
		result |= FPRoundToNearest
	}
	if paramValue&clw.FPRoundToZero != 0 {
		result |= FPRoundToZero
	}
	return result
}
<|endoftext|>


<commit_before>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package compatibility

import (
	"fmt"
	"os"

	"github.com/docker/compose/v2/cmd/compose"
)

func getBoolFlags() []string {
	return []string{
		"--debug", "-D",
		"--verbose",
		"--tls",
		"--tlsverify",
	}
}

func getStringFlags() []string {
	return []string{
		"--tlscacert",
		"--tlscert",
		"--tlskey",
		"--host", "-H",
		"--context",
		"--log-level",
	}
}

var argToActualArgument = map[string]string{
	"--verbose": "--debug",
	// docker cli has deprecated -h to avoid ambiguity with -H, while docker-compose still support it
	"-h": "--help",
	// redirect --version pseudo-command to actual command
	"--version": "version",
	"-v":        "version",
}

// Convert transforms standalone docker-compose args into CLI plugin compliant ones
func Convert(args []string) []string {
	var rootFlags []string
	command := []string{compose.PluginName}
	l := len(args)
	for i := 0; i < l; i++ {
		arg := args[i]
		if arg[0] != '-' {
			// not a top-level flag anymore, keep the rest of the command unmodified
			if arg == compose.PluginName {
				i++
			}
			command = append(command, args[i:]...)
			break
		}
		if actualArgument, ok := argToActualArgument[arg]; ok {
			arg = actualArgument
		}
		if contains(getBoolFlags(), arg) {
			rootFlags = append(rootFlags, arg)
			continue
		}
		if contains(getStringFlags(), arg) {
			i++
			if i >= l {
				fmt.Fprintf(os.Stderr, "flag needs an argument: '%s'\n", arg)
				os.Exit(1)
			}
			rootFlags = append(rootFlags, arg, args[i])
			continue
		}
		command = append(command, arg)
	}
	return append(rootFlags, command...)
}

func contains(array []string, needle string) bool {
	for _, val := range array {
		if val == needle {
			return true
		}
	}
	return false
}
<commit_msg>Use switch/case instead of static map for simplicity<commit_after>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package compatibility

import (
	"fmt"
	"os"

	"github.com/docker/compose/v2/cmd/compose"
)

func getBoolFlags() []string {
	return []string{
		"--debug", "-D",
		"--verbose",
		"--tls",
		"--tlsverify",
	}
}

func getStringFlags() []string {
	return []string{
		"--tlscacert",
		"--tlscert",
		"--tlskey",
		"--host", "-H",
		"--context",
		"--log-level",
	}
}

// Convert transforms standalone docker-compose args into CLI plugin compliant ones
func Convert(args []string) []string {
	var rootFlags []string
	command := []string{compose.PluginName}
	l := len(args)
	for i := 0; i < l; i++ {
		arg := args[i]
		if arg[0] != '-' {
			// not a top-level flag anymore, keep the rest of the command unmodified
			if arg == compose.PluginName {
				i++
			}
			command = append(command, args[i:]...)
			break
		}

		switch arg {
		case "--verbose":
			arg = "--debug"
		case "-h":
			// docker cli has deprecated -h to avoid ambiguity with -H, while docker-compose still support it
			arg = "--help"
		case "--version", "-v":
			// redirect --version pseudo-command to actual command
			arg = "version"
		}

		if contains(getBoolFlags(), arg) {
			rootFlags = append(rootFlags, arg)
			continue
		}
		if contains(getStringFlags(), arg) {
			i++
			if i >= l {
				fmt.Fprintf(os.Stderr, "flag needs an argument: '%s'\n", arg)
				os.Exit(1)
			}
			rootFlags = append(rootFlags, arg, args[i])
			continue
		}
		command = append(command, arg)
	}
	return append(rootFlags, command...)
}

func contains(array []string, needle string) bool {
	for _, val := range array {
		if val == needle {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package crossmodel

import (
	"fmt"
	"strings"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/names"
	"launchpad.net/gnuflag"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cmd/envcmd"
	"github.com/juju/juju/crossmodel"
)

const (
	offerCommandDoc = `
A vendor offers a service endpoints for use by consumers.

Example:
$ juju offer db2:db
$ juju offer db2:db local:db2
$ juju offer -e prod db2:db,log remote:/u/ibm/hosted-db2
$ juju offer hosted-db2:db,log remote:/u/ibm/hosted-db2 --to public
`
	offerCommandAgs = `
<service-name>:<endpoint-name>[,...] [<endpoint-url>] [--to <user-ident>,...]
where

endpoint-url    For local endpoints:
                local:/u/<username>/<envname>/<servicename>

                    $ juju offer db2:db

                endpoint “db” available at local:/u/user-name/env-name/hosted-db2

                For remote endpoints:
                remote:/u/<username>/<servicename>

                    $ juju offer db2:db remote:/u/ibm/hosted-db2

                endpoint “db” available at remote:/u/ibm/hosted-db2
`
)

// NewOfferCommand constructs commands that enables endpoints for export.
func NewOfferCommand() cmd.Command {
	return envcmd.Wrap(&offerCommand{})
}

type offerCommand struct {
	CrossModelCommandBase
	api OfferAPI

	// Service stores service tag.
	Service string

	// Endpoints stores a list of endpoints that are being offered.
	Endpoints []string

	// URL stores juju location where these endpoints are offered from.
	URL string

	// Users stores a list of users that these endpoints are offered to.
	Users []string
}

// Info implements Command.Info.
func (c *offerCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "offer",
		Purpose: "offer service endpoints for consumption",
		Args:    offerCommandAgs,
		Doc:     offerCommandDoc,
	}
}

// Init implements Command.Init.
func (c *offerCommand) Init(args []string) error {
	if len(args) < 1 {
		return fmt.Errorf("an offer must at least specify service endpoint")
	}
	if len(args) > 2 {
		return fmt.Errorf("an offer can only specify service endpoints and url")
	}

	if err := c.parseEndpoints(args[0]); err != nil {
		return err
	}

	if len(args) == 2 {
		hostedURL := args[1]
		if !crossmodel.IsValidURL(hostedURL) {
			return errors.Errorf(`hosted url %q is not valid" `, hostedURL)
		}
		c.URL = hostedURL
	}

	return nil
}

// SetFlags implements Command.SetFlags.
func (c *offerCommand) SetFlags(f *gnuflag.FlagSet) {
	c.CrossModelCommandBase.SetFlags(f)
	f.Var(cmd.NewStringsValue(nil, &c.Users), "to", "users that these endpoints are offered to")
}

// Run implements Command.Run.
func (c *offerCommand) Run(_ *cmd.Context) error {
	if c.api == nil {
		api, err := getOfferAPI(c)
		if err != nil {
			return err
		}
		defer api.Close()
		c.api = api
	}

	userTags := make([]string, len(c.Users))
	for i, user := range c.Users {
		if !names.IsValidUser(user) {
			return errors.Errorf(`user name %q is not valid`, user)
		}
		userTags[i] = names.NewUserTag(user).String()
	}

	results, err := c.api.Offer(c.Service, c.Endpoints, c.URL, userTags)
	if err != nil {
		return err
	}

	// Only display feedback if there were errors
	var errs params.ErrorResults
	for _, result := range results {
		if result.Error != nil {
			errs.Results = append(errs.Results, params.ErrorResult{result.Error})
			continue
		}
	}
	if len(errs.Results) > 0 {
		return errs.Combine()
	}
	return nil
}

// OfferAPI defines the API methods that the offer command uses.
type OfferAPI interface {
	Close() error
	Offer(service string, endpoints []string, url string, users []string) ([]params.CrossModelOfferResult, error)
}

var getOfferAPI = (*offerCommand).getOfferAPI

func (c *offerCommand) getOfferAPI() (OfferAPI, error) {
	return c.NewCrossModelAPI()
}

func (c *offerCommand) parseEndpoints(arg string) error {
	parts := strings.SplitN(arg, ":", -1)

	if len(parts) != 2 {
		return errors.New(`endpoints must conform to format "<service-name>:<endpoint-name>[,...]" `)
	}

	serviceName := parts[0]
	if !names.IsValidService(serviceName) {
		return errors.Errorf(`service name %q is not valid`, serviceName)
	}
	c.Service = names.NewServiceTag(serviceName).String()

	endpoints := strings.SplitN(parts[1], ",", -1)
	if len(endpoints) < 1 || endpoints[0] == "" {
		return errors.Errorf(`specify endpoints for %v" `, serviceName)
	}

	c.Endpoints = endpoints
	return nil
}
<commit_msg>Swapped remote: for vendor:<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package crossmodel

import (
	"fmt"
	"strings"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/names"
	"launchpad.net/gnuflag"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cmd/envcmd"
	"github.com/juju/juju/crossmodel"
)

const (
	offerCommandDoc = `
A vendor offers a service endpoints for use by consumers.

Example:
$ juju offer db2:db
$ juju offer db2:db local:db2
$ juju offer -e prod db2:db,log vendor:/u/ibm/hosted-db2
$ juju offer hosted-db2:db,log vendor:/u/ibm/hosted-db2 --to public
`
	offerCommandAgs = `
<service-name>:<endpoint-name>[,...] [<endpoint-url>] [--to <user-ident>,...]
where

endpoint-url    For local endpoints:
                local:/u/<username>/<envname>/<servicename>

                    $ juju offer db2:db

                endpoint “db” available at local:/u/user-name/env-name/hosted-db2

                For vendor endpoints:
                vendor:/u/<username>/<servicename>

                    $ juju offer db2:db vendor:/u/ibm/hosted-db2

                endpoint “db” available at vendor:/u/ibm/hosted-db2
`
)

// NewOfferCommand constructs commands that enables endpoints for export.
func NewOfferCommand() cmd.Command {
	return envcmd.Wrap(&offerCommand{})
}

type offerCommand struct {
	CrossModelCommandBase
	api OfferAPI

	// Service stores service tag.
	Service string

	// Endpoints stores a list of endpoints that are being offered.
	Endpoints []string

	// URL stores juju location where these endpoints are offered from.
	URL string

	// Users stores a list of users that these endpoints are offered to.
	Users []string
}

// Info implements Command.Info.
func (c *offerCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "offer",
		Purpose: "offer service endpoints for consumption",
		Args:    offerCommandAgs,
		Doc:     offerCommandDoc,
	}
}

// Init implements Command.Init.
func (c *offerCommand) Init(args []string) error {
	if len(args) < 1 {
		return fmt.Errorf("an offer must at least specify service endpoint")
	}
	if len(args) > 2 {
		return fmt.Errorf("an offer can only specify service endpoints and url")
	}

	if err := c.parseEndpoints(args[0]); err != nil {
		return err
	}

	if len(args) == 2 {
		hostedURL := args[1]
		if !crossmodel.IsValidURL(hostedURL) {
			return errors.Errorf(`hosted url %q is not valid" `, hostedURL)
		}
		c.URL = hostedURL
	}

	return nil
}

// SetFlags implements Command.SetFlags.
func (c *offerCommand) SetFlags(f *gnuflag.FlagSet) {
	c.CrossModelCommandBase.SetFlags(f)
	f.Var(cmd.NewStringsValue(nil, &c.Users), "to", "users that these endpoints are offered to")
}

// Run implements Command.Run.
func (c *offerCommand) Run(_ *cmd.Context) error {
	if c.api == nil {
		api, err := getOfferAPI(c)
		if err != nil {
			return err
		}
		defer api.Close()
		c.api = api
	}

	userTags := make([]string, len(c.Users))
	for i, user := range c.Users {
		if !names.IsValidUser(user) {
			return errors.Errorf(`user name %q is not valid`, user)
		}
		userTags[i] = names.NewUserTag(user).String()
	}

	results, err := c.api.Offer(c.Service, c.Endpoints, c.URL, userTags)
	if err != nil {
		return err
	}

	// Only display feedback if there were errors
	var errs params.ErrorResults
	for _, result := range results {
		if result.Error != nil {
			errs.Results = append(errs.Results, params.ErrorResult{result.Error})
			continue
		}
	}
	if len(errs.Results) > 0 {
		return errs.Combine()
	}
	return nil
}

// OfferAPI defines the API methods that the offer command uses.
type OfferAPI interface {
	Close() error
	Offer(service string, endpoints []string, url string, users []string) ([]params.CrossModelOfferResult, error)
}

var getOfferAPI = (*offerCommand).getOfferAPI

func (c *offerCommand) getOfferAPI() (OfferAPI, error) {
	return c.NewCrossModelAPI()
}

func (c *offerCommand) parseEndpoints(arg string) error {
	parts := strings.SplitN(arg, ":", -1)

	if len(parts) != 2 {
		return errors.New(`endpoints must conform to format "<service-name>:<endpoint-name>[,...]" `)
	}

	serviceName := parts[0]
	if !names.IsValidService(serviceName) {
		return errors.Errorf(`service name %q is not valid`, serviceName)
	}
	c.Service = names.NewServiceTag(serviceName).String()

	endpoints := strings.SplitN(parts[1], ",", -1)
	if len(endpoints) < 1 || endpoints[0] == "" {
		return errors.Errorf(`specify endpoints for %v" `, serviceName)
	}

	c.Endpoints = endpoints
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/grafana/metrictank/stats"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	otLog "github.com/opentracing/opentracing-go/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/raintank/tsdb-gw/api/models"
	"github.com/raintank/tsdb-gw/auth"
	"github.com/raintank/tsdb-gw/ingest"
	log "github.com/sirupsen/logrus"
	"gopkg.in/macaron.v1"
)

var (
	requestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "tsdb_gw",
		Name:      "request_duration_seconds",
		Help:      "Time (in seconds) spent serving HTTP requests.",
		// github.com/weaveworks/common/instrument/instrument.DefBuckets
		// Pulling it in brings a ton of dependencies.
		Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100},
	}, []string{"method", "route", "status_code"})
)

func init() {
	prometheus.MustRegister(requestDuration)
}

type TracingResponseWriter struct {
	macaron.ResponseWriter
	errBody []byte // the body in case it is an error
}

func (rw *TracingResponseWriter) Write(b []byte) (int, error) {
	if rw.ResponseWriter.Status() >= 400 {
		rw.errBody = make([]byte, len(b))
		copy(rw.errBody, b)
	}
	return rw.ResponseWriter.Write(b)
}

func (rw *TracingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hijacker, ok := rw.ResponseWriter.(http.Hijacker)
	if !ok {
		return nil, nil, fmt.Errorf("the ResponseWriter doesn't support the Hijacker interface")
	}
	return hijacker.Hijack()
}

func GetContextHandler() macaron.Handler {
	return func(c *macaron.Context) {
		ctx := &models.Context{
			Context: c,
			User:    &auth.User{},
		}
		c.Map(ctx)
	}
}

func RequireAdmin() macaron.Handler {
	return func(ctx *models.Context) {
		if !ctx.IsAdmin {
			log.Infof("HTTP auth: admin required but user %d is not admin -> 403", ctx.ID)
			ctx.JSON(403, "Permision denied")
		}
	}
}

func RequirePublisher() macaron.Handler {
	return func(ctx *models.Context) {
		if !ctx.Role.IsPublisher() {
			log.Infof("HTTP auth: user %v with role %v attempting to publish at %v -> 401", ctx.ID, ctx.Role, ctx.Req.RequestURI)
			ctx.JSON(401, "Unauthorized to publish")
			return
		}
	}
}

func RequireViewer() macaron.Handler {
	return func(ctx *models.Context) {
		if !ctx.Role.IsViewer() {
			log.Infof("HTTP auth: user %v with role %v attempting to view at %v -> 403", ctx.ID, ctx.Role, ctx.Req.RequestURI)
			ctx.JSON(403, "Unauthorized to view")
			return
		}
	}
}

func IngestRateLimiter() macaron.Handler {
	return func(ctx *models.Context) {
		if !ingest.IsRateBudgetAvailable(ctx.Req.Context(), ctx.ID) {
			log.Infof("HTTP ratelimiter: Rejecting request for %d due to rate limit -> 429", ctx.ID)
			ctx.JSON(http.StatusTooManyRequests, "Rate limit is exhausted")
			return
		}
	}
}

func (a *Api) GenerateHandlers(kind string, enforceRoles, datadog, rateLimit bool, handlers ...macaron.Handler) []macaron.Handler {
	combinedHandlers := []macaron.Handler{}
	if kind == "write" {
		if datadog {
			combinedHandlers = append(combinedHandlers, a.DDAuth())
		} else {
			combinedHandlers = append(combinedHandlers, a.Auth())
		}
		if enforceRoles {
			combinedHandlers = append(combinedHandlers, RequirePublisher())
		}
	} else {
		combinedHandlers = append(combinedHandlers, a.Auth())
		if enforceRoles {
			combinedHandlers = append(combinedHandlers, RequirePublisher())
		}
	}

	if ingest.UseRateLimit() && rateLimit {
		combinedHandlers = append(combinedHandlers, IngestRateLimiter())
	}

	return append(combinedHandlers, handlers...)
}

func getAuthCreds(req *http.Request) (user, password string) {
	username, key, ok := req.BasicAuth()
	if !ok {
		// no basicAuth, but we also need to check for a Bearer Token
		header := req.Header.Get("Authorization")
		parts := strings.SplitN(header, " ", 2)
		if len(parts) == 2 && parts[0] == "Bearer" {
			keyParts := strings.SplitN(parts[1], ":", 2)
			if len(keyParts) < 2 {
				key = keyParts[0]
				username = "api_key"
			} else {
				key = keyParts[1]
				username = keyParts[0]
			}
		}
	}
	return username, key
}

func (a *Api) Auth() macaron.Handler {
	return func(ctx *models.Context) {
		username, key := getAuthCreds(ctx.Req.Request)
		if key == "" {
			log.Debugf("HTTP auth: no key specified -> 401")
			ctx.JSON(401, "Unauthorized")
			return
		}

		user, err := a.authPlugin.Auth(username, key)
		if err != nil {
			if err == auth.ErrInvalidCredentials || err == auth.ErrInvalidOrgId || err == auth.ErrInvalidInstanceID {
				ctx.JSON(401, err.Error())
				return
			}
			log.Errorf("HTTP auth: failed to perform authentication: %q -> 500", err.Error())
			ctx.JSON(500, err.Error())
			return
		}

		// allow admin users to impersonate other orgs.
		if user.IsAdmin {
			header := ctx.Req.Header.Get("X-Tsdb-Org")
			if header != "" {
				orgId, err := strconv.ParseInt(header, 10, 64)
				if err == nil && orgId != 0 {
					user.ID = int(orgId)
				}
			}
		}
		ctx.User = user
	}
}

func (a *Api) DDAuth() macaron.Handler {
	return func(ctx *models.Context) {
		var key string
		var username string

		header := ctx.Req.Header.Get("Dd-Api-Key")
		parts := strings.SplitN(header, ":", 2)
		if len(parts) == 1 {
			key = parts[0]
			username = "api_key"
		} else if len(parts) == 2 && parts[1] == "" {
			key = parts[1]
			username = "api_key"
		} else {
			key = parts[1]
			username = parts[0]
		}

		if key == "" {
			log.Debugf("HTTP DDauth: no key specified -> 401")
			ctx.JSON(401, "Unauthorized")
			return
		}

		user, err := a.authPlugin.Auth(username, key)
		if err != nil {
			if err == auth.ErrInvalidCredentials || err == auth.ErrInvalidOrgId || err == auth.ErrInvalidInstanceID {
				ctx.JSON(401, err.Error())
				return
			}
			log.Errorf("HTTP DDauth: failed to perform authentication: %q -> 500", err.Error())
			ctx.JSON(500, err.Error())
			return
		}

		// allow admin users to impersonate other orgs.
		if user.IsAdmin {
			header := ctx.Req.Header.Get("X-Tsdb-Org")
			if header != "" {
				orgId, err := strconv.ParseInt(header, 10, 64)
				if err == nil && orgId != 0 {
					user.ID = int(orgId)
				}
			}
		}
		ctx.User = user
	}
}

type requestStats struct {
	sync.Mutex
	responseCounts    map[string]map[int]*stats.CounterRate32
	latencyHistograms map[string]*stats.LatencyHistogram15s32
	sizeMeters        map[string]*stats.Meter32
}

func (r *requestStats) PathStatusCount(ctx *models.Context, path string, status int) {
	metricKey := fmt.Sprintf("api.request.%s.status.%d", path, status)
	r.Lock()
	p, ok := r.responseCounts[path]
	if !ok {
		p = make(map[int]*stats.CounterRate32)
		r.responseCounts[path] = p
	}
	c, ok := p[status]
	if !ok {
		c = stats.NewCounterRate32(metricKey)
		p[status] = c
	}
	r.Unlock()
	c.Inc()
}

func (r *requestStats) PathLatency(ctx *models.Context, path string, dur time.Duration) {
	r.Lock()
	p, ok := r.latencyHistograms[path]
	if !ok {
		p = stats.NewLatencyHistogram15s32(fmt.Sprintf("api.request.%s", path))
		r.latencyHistograms[path] = p
	}
	r.Unlock()
	p.Value(dur)
}

func (r *requestStats) PathSize(ctx *models.Context, path string, size int) {
	r.Lock()
	p, ok := r.sizeMeters[path]
	if !ok {
		p = stats.NewMeter32(fmt.Sprintf("api.request.%s.size", path), false)
		r.sizeMeters[path] = p
	}
	r.Unlock()
	p.Value(size)
}

// RequestStats returns a middleware that tracks request metrics.
func RequestStats() macaron.Handler {
	stats := requestStats{
		responseCounts:    make(map[string]map[int]*stats.CounterRate32),
		latencyHistograms: make(map[string]*stats.LatencyHistogram15s32),
		sizeMeters:        make(map[string]*stats.Meter32),
	}

	return func(ctx *models.Context) {
		start := time.Now()
		rw := ctx.Resp.(macaron.ResponseWriter)
		// call next handler. This will return after all handlers
		// have completed and the request has been sent.
		ctx.Next()
		status := rw.Status()
		path := pathSlug(ctx.Req.URL.Path)
		stats.PathStatusCount(ctx, path, status)
		stats.PathLatency(ctx, path, time.Since(start))
		// only record the request size if the request succeeded.
		if status < 300 {
			stats.PathSize(ctx, path, rw.Size())
		}
	}
}

func (a *Api) PromStats(handler string) macaron.Handler {
	return func(ctx *models.Context) {
		start := time.Now()
		rw := ctx.Resp.(macaron.ResponseWriter)
		// call next handler. This will return after all handlers
		// have completed and the request has been sent.
		ctx.Next()

		status := strconv.Itoa(rw.Status())
		took := time.Since(start)
		method := ctx.Req.Method

		requestDuration.WithLabelValues(method, handler, status).Observe(took.Seconds())
	}
}

func pathSlug(p string) string {
	slug := strings.TrimPrefix(path.Clean(p), "/")
	if slug == "" {
		slug = "root"
	}
	return strings.Replace(strings.Replace(slug, "/", "_", -1), ".", "_", -1)
}

func Tracer(componentName string) macaron.Handler {
	return func(macCtx *macaron.Context) {
		tracer := opentracing.GlobalTracer()
		path := pathSlug(macCtx.Req.URL.Path)
		spanCtx, _ := tracer.Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(macCtx.Req.Header))
		span := tracer.StartSpan("HTTP "+macCtx.Req.Method+" "+path, ext.RPCServerOption(spanCtx))

		ext.HTTPMethod.Set(span, macCtx.Req.Method)
		ext.HTTPUrl.Set(span, macCtx.Req.URL.String())
		ext.Component.Set(span, componentName+"/api")

		macCtx.Req = macaron.Request{
			Request: macCtx.Req.WithContext(opentracing.ContextWithSpan(macCtx.Req.Context(), span)),
		}
		macCtx.Resp = &TracingResponseWriter{
			ResponseWriter: macCtx.Resp,
		}
		macCtx.MapTo(macCtx.Resp, (*http.ResponseWriter)(nil))

		rw := macCtx.Resp.(*TracingResponseWriter)
		// call next handler. This will return after all handlers
		// have completed and the request has been sent.
		macCtx.Next()
		status := rw.Status()
		ext.HTTPStatusCode.Set(span, uint16(status))
		if status >= 200 && status < 300 {
			span.SetTag("http.size", rw.Size())
		}
		if status >= 400 {
			span.LogFields(otLog.Error(errors.New(string(rw.errBody))))
			if status >= 500 {
				ext.Error.Set(span, true)
			}
		}
		span.Finish()
	}
}

func CaptureBody(c *models.Context) {
	body, err := ioutil.ReadAll(c.Req.Request.Body)
	if err != nil {
		log.Error(3, "HTTP internal error: failed to read request body for proxying: %s -> 500", err)
		c.PlainText(500, []byte("internal error: failed to read request body for proxying"))
	}
	c.Req.Request.Body = ioutil.NopCloser(bytes.NewBuffer(body))
	c.Body = ioutil.NopCloser(bytes.NewBuffer(body))
}<commit_msg>strip out middleware other than graphite metrics<commit_after>package main

import (
	"fmt"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/grafana/metrictank/stats"
	"github.com/raintank/tsdb-gw/api/models"
	"gopkg.in/macaron.v1"
)

type requestStats struct {
	sync.Mutex
	responseCounts    map[string]map[int]*stats.CounterRate32
	latencyHistograms map[string]*stats.LatencyHistogram15s32
	sizeMeters        map[string]*stats.Meter32
}

func (r *requestStats) PathStatusCount(ctx *models.Context, path string, status int) {
	metricKey := fmt.Sprintf("api.request.%s.status.%d", path, status)
	r.Lock()
	p, ok := r.responseCounts[path]
	if !ok {
		p = make(map[int]*stats.CounterRate32)
		r.responseCounts[path] = p
	}
	c, ok := p[status]
	if !ok {
		c = stats.NewCounterRate32(metricKey)
		p[status] = c
	}
	r.Unlock()
	c.Inc()
}

func (r *requestStats) PathLatency(ctx *models.Context, path string, dur time.Duration) {
	r.Lock()
	p, ok := r.latencyHistograms[path]
	if !ok {
		p = stats.NewLatencyHistogram15s32(fmt.Sprintf("api.request.%s", path))
		r.latencyHistograms[path] = p
	}
	r.Unlock()
	p.Value(dur)
}

func (r *requestStats) PathSize(ctx *models.Context, path string, size int) {
	r.Lock()
	p, ok := r.sizeMeters[path]
	if !ok {
		p = stats.NewMeter32(fmt.Sprintf("api.request.%s.size", path), false)
		r.sizeMeters[path] = p
	}
	r.Unlock()
	p.Value(size)
}

// RequestStats returns a middleware that tracks request metrics.
func RequestStats() macaron.Handler {
	stats := requestStats{
		responseCounts:    make(map[string]map[int]*stats.CounterRate32),
		latencyHistograms: make(map[string]*stats.LatencyHistogram15s32),
		sizeMeters:        make(map[string]*stats.Meter32),
	}

	return func(ctx *models.Context) {
		start := time.Now()
		rw := ctx.Resp.(macaron.ResponseWriter)
		// call next handler. This will return after all handlers
		// have completed and the request has been sent.
		ctx.Next()
		status := rw.Status()
		path := pathSlug(ctx.Req.URL.Path)
		stats.PathStatusCount(ctx, path, status)
		stats.PathLatency(ctx, path, time.Since(start))
		// only record the request size if the request succeeded.
		if status < 300 {
			stats.PathSize(ctx, path, rw.Size())
		}
	}
}

func pathSlug(p string) string {
	slug := strings.TrimPrefix(path.Clean(p), "/")
	if slug == "" {
		slug = "root"
	}
	return strings.Replace(strings.Replace(slug, "/", "_", -1), ".", "_", -1)
}
<|endoftext|>