<commit_before>package cliconfig

import (
	"fmt"
	"os"
	"reflect"
	"regexp"
	"strconv"
	"strings"

	"github.com/buildkite/agent/utils"
	"github.com/codegangsta/cli"
	"github.com/oleiade/reflections"
)

type Loader struct {
	// The context that is passed when using a codegangsta/cli action
	CLI *cli.Context

	// The struct that the config values will be loaded into
	Config interface{}

	// A slice of paths to files that should be used as config files
	DefaultConfigFilePaths []string

	// The file that was used when loading this configuration
	File *File
}

var CLISpecialNameRegex = regexp.MustCompile(`(arg):(\d+)`)

// A shortcut for loading a config from the CLI
func Load(c *cli.Context, cfg interface{}) error {
	l := Loader{CLI: c, Config: cfg}

	return l.Load()
}

// Loads the config from the CLI and config files that are present.
func (l *Loader) Load() error {
	// Try and find a config file, either passed in the command line using
	// --config, or in one of the default configuration file paths.
	if l.CLI.String("config") != "" {
		file := File{Path: l.CLI.String("config")}

		// Because this file was passed in manually, we should throw an error
		// if it doesn't exist.
		if file.Exists() {
			l.File = &file
		} else {
			return fmt.Errorf("A configuration file could not be found at: %s", file.AbsolutePath())
		}
	} else if len(l.DefaultConfigFilePaths) > 0 {
		for _, path := range l.DefaultConfigFilePaths {
			file := File{Path: path}

			// If the config file exists, save it to the loader and
			// don't bother checking the others.
			if file.Exists() {
				l.File = &file
				break
			}
		}
	}

	// If a file was found, then we should load it
	if l.File != nil {
		// Attempt to load the config file we've found
		if err := l.File.Load(); err != nil {
			return err
		}
	}

	// Now it's onto actually setting the fields. We start by getting all
	// the fields from the configuration interface
	var fields []string
	fields, _ = reflections.Fields(l.Config)

	// Loop through each of the fields, and look for tags and handle them
	// appropriately
	for _, fieldName := range fields {
		// Start by loading the value from the CLI context if the tag
		// exists
		cliName, _ := reflections.GetFieldTag(l.Config, fieldName, "cli")
		if cliName != "" {
			// Load the value from the CLI Context
			err := l.setFieldValueFromCLI(fieldName, cliName)
			if err != nil {
				return err
			}
		}

		// Are there any normalizations we need to make?
		normalization, _ := reflections.GetFieldTag(l.Config, fieldName, "normalize")
		if normalization != "" {
			// Apply the normalization
			err := l.normalizeField(fieldName, normalization)
			if err != nil {
				return err
			}
		}

		// Check for field deprecation
		deprecationError, _ := reflections.GetFieldTag(l.Config, fieldName, "deprecated")
		if deprecationError != "" {
			// If the deprecated field's value isn't emtpy, then we
			// return the deprecation error message.
			if !l.fieldValueIsEmpty(fieldName) {
				return fmt.Errorf(deprecationError)
			}
		}

		// Perform validations
		validationRules, _ := reflections.GetFieldTag(l.Config, fieldName, "validate")
		if validationRules != "" {
			// Determine the label for the field
			label, _ := reflections.GetFieldTag(l.Config, fieldName, "label")
			if label == "" {
				// Use the cli name if it exists, but if it
				// doesn't, just default to the structs field
				// name. Not great, but works!
				if cliName != "" {
					label = cliName
				} else {
					label = fieldName
				}
			}

			// Validate the fieid, and if it fails, return it's
			// error.
			err := l.validateField(fieldName, label, validationRules)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (l Loader) setFieldValueFromCLI(fieldName string, cliName string) error {
	// Get the kind of field we need to set
	fieldKind, err := reflections.GetFieldKind(l.Config, fieldName)
	if err != nil {
		return fmt.Errorf(`Failed to get the type of struct field %s`, fieldName)
	}

	var value interface{}

	// See the if the cli option is using the special format i.e. (arg:1)
	special := CLISpecialNameRegex.FindStringSubmatch(cliName)
	if len(special) == 3 {
		// Should this cli option be loaded from the CLI arguments?
		if special[1] == "arg" {
			// Convert the arg position to an integer
			i, err := strconv.Atoi(special[2])
			if err != nil {
				return fmt.Errorf("Failed to convert string to int: %s", err)
			}

			// Only set the value if the args are long enough for
			// the position to exist.
			if len(l.CLI.Args()) > i {
				// Get the value from the args
				value = l.CLI.Args()[i]
			}
		}
	} else {
		// If the cli name didn't have the special format, then we need to
		// either load from the context's flags, or from a config file.

		// We start by defaulting the value to what ever was provided
		// by the configuration file
		if l.File != nil {
			if configFileValue, ok := l.File.Config[cliName]; ok {
				// Convert the config file value to it's correct type
				if fieldKind == reflect.String {
					value = configFileValue
				} else if fieldKind == reflect.Slice {
					value = strings.Split(configFileValue, ",")
				} else if fieldKind == reflect.Bool {
					value, _ = strconv.ParseBool(configFileValue)
				} else {
					return fmt.Errorf("Unable to convert string to type %s", fieldKind)
				}
			}
		}

		// If a value hasn't been found in a config file, but there
		// _is_ one provided by the CLI context, then use that.
		if value == nil || l.cliValueIsSet(cliName) {
			if fieldKind == reflect.String {
				value = l.CLI.String(cliName)
			} else if fieldKind == reflect.Slice {
				value = l.CLI.StringSlice(cliName)
			} else if fieldKind == reflect.Bool {
				value = l.CLI.Bool(cliName)
			} else {
				return fmt.Errorf("Unable to handle type: %s", fieldKind)
			}
		}
	}

	// Set the value to the cfg
	if value != nil {
		err = reflections.SetField(l.Config, fieldName, value)
		if err != nil {
			return fmt.Errorf("Could not set value `%s` to field `%s` (%s)", value, fieldName, err)
		}
	}

	return nil
}

func (l Loader) Errorf(format string, v ...interface{}) error {
	suffix := fmt.Sprintf(" See: `%s %s --help`", l.CLI.App.Name, l.CLI.Command.Name)

	return fmt.Errorf(format+suffix, v...)
}

func (l Loader) cliValueIsSet(cliName string) bool {
	if l.CLI.IsSet(cliName) {
		return true
	} else {
		// cli.Context#IsSet only checks to see if the command was set via the cli, not
		// via the environment. So here we do some hacks to find out the name of the
		// EnvVar, and return true if it was set.
		for _, flag := range l.CLI.Command.Flags {
			name, _ := reflections.GetField(flag, "Name")
			envVar, _ := reflections.GetField(flag, "EnvVar")
			if name == cliName && envVar != "" {
				// Make sure envVar is a string
				if envVarStr, ok := envVar.(string); ok {
					envVarStr = strings.TrimSpace(string(envVarStr))

					return os.Getenv(envVarStr) != ""
				}
			}
		}
	}

	return false
}

func (l Loader) fieldValueIsEmpty(fieldName string) bool {
	// We need to use the field kind to determine the type of empty test.
	value, _ := reflections.GetField(l.Config, fieldName)
	fieldKind, _ := reflections.GetFieldKind(l.Config, fieldName)

	if fieldKind == reflect.String {
		return value == ""
	} else if fieldKind == reflect.Slice {
		v := reflect.ValueOf(value)
		return v.Len() == 0
	} else if fieldKind == reflect.Bool {
		return value == false
	} else {
		panic(fmt.Sprintf("Can't determine empty-ness for field type %s", fieldKind))
	}

	return false
}

func (l Loader) validateField(fieldName string, label string, validationRules string) error {
	// Split up the validation rules
	rules := strings.Split(validationRules, ",")

	// Loop through each rule, and perform it
	for _, rule := range rules {
		if rule == "required" {
			if l.fieldValueIsEmpty(fieldName) {
				return l.Errorf("Missing %s.", label)
			}
		} else if rule == "file-exists" {
			value, _ := reflections.GetField(l.Config, fieldName)

			// Make sure the value is converted to a string
			if valueAsString, ok := value.(string); ok {
				// Return an error if the path doesn't exist
				if _, err := os.Stat(valueAsString); err != nil {
					return fmt.Errorf("Could not find %s located at %s", label, value)
				}
			}
		} else {
			return fmt.Errorf("Unknown config validation rule `%s`", rule)
		}
	}

	return nil
}

func (l Loader) normalizeField(fieldName string, normalization string) error {
	if normalization == "filepath" {
		value, _ := reflections.GetField(l.Config, fieldName)
		fieldKind, _ := reflections.GetFieldKind(l.Config, fieldName)

		// Make sure we're normalizing a string filed
		if fieldKind != reflect.String {
			return fmt.Errorf("filepath normalization only works on string fields")
		}

		// Normalize the field to be a filepath
		if valueAsString, ok := value.(string); ok {
			normalizedPath := utils.NormalizeFilePath(valueAsString)
			if err := reflections.SetField(l.Config, fieldName, normalizedPath); err != nil {
				return err
			}
		}
	} else {
		return fmt.Errorf("Unknown normalization `%s`", normalization)
	}

	return nil
}
<commit_msg>Un-generalise the CLI "special" tags matcher<commit_after>package cliconfig

import (
	"fmt"
	"os"
	"reflect"
	"regexp"
	"strconv"
	"strings"

	"github.com/buildkite/agent/utils"
	"github.com/codegangsta/cli"
	"github.com/oleiade/reflections"
)

type Loader struct {
	// The context that is passed when using a codegangsta/cli action
	CLI *cli.Context

	// The struct that the config values will be loaded into
	Config interface{}

	// A slice of paths to files that should be used as config files
	DefaultConfigFilePaths []string

	// The file that was used when loading this configuration
	File *File
}

var ArgCliNameRegexp = regexp.MustCompile(`arg:(\d+)`)

// A shortcut for loading a config from the CLI
func Load(c *cli.Context, cfg interface{}) error {
	l := Loader{CLI: c, Config: cfg}

	return l.Load()
}

// Loads the config from the CLI and config files that are present.
func (l *Loader) Load() error {
	// Try and find a config file, either passed in the command line using
	// --config, or in one of the default configuration file paths.
	if l.CLI.String("config") != "" {
		file := File{Path: l.CLI.String("config")}

		// Because this file was passed in manually, we should throw an error
		// if it doesn't exist.
		if file.Exists() {
			l.File = &file
		} else {
			return fmt.Errorf("A configuration file could not be found at: %s", file.AbsolutePath())
		}
	} else if len(l.DefaultConfigFilePaths) > 0 {
		for _, path := range l.DefaultConfigFilePaths {
			file := File{Path: path}

			// If the config file exists, save it to the loader and
			// don't bother checking the others.
			if file.Exists() {
				l.File = &file
				break
			}
		}
	}

	// If a file was found, then we should load it
	if l.File != nil {
		// Attempt to load the config file we've found
		if err := l.File.Load(); err != nil {
			return err
		}
	}

	// Now it's onto actually setting the fields. We start by getting all
	// the fields from the configuration interface
	var fields []string
	fields, _ = reflections.Fields(l.Config)

	// Loop through each of the fields, and look for tags and handle them
	// appropriately
	for _, fieldName := range fields {
		// Start by loading the value from the CLI context if the tag
		// exists
		cliName, _ := reflections.GetFieldTag(l.Config, fieldName, "cli")
		if cliName != "" {
			// Load the value from the CLI Context
			err := l.setFieldValueFromCLI(fieldName, cliName)
			if err != nil {
				return err
			}
		}

		// Are there any normalizations we need to make?
		normalization, _ := reflections.GetFieldTag(l.Config, fieldName, "normalize")
		if normalization != "" {
			// Apply the normalization
			err := l.normalizeField(fieldName, normalization)
			if err != nil {
				return err
			}
		}

		// Check for field deprecation
		deprecationError, _ := reflections.GetFieldTag(l.Config, fieldName, "deprecated")
		if deprecationError != "" {
			// If the deprecated field's value isn't emtpy, then we
			// return the deprecation error message.
			if !l.fieldValueIsEmpty(fieldName) {
				return fmt.Errorf(deprecationError)
			}
		}

		// Perform validations
		validationRules, _ := reflections.GetFieldTag(l.Config, fieldName, "validate")
		if validationRules != "" {
			// Determine the label for the field
			label, _ := reflections.GetFieldTag(l.Config, fieldName, "label")
			if label == "" {
				// Use the cli name if it exists, but if it
				// doesn't, just default to the structs field
				// name. Not great, but works!
				if cliName != "" {
					label = cliName
				} else {
					label = fieldName
				}
			}

			// Validate the fieid, and if it fails, return it's
			// error.
			err := l.validateField(fieldName, label, validationRules)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (l Loader) setFieldValueFromCLI(fieldName string, cliName string) error {
	// Get the kind of field we need to set
	fieldKind, err := reflections.GetFieldKind(l.Config, fieldName)
	if err != nil {
		return fmt.Errorf(`Failed to get the type of struct field %s`, fieldName)
	}

	var value interface{}

	// See the if the cli option is using the arg format i.e. (arg:1)
	argMatch := ArgCliNameRegexp.FindStringSubmatch(cliName)
	if len(argMatch) > 0 {
		argNum := argMatch[1]

		// Convert the arg position to an integer
		argIndex, err := strconv.Atoi(argNum)
		if err != nil {
			return fmt.Errorf("Failed to convert string to int: %s", err)
		}

		// Only set the value if the args are long enough for
		// the position to exist.
		if len(l.CLI.Args()) > argIndex {
			value = l.CLI.Args()[argIndex]
		}
	} else {
		// If the cli name didn't have the special format, then we need to
		// either load from the context's flags, or from a config file.

		// We start by defaulting the value to what ever was provided
		// by the configuration file
		if l.File != nil {
			if configFileValue, ok := l.File.Config[cliName]; ok {
				// Convert the config file value to it's correct type
				if fieldKind == reflect.String {
					value = configFileValue
				} else if fieldKind == reflect.Slice {
					value = strings.Split(configFileValue, ",")
				} else if fieldKind == reflect.Bool {
					value, _ = strconv.ParseBool(configFileValue)
				} else {
					return fmt.Errorf("Unable to convert string to type %s", fieldKind)
				}
			}
		}

		// If a value hasn't been found in a config file, but there
		// _is_ one provided by the CLI context, then use that.
		if value == nil || l.cliValueIsSet(cliName) {
			if fieldKind == reflect.String {
				value = l.CLI.String(cliName)
			} else if fieldKind == reflect.Slice {
				value = l.CLI.StringSlice(cliName)
			} else if fieldKind == reflect.Bool {
				value = l.CLI.Bool(cliName)
			} else {
				return fmt.Errorf("Unable to handle type: %s", fieldKind)
			}
		}
	}

	// Set the value to the cfg
	if value != nil {
		err = reflections.SetField(l.Config, fieldName, value)
		if err != nil {
			return fmt.Errorf("Could not set value `%s` to field `%s` (%s)", value, fieldName, err)
		}
	}

	return nil
}

func (l Loader) Errorf(format string, v ...interface{}) error {
	suffix := fmt.Sprintf(" See: `%s %s --help`", l.CLI.App.Name, l.CLI.Command.Name)

	return fmt.Errorf(format+suffix, v...)
}

func (l Loader) cliValueIsSet(cliName string) bool {
	if l.CLI.IsSet(cliName) {
		return true
	} else {
		// cli.Context#IsSet only checks to see if the command was set via the cli, not
		// via the environment. So here we do some hacks to find out the name of the
		// EnvVar, and return true if it was set.
		for _, flag := range l.CLI.Command.Flags {
			name, _ := reflections.GetField(flag, "Name")
			envVar, _ := reflections.GetField(flag, "EnvVar")
			if name == cliName && envVar != "" {
				// Make sure envVar is a string
				if envVarStr, ok := envVar.(string); ok {
					envVarStr = strings.TrimSpace(string(envVarStr))

					return os.Getenv(envVarStr) != ""
				}
			}
		}
	}

	return false
}

func (l Loader) fieldValueIsEmpty(fieldName string) bool {
	// We need to use the field kind to determine the type of empty test.
	value, _ := reflections.GetField(l.Config, fieldName)
	fieldKind, _ := reflections.GetFieldKind(l.Config, fieldName)

	if fieldKind == reflect.String {
		return value == ""
	} else if fieldKind == reflect.Slice {
		v := reflect.ValueOf(value)
		return v.Len() == 0
	} else if fieldKind == reflect.Bool {
		return value == false
	} else {
		panic(fmt.Sprintf("Can't determine empty-ness for field type %s", fieldKind))
	}

	return false
}

func (l Loader) validateField(fieldName string, label string, validationRules string) error {
	// Split up the validation rules
	rules := strings.Split(validationRules, ",")

	// Loop through each rule, and perform it
	for _, rule := range rules {
		if rule == "required" {
			if l.fieldValueIsEmpty(fieldName) {
				return l.Errorf("Missing %s.", label)
			}
		} else if rule == "file-exists" {
			value, _ := reflections.GetField(l.Config, fieldName)

			// Make sure the value is converted to a string
			if valueAsString, ok := value.(string); ok {
				// Return an error if the path doesn't exist
				if _, err := os.Stat(valueAsString); err != nil {
					return fmt.Errorf("Could not find %s located at %s", label, value)
				}
			}
		} else {
			return fmt.Errorf("Unknown config validation rule `%s`", rule)
		}
	}

	return nil
}

func (l Loader) normalizeField(fieldName string, normalization string) error {
	if normalization == "filepath" {
		value, _ := reflections.GetField(l.Config, fieldName)
		fieldKind, _ := reflections.GetFieldKind(l.Config, fieldName)

		// Make sure we're normalizing a string filed
		if fieldKind != reflect.String {
			return fmt.Errorf("filepath normalization only works on string fields")
		}

		// Normalize the field to be a filepath
		if valueAsString, ok := value.(string); ok {
			normalizedPath := utils.NormalizeFilePath(valueAsString)
			if err := reflections.SetField(l.Config, fieldName, normalizedPath); err != nil {
				return err
			}
		}
	} else {
		return fmt.Errorf("Unknown normalization `%s`", normalization)
	}

	return nil
}
<|endoftext|>


<commit_before>package disk

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"syscall"

	"code.google.com/p/go.crypto/nacl/secretbox"
	"code.google.com/p/go.crypto/scrypt"
	"code.google.com/p/goprotobuf/proto"
)

const (
	kdfSaltLen    = 32
	kdfKeyLen     = 32
	erasureKeyLen = 32
)

var headerMagic = [8]byte{0xa8, 0x34, 0x64, 0x9e, 0xce, 0x39, 0x94, 0xe3}

// ErasureStorage represents a type of storage that can store, and erase, small
// amounts of data.
type ErasureStorage interface {
	// Create creates a new erasure storage object and fills out header to
	// include the needed values.
	Create(header *Header, key *[kdfKeyLen]byte) error
	// Read reads the current value of the storage.
	Read(key *[kdfKeyLen]byte) (*[erasureKeyLen]byte, error)
	// Write requests that the given value be stored and the old value
	// forgotten.
	Write(key *[kdfKeyLen]byte, value *[erasureKeyLen]byte) error
}

// erasureRegistry is a slice of functions, each of which can inspect a header
// and optionally return an ErasureStorage that loads the mask key specified by
// that header.
var erasureRegistry []func(*Header) ErasureStorage

// StateFile encapsulates information about a state file on diskl
type StateFile struct {
	Path string
	Rand io.Reader
	Log  func(format string, args ...interface{})
	// Erasure is able to store a `mask key' - a random value that is XORed
	// with the key. This is done because an ErasureStorage is believed to
	// be able to erase old mask values.
	Erasure ErasureStorage

	header Header
	key    [kdfKeyLen]byte
	mask   [erasureKeyLen]byte
	valid  bool
}

func NewStateFile(rand io.Reader, path string) *StateFile {
	return &StateFile{
		Rand: rand,
		Path: path,
	}
}

func (sf *StateFile) Lock(create bool) (*Lock, error) {
	flags := os.O_RDWR
	if create {
		flags |= os.O_CREATE | os.O_EXCL
	}
	file, err := os.OpenFile(sf.Path, flags, 0600)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	fd := int(file.Fd())
	newFd, err := syscall.Dup(fd)
	if err != nil {
		return nil, err
	}
	if syscall.Flock(newFd, syscall.LOCK_EX|syscall.LOCK_NB) != nil {
		syscall.Close(newFd)
		return nil, nil
	}
	return &Lock{newFd}, nil
}

func (sf *StateFile) deriveKey(pw string) error {
	if len(pw) == 0 && sf.header.Scrypt != nil {
		return BadPasswordError
	}
	params := sf.header.Scrypt
	key, err := scrypt.Key([]byte(pw), sf.header.KdfSalt, int(params.GetN()), int(params.GetR()), int(params.GetP()), kdfKeyLen)
	if err != nil {
		return err
	}
	copy(sf.key[:], key)
	return nil
}

func (sf *StateFile) Create(pw string) error {
	var salt [kdfSaltLen]byte
	if _, err := io.ReadFull(sf.Rand, salt[:]); err != nil {
		return err
	}

	if len(pw) > 0 {
		sf.header.KdfSalt = salt[:]
		if err := sf.deriveKey(pw); err != nil {
			return err
		}
		sf.header.Scrypt = new(Header_SCrypt)
	}

	if sf.Erasure != nil {
		if err := sf.Erasure.Create(&sf.header, &sf.key); err != nil {
			return err
		}
		if _, err := io.ReadFull(sf.Rand, sf.mask[:]); err != nil {
			return err
		}
		fmt.Printf("ERASURE: %x\n", sf.mask)
		if err := sf.Erasure.Write(&sf.key, &sf.mask); err != nil {
			return err
		}
	} else {
		sf.header.NoErasureStorage = proto.Bool(true)
	}

	sf.valid = true
	return nil
}

func (sf *StateFile) Read(pw string) (*State, error) {
	b, err := ioutil.ReadFile(sf.Path)
	if err != nil {
		return nil, err
	}

	if len(b) < len(headerMagic)+4 {
		return nil, errors.New("state file is too small to be valid")
	}

	if !bytes.Equal(b[:len(headerMagic)], headerMagic[:]) {
		sf.header.NoErasureStorage = proto.Bool(true)
		if len(pw) > 0 {
			sf.header.Scrypt = new(Header_SCrypt)
			sf.header.KdfSalt = b[:32]
			if err := sf.deriveKey(pw); err != nil {
				return nil, err
			}
		}
		b = b[32:]
		state, err := sf.readOldStyle(b)
		if err != nil {
			return nil, err
		}
		return state, nil
	}

	b = b[len(headerMagic):]
	headerLen := binary.LittleEndian.Uint32(b)
	b = b[4:]
	if headerLen > 1<<16 {
		return nil, errors.New("state file corrupt")
	}
	if len(b) < int(headerLen) {
		return nil, errors.New("state file truncated")
	}
	headerBytes := b[:int(headerLen)]
	b = b[int(headerLen):]

	if err := proto.Unmarshal(headerBytes, &sf.header); err != nil {
		return nil, err
	}
	if len(pw) > 0 {
		if err := sf.deriveKey(pw); err != nil {
			return nil, err
		}
	}

	if !sf.header.GetNoErasureStorage() {
		for _, erasureMethod := range erasureRegistry {
			sf.Erasure = erasureMethod(&sf.header)
			if sf.Erasure != nil {
				break
			}
		}
		if sf.Erasure == nil {
			return nil, errors.New("unknown erasure storage method")
		}

		mask, err := sf.Erasure.Read(&sf.key)
		if err != nil {
			fmt.Printf("ERASURE ERROR: %s\n", err)
			return nil, err
		}
		fmt.Printf("ERASURE: %x\n", mask)
		copy(sf.mask[:], mask[:])
	}

	smearedCopies := int(sf.header.GetNonceSmearCopies())

	if len(b) < 24*smearedCopies {
		return nil, errors.New("state file truncated")
	}

	var nonce [24]byte
	for i := 0; i < smearedCopies; i++ {
		for j := 0; j < 24; j++ {
			nonce[j] ^= b[24*i+j]
		}
	}

	b = b[24*smearedCopies:]

	var effectiveKey [kdfKeyLen]byte
	for i := range effectiveKey {
		effectiveKey[i] = sf.mask[i] ^ sf.key[i]
	}
	plaintext, ok := secretbox.Open(nil, b, &nonce, &effectiveKey)
	if !ok {
		return nil, BadPasswordError
	}
	if len(plaintext) < 4 {
		return nil, errors.New("state file corrupt")
	}
	length := binary.LittleEndian.Uint32(plaintext[:4])
	plaintext = plaintext[4:]
	if length > 1<<31 || length > uint32(len(plaintext)) {
		return nil, errors.New("state file corrupt")
	}
	plaintext = plaintext[:int(length)]

	var state State
	if err := proto.Unmarshal(plaintext, &state); err != nil {
		return nil, err
	}

	return &state, nil
}

func (sf *StateFile) readOldStyle(b []byte) (*State, error) {
	return loadOldState(b, &sf.key)
}

type NewState struct {
	State                []byte
	RotateErasureStorage bool
}

func (sf *StateFile) StartWriter(states chan NewState, done chan struct{}) {
	for {
		newState, ok := <-states
		if !ok {
			close(done)
			return
		}

		s := newState.State

		length := uint32(len(s)) + 4
		for i := uint(17); i < 32; i++ {
			if n := (uint32(1) << i); n >= length {
				length = n
				break
			}
		}

		plaintext := make([]byte, length)
		copy(plaintext[4:], s)
		if _, err := io.ReadFull(sf.Rand, plaintext[len(s)+4:]); err != nil {
			panic(err)
		}
		binary.LittleEndian.PutUint32(plaintext, uint32(len(s)))

		smearCopies := int(sf.header.GetNonceSmearCopies())
		nonceSmear := make([]byte, 24*smearCopies)
		if _, err := io.ReadFull(sf.Rand, nonceSmear[:]); err != nil {
			panic(err)
		}

		var nonce [24]byte
		for i := 0; i < smearCopies; i++ {
			for j := 0; j < 24; j++ {
				nonce[j] ^= nonceSmear[24*i+j]
			}
		}

		if sf.Erasure != nil && newState.RotateErasureStorage {
			var newMask [erasureKeyLen]byte
			if _, err := io.ReadFull(sf.Rand, newMask[:]); err != nil {
				panic(err)
			}
			if err := sf.Erasure.Write(&sf.key, &newMask); err != nil {
				sf.Log("Failed to write new erasure value: %s", err)
			} else {
				copy(sf.mask[:], newMask[:])
			}
		}

		var effectiveKey [kdfKeyLen]byte
		for i := range effectiveKey {
			effectiveKey[i] = sf.mask[i] ^ sf.key[i]
		}
		ciphertext := secretbox.Seal(nil, plaintext, &nonce, &effectiveKey)

		out, err := os.OpenFile(sf.Path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0600)
		if err != nil {
			panic(err)
		}
		headerBytes, err := proto.Marshal(&sf.header)
		if err != nil {
			panic(err)
		}
		if _, err := out.Write(headerMagic[:]); err != nil {
			panic(err)
		}
		if err := binary.Write(out, binary.LittleEndian, uint32(len(headerBytes))); err != nil {
			panic(err)
		}
		if _, err := out.Write(headerBytes); err != nil {
			panic(err)
		}
		if _, err := out.Write(nonceSmear[:]); err != nil {
			panic(err)
		}
		if _, err := out.Write(ciphertext); err != nil {
			panic(err)
		}
		out.Close()
	}
}

type Lock struct {
	fd int
}

func (l *Lock) Close() {
	syscall.Flock(l.fd, syscall.LOCK_UN)
	syscall.Close(l.fd)
}

var BadPasswordError = errors.New("bad password")

func loadOldState(b []byte, key *[32]byte) (*State, error) {
	const (
		SCryptSaltLen = 32
		smearedCopies = 32768 / 24
	)

	if len(b) < SCryptSaltLen+24*smearedCopies {
		return nil, errors.New("state file is too small to be valid")
	}

	var nonce [24]byte
	for i := 0; i < smearedCopies; i++ {
		for j := 0; j < 24; j++ {
			nonce[j] ^= b[24*i+j]
		}
	}

	b = b[24*smearedCopies:]
	plaintext, ok := secretbox.Open(nil, b, &nonce, key)
	if !ok {
		return nil, BadPasswordError
	}
	if len(plaintext) < 4 {
		return nil, errors.New("state file corrupt")
	}
	length := binary.LittleEndian.Uint32(plaintext[:4])
	plaintext = plaintext[4:]
	if length > 1<<31 || length > uint32(len(plaintext)) {
		return nil, errors.New("state file corrupt")
	}
	plaintext = plaintext[:int(length)]

	var state State
	if err := proto.Unmarshal(plaintext, &state); err != nil {
		return nil, err
	}

	return &state, nil
}
<commit_msg>Remove debug logging from the disk code.<commit_after>package disk

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"syscall"

	"code.google.com/p/go.crypto/nacl/secretbox"
	"code.google.com/p/go.crypto/scrypt"
	"code.google.com/p/goprotobuf/proto"
)

const (
	kdfSaltLen    = 32
	kdfKeyLen     = 32
	erasureKeyLen = 32
)

var headerMagic = [8]byte{0xa8, 0x34, 0x64, 0x9e, 0xce, 0x39, 0x94, 0xe3}

// ErasureStorage represents a type of storage that can store, and erase, small
// amounts of data.
type ErasureStorage interface {
	// Create creates a new erasure storage object and fills out header to
	// include the needed values.
	Create(header *Header, key *[kdfKeyLen]byte) error
	// Read reads the current value of the storage.
	Read(key *[kdfKeyLen]byte) (*[erasureKeyLen]byte, error)
	// Write requests that the given value be stored and the old value
	// forgotten.
	Write(key *[kdfKeyLen]byte, value *[erasureKeyLen]byte) error
}

// erasureRegistry is a slice of functions, each of which can inspect a header
// and optionally return an ErasureStorage that loads the mask key specified by
// that header.
var erasureRegistry []func(*Header) ErasureStorage

// StateFile encapsulates information about a state file on diskl
type StateFile struct {
	Path string
	Rand io.Reader
	Log  func(format string, args ...interface{})
	// Erasure is able to store a `mask key' - a random value that is XORed
	// with the key. This is done because an ErasureStorage is believed to
	// be able to erase old mask values.
	Erasure ErasureStorage

	header Header
	key    [kdfKeyLen]byte
	mask   [erasureKeyLen]byte
	valid  bool
}

func NewStateFile(rand io.Reader, path string) *StateFile {
	return &StateFile{
		Rand: rand,
		Path: path,
	}
}

func (sf *StateFile) Lock(create bool) (*Lock, error) {
	flags := os.O_RDWR
	if create {
		flags |= os.O_CREATE | os.O_EXCL
	}
	file, err := os.OpenFile(sf.Path, flags, 0600)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	fd := int(file.Fd())
	newFd, err := syscall.Dup(fd)
	if err != nil {
		return nil, err
	}
	if syscall.Flock(newFd, syscall.LOCK_EX|syscall.LOCK_NB) != nil {
		syscall.Close(newFd)
		return nil, nil
	}
	return &Lock{newFd}, nil
}

func (sf *StateFile) deriveKey(pw string) error {
	if len(pw) == 0 && sf.header.Scrypt != nil {
		return BadPasswordError
	}
	params := sf.header.Scrypt
	key, err := scrypt.Key([]byte(pw), sf.header.KdfSalt, int(params.GetN()), int(params.GetR()), int(params.GetP()), kdfKeyLen)
	if err != nil {
		return err
	}
	copy(sf.key[:], key)
	return nil
}

func (sf *StateFile) Create(pw string) error {
	var salt [kdfSaltLen]byte
	if _, err := io.ReadFull(sf.Rand, salt[:]); err != nil {
		return err
	}

	if len(pw) > 0 {
		sf.header.KdfSalt = salt[:]
		if err := sf.deriveKey(pw); err != nil {
			return err
		}
		sf.header.Scrypt = new(Header_SCrypt)
	}

	if sf.Erasure != nil {
		if err := sf.Erasure.Create(&sf.header, &sf.key); err != nil {
			return err
		}
		if _, err := io.ReadFull(sf.Rand, sf.mask[:]); err != nil {
			return err
		}
		if err := sf.Erasure.Write(&sf.key, &sf.mask); err != nil {
			return err
		}
	} else {
		sf.header.NoErasureStorage = proto.Bool(true)
	}

	sf.valid = true
	return nil
}

func (sf *StateFile) Read(pw string) (*State, error) {
	b, err := ioutil.ReadFile(sf.Path)
	if err != nil {
		return nil, err
	}

	if len(b) < len(headerMagic)+4 {
		return nil, errors.New("state file is too small to be valid")
	}

	if !bytes.Equal(b[:len(headerMagic)], headerMagic[:]) {
		sf.header.NoErasureStorage = proto.Bool(true)
		if len(pw) > 0 {
			sf.header.Scrypt = new(Header_SCrypt)
			sf.header.KdfSalt = b[:32]
			if err := sf.deriveKey(pw); err != nil {
				return nil, err
			}
		}
		b = b[32:]
		state, err := sf.readOldStyle(b)
		if err != nil {
			return nil, err
		}
		return state, nil
	}

	b = b[len(headerMagic):]
	headerLen := binary.LittleEndian.Uint32(b)
	b = b[4:]
	if headerLen > 1<<16 {
		return nil, errors.New("state file corrupt")
	}
	if len(b) < int(headerLen) {
		return nil, errors.New("state file truncated")
	}
	headerBytes := b[:int(headerLen)]
	b = b[int(headerLen):]

	if err := proto.Unmarshal(headerBytes, &sf.header); err != nil {
		return nil, err
	}
	if len(pw) > 0 {
		if err := sf.deriveKey(pw); err != nil {
			return nil, err
		}
	}

	if !sf.header.GetNoErasureStorage() {
		for _, erasureMethod := range erasureRegistry {
			sf.Erasure = erasureMethod(&sf.header)
			if sf.Erasure != nil {
				break
			}
		}
		if sf.Erasure == nil {
			return nil, errors.New("unknown erasure storage method")
		}

		mask, err := sf.Erasure.Read(&sf.key)
		if err != nil {
			return nil, err
		}
		copy(sf.mask[:], mask[:])
	}

	smearedCopies := int(sf.header.GetNonceSmearCopies())

	if len(b) < 24*smearedCopies {
		return nil, errors.New("state file truncated")
	}

	var nonce [24]byte
	for i := 0; i < smearedCopies; i++ {
		for j := 0; j < 24; j++ {
			nonce[j] ^= b[24*i+j]
		}
	}

	b = b[24*smearedCopies:]

	var effectiveKey [kdfKeyLen]byte
	for i := range effectiveKey {
		effectiveKey[i] = sf.mask[i] ^ sf.key[i]
	}
	plaintext, ok := secretbox.Open(nil, b, &nonce, &effectiveKey)
	if !ok {
		return nil, BadPasswordError
	}
	if len(plaintext) < 4 {
		return nil, errors.New("state file corrupt")
	}
	length := binary.LittleEndian.Uint32(plaintext[:4])
	plaintext = plaintext[4:]
	if length > 1<<31 || length > uint32(len(plaintext)) {
		return nil, errors.New("state file corrupt")
	}
	plaintext = plaintext[:int(length)]

	var state State
	if err := proto.Unmarshal(plaintext, &state); err != nil {
		return nil, err
	}

	return &state, nil
}

func (sf *StateFile) readOldStyle(b []byte) (*State, error) {
	return loadOldState(b, &sf.key)
}

type NewState struct {
	State                []byte
	RotateErasureStorage bool
}

func (sf *StateFile) StartWriter(states chan NewState, done chan struct{}) {
	for {
		newState, ok := <-states
		if !ok {
			close(done)
			return
		}

		s := newState.State

		length := uint32(len(s)) + 4
		for i := uint(17); i < 32; i++ {
			if n := (uint32(1) << i); n >= length {
				length = n
				break
			}
		}

		plaintext := make([]byte, length)
		copy(plaintext[4:], s)
		if _, err := io.ReadFull(sf.Rand, plaintext[len(s)+4:]); err != nil {
			panic(err)
		}
		binary.LittleEndian.PutUint32(plaintext, uint32(len(s)))

		smearCopies := int(sf.header.GetNonceSmearCopies())
		nonceSmear := make([]byte, 24*smearCopies)
		if _, err := io.ReadFull(sf.Rand, nonceSmear[:]); err != nil {
			panic(err)
		}

		var nonce [24]byte
		for i := 0; i < smearCopies; i++ {
			for j := 0; j < 24; j++ {
				nonce[j] ^= nonceSmear[24*i+j]
			}
		}

		if sf.Erasure != nil && newState.RotateErasureStorage {
			var newMask [erasureKeyLen]byte
			if _, err := io.ReadFull(sf.Rand, newMask[:]); err != nil {
				panic(err)
			}
			if err := sf.Erasure.Write(&sf.key, &newMask); err != nil {
				sf.Log("Failed to write new erasure value: %s", err)
			} else {
				copy(sf.mask[:], newMask[:])
			}
		}

		var effectiveKey [kdfKeyLen]byte
		for i := range effectiveKey {
			effectiveKey[i] = sf.mask[i] ^ sf.key[i]
		}
		ciphertext := secretbox.Seal(nil, plaintext, &nonce, &effectiveKey)

		out, err := os.OpenFile(sf.Path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0600)
		if err != nil {
			panic(err)
		}
		headerBytes, err := proto.Marshal(&sf.header)
		if err != nil {
			panic(err)
		}
		if _, err := out.Write(headerMagic[:]); err != nil {
			panic(err)
		}
		if err := binary.Write(out, binary.LittleEndian, uint32(len(headerBytes))); err != nil {
			panic(err)
		}
		if _, err := out.Write(headerBytes); err != nil {
			panic(err)
		}
		if _, err := out.Write(nonceSmear[:]); err != nil {
			panic(err)
		}
		if _, err := out.Write(ciphertext); err != nil {
			panic(err)
		}
		out.Close()
	}
}

type Lock struct {
	fd int
}

func (l *Lock) Close() {
	syscall.Flock(l.fd, syscall.LOCK_UN)
	syscall.Close(l.fd)
}

var BadPasswordError = errors.New("bad password")

func loadOldState(b []byte, key *[32]byte) (*State, error) {
	const (
		SCryptSaltLen = 32
		smearedCopies = 32768 / 24
	)

	if len(b) < SCryptSaltLen+24*smearedCopies {
		return nil, errors.New("state file is too small to be valid")
	}

	var nonce [24]byte
	for i := 0; i < smearedCopies; i++ {
		for j := 0; j < 24; j++ {
			nonce[j] ^= b[24*i+j]
		}
	}

	b = b[24*smearedCopies:]
	plaintext, ok := secretbox.Open(nil, b, &nonce, key)
	if !ok {
		return nil, BadPasswordError
	}
	if len(plaintext) < 4 {
		return nil, errors.New("state file corrupt")
	}
	length := binary.LittleEndian.Uint32(plaintext[:4])
	plaintext = plaintext[4:]
	if length > 1<<31 || length > uint32(len(plaintext)) {
		return nil, errors.New("state file corrupt")
	}
	plaintext = plaintext[:int(length)]

	var state State
	if err := proto.Unmarshal(plaintext, &state); err != nil {
		return nil, err
	}

	return &state, nil
}
<|endoftext|>


<commit_before>package dataframe

import (
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"go.skia.org/infra/go/query"
	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/vcsinfo"
	"go.skia.org/infra/perf/go/ptracestore"
)

type mockVcs struct {
	commits    []*vcsinfo.IndexCommit
	updateFail bool
}

func (m *mockVcs) From(start time.Time) []string                     { return nil }
func (m *mockVcs) Range(begin, end time.Time) []*vcsinfo.IndexCommit { return nil }
func (m *mockVcs) Details(hash string, includeBranchInfo bool) (*vcsinfo.LongCommit, error) {
	return nil, nil
}

func (m *mockVcs) Update(pull, allBranches bool) error {
	if m.updateFail {
		return fmt.Errorf("Failed to update.")
	}
	return nil
}

func (m *mockVcs) LastNIndex(N int) []*vcsinfo.IndexCommit {
	return m.commits
}

type mockPTraceStore struct {
	traceSet  ptracestore.TraceSet
	matchFail bool
}

func (m mockPTraceStore) Add(commitID *ptracestore.CommitID, values map[string]float32, sourceFile string) error {
	return nil
}

func (m mockPTraceStore) Details(commitID *ptracestore.CommitID, traceID string) (string, float32, error) {
	return "", 0, nil
}

func (m mockPTraceStore) Match(commitIDs []*ptracestore.CommitID, q query.Query) (ptracestore.TraceSet, error) {
	if m.matchFail {
		return nil, fmt.Errorf("Failed to retrieve traces.")
	}
	return m.traceSet, nil
}

var (
	ts0 = time.Unix(1406721642, 0).UTC()
	ts1 = time.Unix(1406721715, 0).UTC()

	commits = []*vcsinfo.IndexCommit{
		&vcsinfo.IndexCommit{
			Hash:      "7a669cfa3f4cd3482a4fd03989f75efcc7595f7f",
			Index:     0,
			Timestamp: ts0,
		},
		&vcsinfo.IndexCommit{
			Hash:      "8652a6df7dc8a7e6addee49f6ed3c2308e36bd18",
			Index:     1,
			Timestamp: ts1,
		},
	}

	store = mockPTraceStore{
		traceSet: ptracestore.TraceSet{
			",arch=x86,config=565,":  ptracestore.Trace{1.2, 2.1},
			",arch=x86,config=8888,": ptracestore.Trace{1.3, 3.1},
			",arch=x86,config=gpu,":  ptracestore.Trace{1.4, 4.1},
		},
	}
)

func TestRangeImpl(t *testing.T) {

	expected_headers := []*ColumnHeader{
		&ColumnHeader{
			Source:    "master",
			ID:        "0",
			Desc:      "",
			Timestamp: ts0,
		},
		&ColumnHeader{
			Source:    "master",
			ID:        "1",
			Desc:      "",
			Timestamp: ts1,
		},
	}
	expected_pcommits := []*ptracestore.CommitID{
		&ptracestore.CommitID{
			Offset: 0,
			Source: "master",
		},
		&ptracestore.CommitID{
			Offset: 1,
			Source: "master",
		},
	}

	headers, pcommits := rangeImpl(commits)
	assert.Equal(t, 2, len(headers))
	assert.Equal(t, 2, len(pcommits))
	testutils.AssertDeepEqual(t, expected_headers, headers)
	testutils.AssertDeepEqual(t, expected_pcommits, pcommits)

	headers, pcommits = rangeImpl([]*vcsinfo.IndexCommit{})
	assert.Equal(t, 0, len(headers))
	assert.Equal(t, 0, len(pcommits))
}

func TestNew(t *testing.T) {
	colHeaders := []*ColumnHeader{
		&ColumnHeader{
			Source:    "master",
			ID:        "0",
			Desc:      "",
			Timestamp: ts0,
		},
		&ColumnHeader{
			Source:    "master",
			ID:        "1",
			Desc:      "",
			Timestamp: ts1,
		},
	}
	pcommits := []*ptracestore.CommitID{
		&ptracestore.CommitID{
			Offset: 0,
			Source: "master",
		},
		&ptracestore.CommitID{
			Offset: 1,
			Source: "master",
		},
	}
	store.matchFail = false

	d, err := _new(colHeaders, pcommits, query.Query{}, store)
	assert.NoError(t, err)
	assert.Equal(t, 3, len(d.TraceSet))
	assert.True(t, util.SSliceEqual(d.ParamSet["arch"], []string{"x86"}))
	assert.True(t, util.SSliceEqual(d.ParamSet["config"], []string{"8888", "565", "gpu"}))
}

func TestVCS(t *testing.T) {
	vcs := &mockVcs{
		commits: commits,
	}
	store.matchFail = false

	d, err := New(vcs, store)
	assert.NoError(t, err)
	assert.Equal(t, 3, len(d.TraceSet))

	d, err = NewFromQueryAndRange(vcs, store, ts0, ts1.Add(time.Second), query.Query{})
	assert.NoError(t, err)
	assert.Equal(t, 3, len(d.TraceSet))

	// Test error conditions, i.e. that we log only and don't return an error.
	vcs.updateFail = true
	_, err = New(vcs, store)
	assert.NoError(t, err)
	_, err = NewFromQueryAndRange(vcs, store, ts0, ts1.Add(time.Second), query.Query{})
	assert.NoError(t, err)

	store.matchFail = true
	// Test error conditions if the store fails.
	_, err = New(vcs, store)
	assert.Error(t, err)
	_, err = NewFromQueryAndRange(vcs, store, ts0, ts1.Add(time.Second), query.Query{})
	assert.Error(t, err)
}
<commit_msg>dataframe: work around a bug in go vet.<commit_after>package dataframe

import (
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"go.skia.org/infra/go/query"
	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/vcsinfo"
	"go.skia.org/infra/perf/go/ptracestore"
)

type mockVcs struct {
	commits    []*vcsinfo.IndexCommit
	updateFail bool
}

func (m *mockVcs) From(start time.Time) []string                     { return nil }
func (m *mockVcs) Range(begin, end time.Time) []*vcsinfo.IndexCommit { return nil }
func (m *mockVcs) Details(hash string, includeBranchInfo bool) (*vcsinfo.LongCommit, error) {
	return nil, nil
}

func (m *mockVcs) Update(pull, allBranches bool) error {
	if m.updateFail {
		return fmt.Errorf("Failed to update.")
	}
	return nil
}

func (m *mockVcs) LastNIndex(N int) []*vcsinfo.IndexCommit {
	return m.commits
}

type mockPTraceStore struct {
	traceSet  ptracestore.TraceSet
	matchFail bool
}

func (m mockPTraceStore) Add(commitID *ptracestore.CommitID, values map[string]float32, sourceFile string) error {
	return nil
}

func (m mockPTraceStore) Details(commitID *ptracestore.CommitID, traceID string) (string, float32, error) {
	return "", 0, nil
}

func (m mockPTraceStore) Match(commitIDs []*ptracestore.CommitID, q query.Query) (ptracestore.TraceSet, error) {
	if m.matchFail {
		return nil, fmt.Errorf("Failed to retrieve traces.")
	}
	return m.traceSet, nil
}

var (
	ts0 = time.Unix(1406721642, 0).UTC()
	ts1 = time.Unix(1406721715, 0).UTC()

	commits = []*vcsinfo.IndexCommit{
		&vcsinfo.IndexCommit{
			Hash:      "7a669cfa3f4cd3482a4fd03989f75efcc7595f7f",
			Index:     0,
			Timestamp: ts0,
		},
		&vcsinfo.IndexCommit{
			Hash:      "8652a6df7dc8a7e6addee49f6ed3c2308e36bd18",
			Index:     1,
			Timestamp: ts1,
		},
	}

	store = mockPTraceStore{
		traceSet: ptracestore.TraceSet{
			",arch=x86,config=565,":  ptracestore.Trace([]float32{1.2, 2.1}),
			",arch=x86,config=8888,": ptracestore.Trace([]float32{1.3, 3.1}),
			",arch=x86,config=gpu,":  ptracestore.Trace([]float32{1.4, 4.1}),
		},
	}
)

func TestRangeImpl(t *testing.T) {

	expected_headers := []*ColumnHeader{
		&ColumnHeader{
			Source:    "master",
			ID:        "0",
			Desc:      "",
			Timestamp: ts0,
		},
		&ColumnHeader{
			Source:    "master",
			ID:        "1",
			Desc:      "",
			Timestamp: ts1,
		},
	}
	expected_pcommits := []*ptracestore.CommitID{
		&ptracestore.CommitID{
			Offset: 0,
			Source: "master",
		},
		&ptracestore.CommitID{
			Offset: 1,
			Source: "master",
		},
	}

	headers, pcommits := rangeImpl(commits)
	assert.Equal(t, 2, len(headers))
	assert.Equal(t, 2, len(pcommits))
	testutils.AssertDeepEqual(t, expected_headers, headers)
	testutils.AssertDeepEqual(t, expected_pcommits, pcommits)

	headers, pcommits = rangeImpl([]*vcsinfo.IndexCommit{})
	assert.Equal(t, 0, len(headers))
	assert.Equal(t, 0, len(pcommits))
}

func TestNew(t *testing.T) {
	colHeaders := []*ColumnHeader{
		&ColumnHeader{
			Source:    "master",
			ID:        "0",
			Desc:      "",
			Timestamp: ts0,
		},
		&ColumnHeader{
			Source:    "master",
			ID:        "1",
			Desc:      "",
			Timestamp: ts1,
		},
	}
	pcommits := []*ptracestore.CommitID{
		&ptracestore.CommitID{
			Offset: 0,
			Source: "master",
		},
		&ptracestore.CommitID{
			Offset: 1,
			Source: "master",
		},
	}
	store.matchFail = false

	d, err := _new(colHeaders, pcommits, query.Query{}, store)
	assert.NoError(t, err)
	assert.Equal(t, 3, len(d.TraceSet))
	assert.True(t, util.SSliceEqual(d.ParamSet["arch"], []string{"x86"}))
	assert.True(t, util.SSliceEqual(d.ParamSet["config"], []string{"8888", "565", "gpu"}))
}

func TestVCS(t *testing.T) {
	vcs := &mockVcs{
		commits: commits,
	}
	store.matchFail = false

	d, err := New(vcs, store)
	assert.NoError(t, err)
	assert.Equal(t, 3, len(d.TraceSet))

	d, err = NewFromQueryAndRange(vcs, store, ts0, ts1.Add(time.Second), query.Query{})
	assert.NoError(t, err)
	assert.Equal(t, 3, len(d.TraceSet))

	// Test error conditions, i.e. that we log only and don't return an error.
	vcs.updateFail = true
	_, err = New(vcs, store)
	assert.NoError(t, err)
	_, err = NewFromQueryAndRange(vcs, store, ts0, ts1.Add(time.Second), query.Query{})
	assert.NoError(t, err)

	store.matchFail = true
	// Test error conditions if the store fails.
	_, err = New(vcs, store)
	assert.Error(t, err)
	_, err = NewFromQueryAndRange(vcs, store, ts0, ts1.Add(time.Second), query.Query{})
	assert.Error(t, err)
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"time"

	"cloud.google.com/go/bigquery"

	"go.chromium.org/luci/resultdb/internal/span"
	"go.chromium.org/luci/resultdb/pbutil"
	pb "go.chromium.org/luci/resultdb/proto/rpc/v1"
	typepb "go.chromium.org/luci/resultdb/proto/type"
)

// StringPair is a copy of typepb.StringPair, suitable for representing a
// key:value pair in a BQ table.
// Inferred to be a field of type RECORD with Key and Value string fields.
type StringPair struct {
	Key   string `bigquery:"key"`
	Value string `bigquery:"value"`
}

// Invocation is a subset of pb.Invocation for the invocation fields that need
// to be saved in a BQ table.
type Invocation struct {
	// ID is the ID of the invocation.
	ID string `bigquery:"id"`

	// Interrupted is a flag indicating whether the invocation is interrupted or not.
	// For more details, refer to pb.Invocation.Interrupted.
	Interrupted bool `bigquery:"interrupted"`

	// Tags represents Invocation-level string key-value pairs.
	// A key can be repeated.
	Tags []StringPair `bigquery:"tags"`
}

// TestResultRow represents a row in a BigQuery table for result of a functional
// test case.
type TestResultRow struct {
	// ExportedInvocation contains info of the exported invocation.
	// Note that it's possible that this invocation is not the result's
	// immediate parent invocation, but the including invocation.
	ExportedInvocation Invocation `bigquery:"exported"`

	// ParentInvocation contains info of the result's immediate parent
	// invocation.
	ParentInvocation Invocation `bigquery:"parent"`

	// TestID is a unique identifier of the test in a LUCI project.
	// Refer to pb.TestResult.TestId for details.
	TestID string `bigquery:"test_id"`

	// ResultID identifies a test result in a given invocation and test id.
	ResultID string `bigquery:"result_id"`

	// Variant describes one specific way of running the test,
	//  e.g. a specific bucket, builder and a test suite.
	Variant []StringPair `bigquery:"variant"`

	// Expected is a flag indicating whether the result of test case execution is expected.
	// Refer to pb.TestResult.Expected for details.
	Expected bool `bigquery:"expected"`

	// Status of the test result.
	Status string `bigquery:"status"`

	// SummaryHTML is a human-readable explanation of the result, in HTML.
	SummaryHTML string `bigquery:"summary_html"`

	// StartTime is the point in time when the test case started to execute.
	StartTime time.Time `bigquery:"start_time,nullable"`

	// Duration of the test case execution in seconds.
	Duration float64 `bigquery:"duration,nullable"`

	// Tags contains metadata for this test result.
	// It might describe this particular execution or the test case.
	Tags []StringPair `bigquery:"tags"`

	// If the failures of the test variant are exonerated.
	// Note: the exoneration is at the test variant level, not result level.
	Exonerated bool `bigquery:"exonerated"`
}

// stringPairProtosToStringPairs returns a slice of StringPair derived from *typepb.StringPair.
func stringPairProtosToStringPairs(pairs []*typepb.StringPair) []StringPair {
	if len(pairs) == 0 {
		return nil
	}

	sp := make([]StringPair, len(pairs))
	for i, p := range pairs {
		sp[i] = StringPair{
			Key:   p.Key,
			Value: p.Value,
		}
	}
	return sp
}

// variantToStringPairs returns a slice of StringPair derived from *typepb.Variant.
func variantToStringPairs(vr *typepb.Variant) []StringPair {
	defMap := vr.GetDef()
	if len(defMap) == 0 {
		return nil
	}

	keys := pbutil.SortedVariantKeys(vr)
	sp := make([]StringPair, len(keys))
	for i, k := range keys {
		sp[i] = StringPair{
			Key:   k,
			Value: defMap[k],
		}
	}
	return sp
}

func invocationProtoToInvocation(inv *pb.Invocation) Invocation {
	return Invocation{
		ID:          string(span.MustParseInvocationName(inv.Name)),
		Interrupted: inv.Interrupted,
		Tags:        stringPairProtosToStringPairs(inv.Tags),
	}
}

// generateBQRow returns a *bigquery.StructSaver to be inserted into BQ.
func generateBQRow(exported, parent *pb.Invocation, tr *pb.TestResult, exonerated bool) *bigquery.StructSaver {
	trr := &TestResultRow{
		ExportedInvocation: invocationProtoToInvocation(exported),
		ParentInvocation:   invocationProtoToInvocation(parent),
		TestID:             tr.TestId,
		ResultID:           tr.ResultId,
		Variant:            variantToStringPairs(tr.Variant),
		Expected:           tr.Expected,
		Status:             tr.Status.String(),
		SummaryHTML:        tr.SummaryHtml,
		Tags:               stringPairProtosToStringPairs(tr.Tags),
		Exonerated:         exonerated,
	}

	if tr.StartTime != nil {
		trr.StartTime = pbutil.MustTimestamp(tr.StartTime)
	}

	if tr.Duration != nil {
		trr.Duration = pbutil.MustDuration(tr.Duration).Seconds()
	}

	return &bigquery.StructSaver{
		InsertID: tr.Name,
		Struct:   trr,
	}
}
<commit_msg>[resultdb] BigQuery Export: Use NullXXX types for nullable fields.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"cloud.google.com/go/bigquery"

	"go.chromium.org/luci/resultdb/internal/span"
	"go.chromium.org/luci/resultdb/pbutil"
	pb "go.chromium.org/luci/resultdb/proto/rpc/v1"
	typepb "go.chromium.org/luci/resultdb/proto/type"
)

// StringPair is a copy of typepb.StringPair, suitable for representing a
// key:value pair in a BQ table.
// Inferred to be a field of type RECORD with Key and Value string fields.
type StringPair struct {
	Key   string `bigquery:"key"`
	Value string `bigquery:"value"`
}

// Invocation is a subset of pb.Invocation for the invocation fields that need
// to be saved in a BQ table.
type Invocation struct {
	// ID is the ID of the invocation.
	ID string `bigquery:"id"`

	// Interrupted is a flag indicating whether the invocation is interrupted or not.
	// For more details, refer to pb.Invocation.Interrupted.
	Interrupted bool `bigquery:"interrupted"`

	// Tags represents Invocation-level string key-value pairs.
	// A key can be repeated.
	Tags []StringPair `bigquery:"tags"`
}

// TestResultRow represents a row in a BigQuery table for result of a functional
// test case.
type TestResultRow struct {
	// ExportedInvocation contains info of the exported invocation.
	// Note that it's possible that this invocation is not the result's
	// immediate parent invocation, but the including invocation.
	ExportedInvocation Invocation `bigquery:"exported"`

	// ParentInvocation contains info of the result's immediate parent
	// invocation.
	ParentInvocation Invocation `bigquery:"parent"`

	// TestID is a unique identifier of the test in a LUCI project.
	// Refer to pb.TestResult.TestId for details.
	TestID string `bigquery:"test_id"`

	// ResultID identifies a test result in a given invocation and test id.
	ResultID string `bigquery:"result_id"`

	// Variant describes one specific way of running the test,
	//  e.g. a specific bucket, builder and a test suite.
	Variant []StringPair `bigquery:"variant"`

	// Expected is a flag indicating whether the result of test case execution is expected.
	// Refer to pb.TestResult.Expected for details.
	Expected bool `bigquery:"expected"`

	// Status of the test result.
	Status string `bigquery:"status"`

	// SummaryHTML is a human-readable explanation of the result, in HTML.
	SummaryHTML string `bigquery:"summary_html"`

	// StartTime is the point in time when the test case started to execute.
	StartTime bigquery.NullTimestamp `bigquery:"start_time"`

	// Duration of the test case execution in seconds.
	Duration bigquery.NullFloat64 `bigquery:"duration"`

	// Tags contains metadata for this test result.
	// It might describe this particular execution or the test case.
	Tags []StringPair `bigquery:"tags"`

	// If the failures of the test variant are exonerated.
	// Note: the exoneration is at the test variant level, not result level.
	Exonerated bool `bigquery:"exonerated"`
}

// stringPairProtosToStringPairs returns a slice of StringPair derived from *typepb.StringPair.
func stringPairProtosToStringPairs(pairs []*typepb.StringPair) []StringPair {
	if len(pairs) == 0 {
		return nil
	}

	sp := make([]StringPair, len(pairs))
	for i, p := range pairs {
		sp[i] = StringPair{
			Key:   p.Key,
			Value: p.Value,
		}
	}
	return sp
}

// variantToStringPairs returns a slice of StringPair derived from *typepb.Variant.
func variantToStringPairs(vr *typepb.Variant) []StringPair {
	defMap := vr.GetDef()
	if len(defMap) == 0 {
		return nil
	}

	keys := pbutil.SortedVariantKeys(vr)
	sp := make([]StringPair, len(keys))
	for i, k := range keys {
		sp[i] = StringPair{
			Key:   k,
			Value: defMap[k],
		}
	}
	return sp
}

func invocationProtoToInvocation(inv *pb.Invocation) Invocation {
	return Invocation{
		ID:          string(span.MustParseInvocationName(inv.Name)),
		Interrupted: inv.Interrupted,
		Tags:        stringPairProtosToStringPairs(inv.Tags),
	}
}

// generateBQRow returns a *bigquery.StructSaver to be inserted into BQ.
func generateBQRow(exported, parent *pb.Invocation, tr *pb.TestResult, exonerated bool) *bigquery.StructSaver {
	trr := &TestResultRow{
		ExportedInvocation: invocationProtoToInvocation(exported),
		ParentInvocation:   invocationProtoToInvocation(parent),
		TestID:             tr.TestId,
		ResultID:           tr.ResultId,
		Variant:            variantToStringPairs(tr.Variant),
		Expected:           tr.Expected,
		Status:             tr.Status.String(),
		SummaryHTML:        tr.SummaryHtml,
		Tags:               stringPairProtosToStringPairs(tr.Tags),
		Exonerated:         exonerated,
	}

	if tr.StartTime != nil {
		trr.StartTime = bigquery.NullTimestamp{
			Timestamp: pbutil.MustTimestamp(tr.StartTime),
			Valid: true,
		}
	}

	if tr.Duration != nil {
		trr.Duration = bigquery.NullFloat64{
			Float64: pbutil.MustDuration(tr.Duration).Seconds(),
			Valid: true,
		}
	}

	return &bigquery.StructSaver{
		InsertID: tr.Name,
		Struct:   trr,
	}
}
<|endoftext|>


<commit_before>package plugin

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"
	"strings"

	"github.com/neovim/go-client/nvim"
)

// Plugin represents a remote plugin.
type Plugin struct {
	Nvim        *nvim.Nvim
	pluginSpecs []*pluginSpec

	// Event/pattern counters used to generate unique paths for autocmds.
	eventPathCounts map[string]int
}

// New returns an intialized plugin.
func New(v *nvim.Nvim) *Plugin {
	p := &Plugin{
		Nvim:            v,
		eventPathCounts: make(map[string]int),
	}

	// Disable support for "specs" method until path mechanism for supporting
	// binary exectables with Nvim is worked out.
	// err := v.RegisterHandler("specs", func(path string) ([]*pluginSpec, error) {
	//  return p.pluginSpecs, nil
	// })

	return p
}

type pluginSpec struct {
	sm   string
	Type string            `msgpack:"type"`
	Name string            `msgpack:"name"`
	Sync bool              `msgpack:"sync"`
	Opts map[string]string `msgpack:"opts"`
}

func (spec *pluginSpec) path() string {
	if i := strings.Index(spec.sm, ":"); i > 0 {
		return spec.sm[:i]
	}
	return ""
}

func isSync(f interface{}) bool {
	t := reflect.TypeOf(f)
	return t.Kind() == reflect.Func && t.NumOut() > 0
}

func (p *Plugin) handle(fn interface{}, spec *pluginSpec) {
	p.pluginSpecs = append(p.pluginSpecs, spec)
	if p.Nvim == nil {
		return
	}
	if err := p.Nvim.RegisterHandler(spec.sm, fn); err != nil {
		panic(err)
	}
}

// Handle registers fn as a MessagePack RPC handler for the specified method
// name. The function signature for fn is one of
//
//  func([v *nvim.Nvim,] {args}) ({resultType}, error)
//  func([v *nvim.Nvim,] {args}) error
//  func([v *nvim.Nvim,] {args})
//
// where {args} is zero or more arguments and {resultType} is the type of of a
// return value. Call the handler from Nvim using the rpcnotify and rpcrequest
// functions:
//
//  :help rpcrequest()
//  :help rpcnotify()
func (p *Plugin) Handle(method string, fn interface{}) {
	if p.Nvim == nil {
		return
	}
	if err := p.Nvim.RegisterHandler(method, fn); err != nil {
		panic(err)
	}
}

// FunctionOptions specifies function options.
type FunctionOptions struct {
	// Name is the name of the function in Nvim. The name must be made of
	// alphanumeric characters and '_', and must start with a capital letter.
	Name string

	// Eval is an expression evaluated in Nvim. The result is passed the
	// handler function.
	Eval string
}

// HandleFunction registers fn as a handler for a Nvim function. The function
// signature for fn is one of
//
//  func([v *nvim.Nvim,] args {arrayType} [, eval {evalType}]) ({resultType}, error)
//  func([v *nvim.Nvim,] args {arrayType} [, eval {evalType}]) error
//
// where {arrayType} is a type that can be unmarshaled from a MessagePack
// array, {evalType} is a type compatible with the Eval option expression and
// {resultType} is the type of function result.
//
// If options.Eval == "*", then HandleFunction constructs the expression to
// evaluate in Nvim from the type of fn's last argument. The last argument is
// assumed to be a pointer to a struct type with 'eval' field tags set to the
// expression to evaluate for each field. Nested structs are supported. The
// expression for the function
//
//  func example(eval *struct{
//  	GOPATH string `eval:"$GOPATH"`
//  	Cwd    string `eval:"getcwd()"`
//  })
//
// is
//
//  {'GOPATH': $GOPATH, Cwd: getcwd()}
func (p *Plugin) HandleFunction(options *FunctionOptions, fn interface{}) {
	m := make(map[string]string)
	if options.Eval != "" {
		m[`eval`] = eval(options.Eval, fn)
	}
	p.handle(fn, &pluginSpec{
		sm:   `0:function:` + options.Name,
		Type: `function`,
		Name: options.Name,
		Sync: isSync(fn),
		Opts: m,
	})
}

// CommandOptions specifies command options.
type CommandOptions struct {
	// Name is the name of the command in Nvim. The name must be made of
	// alphanumeric characters and '_', and must start with a capital
	// letter.
	Name string

	// NArgs specifies the number command arguments.
	//
	//  0   No arguments are allowed
	//  1   Exactly one argument is required, it includes spaces
	//  *   Any number of arguments are allowed (0, 1, or many),
	//      separated by white space
	//  ?   0 or 1 arguments are allowed
	//  +   Arguments must be supplied, but any number are allowed
	NArgs string

	// Range specifies that the command accepts a range.
	//
	//  .   Range allowed, default is current line. The value
	//      "." is converted to "" for Nvim.
	//  %   Range allowed, default is whole file (1,$)
	//  N   A count (default N) which is specified in the line
	//      number position (like |:split|); allows for zero line
	//	    number.
	//
	//  :help :command-range
	Range string

	// Count specfies that thecommand accepts a count.
	//
	//  N   A count (default N) which is specified either in the line
	//	    number position, or as an initial argument (like |:Next|).
	//      Specifying -count (without a default) acts like -count=0
	//
	//  :help :command-count
	Count string

	// Addr sepcifies the domain for the range option
	//
	//  lines           Range of lines (this is the default)
	//  arguments       Range for arguments
	//  buffers         Range for buffers (also not loaded buffers)
	//  loaded_buffers  Range for loaded buffers
	//  windows         Range for windows
	//  tabs            Range for tab pages
	//
	//  :help command-addr
	Addr string

	// Eval is evaluated in Nvim and the result is passed as an argument.
	Eval string

	// Complete specifies command completion.
	//
	//  :help :command-complete
	Complete string

	// Bang specifies that the command can take a ! modifier (like :q or :w).
	Bang bool

	// Register specifes that the first argument to the command can be an
	// optional register name (like :del, :put, :yank).
	Register bool

	// Bar specifies that the command can be followed by a "|" and another
	// command.  A "|" inside the command argument is not allowed then. Also
	// checks for a " to start a comment.
	Bar bool
}

// HandleCommand registers fn as a handler for a Nvim command. The arguments
// to the function fn are:
//
//  v *nvim.Nvim        optional
//  args []string       when options.NArgs != ""
//  range [2]int        when options.Range == "." or Range == "%"
//  range int           when options.Range == N or Count != ""
//  bang bool           when options.Bang == true
//  register string     when options.Register == true
//  eval interface{}    when options.Eval != ""
//
// The function fn must return an error.
//
// If options.Eval == "*", then HandleCommand constructs the expression to
// evaluate in Nvim from the type of fn's last argument. See the
// HandleFunction documentation for information on how the expression is
// generated.
func (p *Plugin) HandleCommand(options *CommandOptions, fn interface{}) {
	m := make(map[string]string)

	if options.NArgs != "" {
		m[`nargs`] = options.NArgs
	}

	if options.Range != "" {
		if options.Range == `.` {
			options.Range = ""
		}
		m[`range`] = options.Range
	} else if options.Count != "" {
		m[`count`] = options.Count
	}

	if options.Bang {
		m[`bang`] = ""
	}

	if options.Register {
		m[`register`] = ""
	}

	if options.Eval != "" {
		m[`eval`] = eval(options.Eval, fn)
	}

	if options.Addr != "" {
		m[`addr`] = options.Addr
	}

	if options.Bar {
		m[`bar`] = ""
	}

	if options.Complete != "" {
		m[`complete`] = options.Complete
	}

	p.handle(fn, &pluginSpec{
		sm:   `0:command:` + options.Name,
		Type: `command`,
		Name: options.Name,
		Sync: isSync(fn),
		Opts: m,
	})
}

// AutocmdOptions specifies autocmd options.
type AutocmdOptions struct {
	// Event is the event name.
	Event string

	// Group specifies the autocmd group.
	Group string

	// Pattern specifies an autocmd pattern.
	//
	//  :help autocmd-patterns
	Pattern string

	// Nested allows nested autocmds.
	//
	//  :help autocmd-nested
	Nested bool

	// Once supplys the command is executed once, then removed ("one shot").
	//
	//  :help autocmd-once
	Once bool

	// Eval is evaluated in Nvim and the result is passed the the handler
	// function.
	Eval string
}

// HandleAutocmd registers fn as a handler an autocmnd event.
//
// If options.Eval == "*", then HandleAutocmd constructs the expression to
// evaluate in Nvim from the type of fn's last argument. See the HandleFunction
// documentation for information on how the expression is generated.
func (p *Plugin) HandleAutocmd(options *AutocmdOptions, fn interface{}) {
	pattern := ""
	m := make(map[string]string)
	if options.Group != "" {
		m[`group`] = options.Group
	}
	if options.Pattern != "" {
		m[`pattern`] = options.Pattern
		pattern = options.Pattern
	}
	if options.Nested {
		m[`nested`] = "1"
	}
	if options.Once {
		m[`once`] = "1"
	}
	if options.Eval != "" {
		m[`eval`] = eval(options.Eval, fn)
	}

	// Compute unique path for event and pattern.
	ep := options.Event + ":" + pattern
	i := p.eventPathCounts[ep]
	p.eventPathCounts[ep] = i + 1
	sm := fmt.Sprintf(`%d:autocmd:%s`, i, ep)

	p.handle(fn, &pluginSpec{
		sm:   sm,
		Type: `autocmd`,
		Name: options.Event,
		Sync: isSync(fn),
		Opts: m,
	})
}

// RegisterForTests registers the plugin with Nvim. Use this method for testing
// plugins in an embedded instance of Nvim.
func (p *Plugin) RegisterForTests() error {
	specs := make(map[string][]*pluginSpec)
	for _, spec := range p.pluginSpecs {
		specs[spec.path()] = append(specs[spec.path()], spec)
	}
	const host = "nvim-go-test"
	for path, specs := range specs {
		if err := p.Nvim.Call(`remote#host#RegisterPlugin`, nil, host, path, specs); err != nil {
			return err
		}
	}
	err := p.Nvim.Call(`remote#host#Register`, nil, host, `x`, p.Nvim.ChannelID())

	return err
}

func eval(eval string, f interface{}) string {
	if eval != `*` {
		return eval
	}
	ft := reflect.TypeOf(f)
	if ft.Kind() != reflect.Func || ft.NumIn() < 1 {
		panic(`Eval: "*" option requires function with at least one argument`)
	}
	argt := ft.In(ft.NumIn() - 1)
	if argt.Kind() != reflect.Ptr || argt.Elem().Kind() != reflect.Struct {
		panic(`Eval: "*" option requires function with pointer to struct as last argument`)
	}
	return structEval(argt.Elem())
}

func structEval(t reflect.Type) string {
	buf := []byte{'{'}
	sep := ""
	for i := 0; i < t.NumField(); i++ {
		sf := t.Field(i)
		if sf.Anonymous {
			panic(`Eval: "*" does not support anonymous fields`)
		}

		eval := sf.Tag.Get("eval")
		if eval == "" {
			ft := sf.Type
			if ft.Kind() == reflect.Ptr {
				ft = ft.Elem()
			}
			if ft.Kind() == reflect.Struct {
				eval = structEval(ft)
			}
		}

		if eval == "" {
			continue
		}

		name := strings.Split(sf.Tag.Get("msgpack"), ",")[0]
		if name == "" {
			name = sf.Name
		}

		buf = append(buf, sep...)
		buf = append(buf, `'`...)
		buf = append(buf, name...)
		buf = append(buf, `':`...)
		buf = append(buf, eval...)
		sep = `, `
	}
	buf = append(buf, '}')
	return string(buf)
}

type byServiceMethod []*pluginSpec

func (a byServiceMethod) Len() int           { return len(a) }
func (a byServiceMethod) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byServiceMethod) Less(i, j int) bool { return a[i].sm < a[j].sm }

func (p *Plugin) Manifest(host string) []byte {
	var buf bytes.Buffer

	// Sort for consistent order on output.
	sort.Sort(byServiceMethod(p.pluginSpecs))
	escape := strings.NewReplacer(`'`, `''`).Replace

	prevPath := ""
	for _, spec := range p.pluginSpecs {
		path := spec.path()
		if path != prevPath {
			if prevPath != "" {
				fmt.Fprintf(&buf, `\\ )`)
			}
			fmt.Fprintf(&buf, `call remote#host#RegisterPlugin('%s', '%s', [\n`, host, path)
			prevPath = path
		}

		sync := `0`
		if spec.Sync {
			sync = `1`
		}

		fmt.Fprintf(&buf, `\\ {'type': '%s', 'name': '%s', 'sync': %s, 'opts': {`, spec.Type, spec.Name, sync)

		var keys []string
		for k := range spec.Opts {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		optDelim := ""
		for _, k := range keys {
			fmt.Fprintf(&buf, `%s'%s': '%s'`, optDelim, k, escape(spec.Opts[k]))
			optDelim = `,`
		}

		fmt.Fprintf(&buf, `}},\n`)
	}
	if prevPath != "" {
		fmt.Fprintf(&buf, `\\ ])\n`)
	}
	return buf.Bytes()
}
<commit_msg>nvim/plugin: cleanup and optimize eval<commit_after>package plugin

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"
	"strings"

	"github.com/neovim/go-client/nvim"
)

// Plugin represents a remote plugin.
type Plugin struct {
	Nvim        *nvim.Nvim
	pluginSpecs []*pluginSpec

	// Event/pattern counters used to generate unique paths for autocmds.
	eventPathCounts map[string]int
}

// New returns an intialized plugin.
func New(v *nvim.Nvim) *Plugin {
	p := &Plugin{
		Nvim:            v,
		eventPathCounts: make(map[string]int),
	}

	// Disable support for "specs" method until path mechanism for supporting
	// binary exectables with Nvim is worked out.
	// err := v.RegisterHandler("specs", func(path string) ([]*pluginSpec, error) {
	//  return p.pluginSpecs, nil
	// })

	return p
}

type pluginSpec struct {
	sm   string
	Type string            `msgpack:"type"`
	Name string            `msgpack:"name"`
	Sync bool              `msgpack:"sync"`
	Opts map[string]string `msgpack:"opts"`
}

func (spec *pluginSpec) path() string {
	if i := strings.Index(spec.sm, ":"); i > 0 {
		return spec.sm[:i]
	}

	return ""
}

func isSync(f interface{}) bool {
	t := reflect.TypeOf(f)

	return t.Kind() == reflect.Func && t.NumOut() > 0
}

func (p *Plugin) handle(fn interface{}, spec *pluginSpec) {
	p.pluginSpecs = append(p.pluginSpecs, spec)
	if p.Nvim == nil {
		return
	}

	if err := p.Nvim.RegisterHandler(spec.sm, fn); err != nil {
		panic(err)
	}
}

// Handle registers fn as a MessagePack RPC handler for the specified method
// name. The function signature for fn is one of
//
//  func([v *nvim.Nvim,] {args}) ({resultType}, error)
//  func([v *nvim.Nvim,] {args}) error
//  func([v *nvim.Nvim,] {args})
//
// where {args} is zero or more arguments and {resultType} is the type of of a
// return value. Call the handler from Nvim using the rpcnotify and rpcrequest
// functions:
//
//  :help rpcrequest()
//  :help rpcnotify()
func (p *Plugin) Handle(method string, fn interface{}) {
	if p.Nvim == nil {
		return
	}

	if err := p.Nvim.RegisterHandler(method, fn); err != nil {
		panic(err)
	}
}

// FunctionOptions specifies function options.
type FunctionOptions struct {
	// Name is the name of the function in Nvim. The name must be made of
	// alphanumeric characters and '_', and must start with a capital letter.
	Name string

	// Eval is an expression evaluated in Nvim. The result is passed the
	// handler function.
	Eval string
}

// HandleFunction registers fn as a handler for a Nvim function. The function
// signature for fn is one of
//
//  func([v *nvim.Nvim,] args {arrayType} [, eval {evalType}]) ({resultType}, error)
//  func([v *nvim.Nvim,] args {arrayType} [, eval {evalType}]) error
//
// where {arrayType} is a type that can be unmarshaled from a MessagePack
// array, {evalType} is a type compatible with the Eval option expression and
// {resultType} is the type of function result.
//
// If options.Eval == "*", then HandleFunction constructs the expression to
// evaluate in Nvim from the type of fn's last argument. The last argument is
// assumed to be a pointer to a struct type with 'eval' field tags set to the
// expression to evaluate for each field. Nested structs are supported. The
// expression for the function
//
//  func example(eval *struct{
//  	GOPATH string `eval:"$GOPATH"`
//  	Cwd    string `eval:"getcwd()"`
//  })
//
// is
//
//  {'GOPATH': $GOPATH, Cwd: getcwd()}
func (p *Plugin) HandleFunction(options *FunctionOptions, fn interface{}) {
	m := make(map[string]string)

	if options.Eval != "" {
		m["eval"] = eval(options.Eval, fn)
	}

	p.handle(fn, &pluginSpec{
		sm:   `0:function:` + options.Name,
		Type: `function`,
		Name: options.Name,
		Sync: isSync(fn),
		Opts: m,
	})
}

// CommandOptions specifies command options.
type CommandOptions struct {
	// Name is the name of the command in Nvim. The name must be made of
	// alphanumeric characters and '_', and must start with a capital
	// letter.
	Name string

	// NArgs specifies the number command arguments.
	//
	//  0   No arguments are allowed
	//  1   Exactly one argument is required, it includes spaces
	//  *   Any number of arguments are allowed (0, 1, or many),
	//      separated by white space
	//  ?   0 or 1 arguments are allowed
	//  +   Arguments must be supplied, but any number are allowed
	NArgs string

	// Range specifies that the command accepts a range.
	//
	//  .   Range allowed, default is current line. The value
	//      "." is converted to "" for Nvim.
	//  %   Range allowed, default is whole file (1,$)
	//  N   A count (default N) which is specified in the line
	//      number position (like |:split|); allows for zero line
	//	    number.
	//
	//  :help :command-range
	Range string

	// Count specfies that thecommand accepts a count.
	//
	//  N   A count (default N) which is specified either in the line
	//	    number position, or as an initial argument (like |:Next|).
	//      Specifying -count (without a default) acts like -count=0
	//
	//  :help :command-count
	Count string

	// Addr sepcifies the domain for the range option
	//
	//  lines           Range of lines (this is the default)
	//  arguments       Range for arguments
	//  buffers         Range for buffers (also not loaded buffers)
	//  loaded_buffers  Range for loaded buffers
	//  windows         Range for windows
	//  tabs            Range for tab pages
	//
	//  :help command-addr
	Addr string

	// Eval is evaluated in Nvim and the result is passed as an argument.
	Eval string

	// Complete specifies command completion.
	//
	//  :help :command-complete
	Complete string

	// Bang specifies that the command can take a ! modifier (like :q or :w).
	Bang bool

	// Register specifes that the first argument to the command can be an
	// optional register name (like :del, :put, :yank).
	Register bool

	// Bar specifies that the command can be followed by a "|" and another
	// command.  A "|" inside the command argument is not allowed then. Also
	// checks for a " to start a comment.
	Bar bool
}

// HandleCommand registers fn as a handler for a Nvim command. The arguments
// to the function fn are:
//
//  v *nvim.Nvim        optional
//  args []string       when options.NArgs != ""
//  range [2]int        when options.Range == "." or Range == "%"
//  range int           when options.Range == N or Count != ""
//  bang bool           when options.Bang == true
//  register string     when options.Register == true
//  eval interface{}    when options.Eval != ""
//
// The function fn must return an error.
//
// If options.Eval == "*", then HandleCommand constructs the expression to
// evaluate in Nvim from the type of fn's last argument. See the
// HandleFunction documentation for information on how the expression is
// generated.
func (p *Plugin) HandleCommand(options *CommandOptions, fn interface{}) {
	m := make(map[string]string)

	if options.NArgs != "" {
		m[`nargs`] = options.NArgs
	}

	switch {
	case options.Range == `.`:
		options.Range = ""
		fallthrough
	case options.Range != "":
		m[`range`] = options.Range
	case options.Count != "":
		m[`count`] = options.Count
	}

	if options.Bang {
		m[`bang`] = ""
	}

	if options.Register {
		m[`register`] = ""
	}

	if options.Eval != "" {
		m[`eval`] = eval(options.Eval, fn)
	}

	if options.Addr != "" {
		m[`addr`] = options.Addr
	}

	if options.Bar {
		m[`bar`] = ""
	}

	if options.Complete != "" {
		m[`complete`] = options.Complete
	}

	p.handle(fn, &pluginSpec{
		sm:   `0:command:` + options.Name,
		Type: `command`,
		Name: options.Name,
		Sync: isSync(fn),
		Opts: m,
	})
}

// AutocmdOptions specifies autocmd options.
type AutocmdOptions struct {
	// Event is the event name.
	Event string

	// Group specifies the autocmd group.
	Group string

	// Pattern specifies an autocmd pattern.
	//
	//  :help autocmd-patterns
	Pattern string

	// Nested allows nested autocmds.
	//
	//  :help autocmd-nested
	Nested bool

	// Once supplys the command is executed once, then removed ("one shot").
	//
	//  :help autocmd-once
	Once bool

	// Eval is evaluated in Nvim and the result is passed the the handler
	// function.
	Eval string
}

// HandleAutocmd registers fn as a handler an autocmnd event.
//
// If options.Eval == "*", then HandleAutocmd constructs the expression to
// evaluate in Nvim from the type of fn's last argument. See the HandleFunction
// documentation for information on how the expression is generated.
func (p *Plugin) HandleAutocmd(options *AutocmdOptions, fn interface{}) {
	pattern := ""

	m := make(map[string]string)

	if options.Group != "" {
		m[`group`] = options.Group
	}

	if options.Pattern != "" {
		m[`pattern`] = options.Pattern
		pattern = options.Pattern
	}

	if options.Nested {
		m[`nested`] = `1`
	}

	if options.Once {
		m[`once`] = `1`
	}

	if options.Eval != "" {
		m[`eval`] = eval(options.Eval, fn)
	}

	// Compute unique path for event and pattern.
	ep := options.Event + ":" + pattern
	i := p.eventPathCounts[ep]
	p.eventPathCounts[ep] = i + 1

	sm := fmt.Sprintf(`%d:autocmd:%s`, i, ep)

	p.handle(fn, &pluginSpec{
		sm:   sm,
		Type: `autocmd`,
		Name: options.Event,
		Sync: isSync(fn),
		Opts: m,
	})
}

// RegisterForTests registers the plugin with Nvim. Use this method for testing
// plugins in an embedded instance of Nvim.
func (p *Plugin) RegisterForTests() error {
	specs := make(map[string][]*pluginSpec)
	for _, spec := range p.pluginSpecs {
		specs[spec.path()] = append(specs[spec.path()], spec)
	}

	const host = "nvim-go-test"
	for path, specs := range specs {
		if err := p.Nvim.Call(`remote#host#RegisterPlugin`, nil, host, path, specs); err != nil {
			return err
		}
	}
	err := p.Nvim.Call(`remote#host#Register`, nil, host, `x`, p.Nvim.ChannelID())

	return err
}

func eval(eval string, f interface{}) string {
	if eval != `*` {
		return eval
	}

	ft := reflect.TypeOf(f)
	if ft.Kind() != reflect.Func || ft.NumIn() < 1 {
		panic(`Eval: "*" option requires function with at least one argument`)
	}

	argt := ft.In(ft.NumIn() - 1)
	if argt.Kind() != reflect.Ptr || argt.Elem().Kind() != reflect.Struct {
		panic(`Eval: "*" option requires function with pointer to struct as last argument`)
	}

	return structEval(argt.Elem())
}

func structEval(t reflect.Type) string {
	var sb strings.Builder

	sb.WriteByte('{')
	sep := ""

	for i := 0; i < t.NumField(); i++ {
		sf := t.Field(i)
		if sf.Anonymous {
			panic(`Eval: "*" does not support anonymous fields`)
		}

		eval := sf.Tag.Get("eval")
		if eval == "" {
			ft := sf.Type
			if ft.Kind() == reflect.Ptr {
				ft = ft.Elem()
			}

			if ft.Kind() == reflect.Struct {
				eval = structEval(ft)
			}
		}
		if eval == "" {
			continue
		}

		name := strings.Split(sf.Tag.Get("msgpack"), ",")[0]
		if name == "" {
			name = sf.Name
		}

		sb.WriteString(sep)
		sb.WriteByte('\'')
		sb.WriteString(name)
		sb.WriteString("': ")
		sb.WriteString(eval)
		sep = ", "
	}
	sb.WriteByte('}')

	return sb.String()
}

type byServiceMethod []*pluginSpec

func (a byServiceMethod) Len() int           { return len(a) }
func (a byServiceMethod) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byServiceMethod) Less(i, j int) bool { return a[i].sm < a[j].sm }

func (p *Plugin) Manifest(host string) []byte {
	var buf bytes.Buffer

	// Sort for consistent order on output.
	sort.Sort(byServiceMethod(p.pluginSpecs))
	escape := strings.NewReplacer(`'`, `''`).Replace

	prevPath := ""
	for _, spec := range p.pluginSpecs {
		path := spec.path()
		if path != prevPath {
			if prevPath != "" {
				fmt.Fprintf(&buf, "\\ )")
			}
			fmt.Fprintf(&buf, "call remote#host#RegisterPlugin('%s', '%s', [\n", host, path)
			prevPath = path
		}

		sync := "0"
		if spec.Sync {
			sync = "1"
		}

		fmt.Fprintf(&buf, "\\ {'type': '%s', 'name': '%s', 'sync': %s, 'opts': {", spec.Type, spec.Name, sync)

		var keys []string
		for k := range spec.Opts {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		optDelim := ""
		for _, k := range keys {
			fmt.Fprintf(&buf, "%s'%s': '%s'", optDelim, k, escape(spec.Opts[k]))
			optDelim = ","
		}

		fmt.Fprintf(&buf, "}},\n")
	}
	if prevPath != "" {
		fmt.Fprintf(&buf, "\\ ])\n")
	}

	return buf.Bytes()
}
<|endoftext|>


<commit_before>package cmdtest

import (
	"bytes"
	"fmt"
	"io"
	"regexp"
	"sync"
	"time"
)

type Expector struct {
	output         io.Reader
	defaultTimeout time.Duration

	outputError chan error
	listen      chan bool

	offset int
	buffer *bytes.Buffer

	sync.RWMutex
}

type ExpectationFailed struct {
	Wanted string
	Got    string
}

func (e ExpectationFailed) Error() string {
	return fmt.Sprintf("Expected to see '%s', got: %#v", e.Wanted, e.Got)
}

func NewExpector(out io.Reader, defaultTimeout time.Duration) *Expector {
	e := &Expector{
		output:         out,
		defaultTimeout: defaultTimeout,

		outputError: make(chan error),
		listen:      make(chan bool),

		buffer: new(bytes.Buffer),
	}

	go e.monitor()

	return e
}

func (e *Expector) Expect(pattern string) error {
	return e.ExpectWithTimeout(pattern, e.defaultTimeout)
}

func (e *Expector) ExpectWithTimeout(pattern string, timeout time.Duration) error {
	regexp, err := regexp.Compile(pattern)
	if err != nil {
		return err
	}

	cancel := make(chan bool, 1)

	select {
	case <-e.match(regexp, cancel):
		return nil
	case err := <-e.outputError:
		return err
	case <-time.After(timeout):
		cancel <- true

		return ExpectationFailed{
			Wanted: pattern,
			Got:    string(e.nextOutput()),
		}
	}
}

func (e *Expector) match(regexp *regexp.Regexp, cancel chan bool) chan bool {
	matchResult := make(chan bool)

	go func() {
		for {
			found := regexp.FindIndex(e.nextOutput())
			if found != nil {
				e.forwardOutput(found[1])
				matchResult <- true
				break
			}

			select {
			case <-e.listen:
			case <-time.After(100 * time.Millisecond):
			case <-cancel:
				return
			}
		}
	}()

	return matchResult
}

func (e *Expector) monitor() {
	var buf [1024]byte

	for {
		read, err := e.output.Read(buf[:])

		if read > 0 {
			e.addOutput(buf[:read])
		}

		e.notify()

		if err != nil {
			e.outputError <- err
			break
		}
	}
}

func (e *Expector) addOutput(out []byte) {
	e.Lock()
	defer e.Unlock()

	e.buffer.Write(out)
}

func (e *Expector) forwardOutput(count int) {
	e.Lock()
	defer e.Unlock()

	e.buffer.Next(count)
}

func (e *Expector) nextOutput() []byte {
	e.RLock()
	defer e.RUnlock()

	return e.buffer.Bytes()
}

func (e *Expector) notify() {
	select {
	case e.listen <- true:
	default:
	}
}
<commit_msg>expect: put full output in error message<commit_after>package cmdtest

import (
	"bytes"
	"fmt"
	"io"
	"regexp"
	"sync"
	"time"
)

type Expector struct {
	output         io.Reader
	defaultTimeout time.Duration

	outputError chan error
	listen      chan bool

	offset     int
	buffer     *bytes.Buffer
	fullBuffer *bytes.Buffer

	sync.RWMutex
}

type ExpectationFailed struct {
	Wanted string
	Next   string
	Output string
}

func (e ExpectationFailed) Error() string {
	return fmt.Sprintf("Expected to see '%s', got stuck at: %#v.\n\nFull output:\n\n%s", e.Wanted, e.Next, e.Output)
}

func NewExpector(out io.Reader, defaultTimeout time.Duration) *Expector {
	e := &Expector{
		output:         out,
		defaultTimeout: defaultTimeout,

		outputError: make(chan error),
		listen:      make(chan bool),

		buffer:     new(bytes.Buffer),
		fullBuffer: new(bytes.Buffer),
	}

	go e.monitor()

	return e
}

func (e *Expector) Expect(pattern string) error {
	return e.ExpectWithTimeout(pattern, e.defaultTimeout)
}

func (e *Expector) ExpectWithTimeout(pattern string, timeout time.Duration) error {
	regexp, err := regexp.Compile(pattern)
	if err != nil {
		return err
	}

	cancel := make(chan bool, 1)

	select {
	case <-e.match(regexp, cancel):
		return nil
	case err := <-e.outputError:
		if err == io.EOF {
			return e.failedMatch(pattern)
		} else {
			return err
		}
	case <-time.After(timeout):
		cancel <- true
		return e.failedMatch(pattern)
	}
}

func (e *Expector) matchFailure(pattern string) ExpectionFailed {
	return ExpectationFailed{
		Wanted: pattern,
		Next:   string(e.nextOutput()),
		Output: string(e.fullOutput()),
	}
}

func (e *Expector) match(regexp *regexp.Regexp, cancel chan bool) chan bool {
	matchResult := make(chan bool)

	go func() {
		for {
			found := regexp.FindIndex(e.nextOutput())
			if found != nil {
				e.forwardOutput(found[1])
				matchResult <- true
				break
			}

			select {
			case <-e.listen:
			case <-time.After(100 * time.Millisecond):
			case <-cancel:
				return
			}
		}
	}()

	return matchResult
}

func (e *Expector) monitor() {
	var buf [1024]byte

	for {
		read, err := e.output.Read(buf[:])

		if read > 0 {
			e.addOutput(buf[:read])
		}

		e.notify()

		if err != nil {
			e.outputError <- err
			break
		}
	}
}

func (e *Expector) addOutput(out []byte) {
	e.Lock()
	defer e.Unlock()

	e.buffer.Write(out)
	e.fullBuffer.Write(out)
}

func (e *Expector) forwardOutput(count int) {
	e.Lock()
	defer e.Unlock()

	e.buffer.Next(count)
}

func (e *Expector) nextOutput() []byte {
	e.RLock()
	defer e.RUnlock()

	return e.buffer.Bytes()
}

func (e *Expector) fullOutput() []byte {
	e.RLock()
	defer e.RUnlock()

	return e.fullBuffer.Bytes()
}

func (e *Expector) notify() {
	select {
	case e.listen <- true:
	default:
	}
}
<|endoftext|>


<commit_before>package grid

import (
	"fmt"
	"mime"
	"os"
)

// ExportService handles communication with the Export related
// methods of the GRiD API.
//
// GRiD API docs: https://github.com/CRREL/GRiD-API/blob/v0.0/composed_api.rst#get-export-details
type ExportService struct {
	client *Client
}

type File struct {
	URL  string `json:"url"`
	Pk   int    `json:"pk"`
	Name string `json:"name"`
}

type ExportDetail struct {
	ExportFiles []File `json:"exportfiles"`
}

func (s *ExportService) ListByPk(pk int) ([]File, *Response, error) {
	url := fmt.Sprintf("api/v0/export/%v/", pk)

	req, err := s.client.NewRequest("GET", url, nil)

	exportDetail := new(ExportDetail)
	resp, err := s.client.Do(req, exportDetail)
	return exportDetail.ExportFiles, resp, err
}

func (s *ExportService) DownloadByPk(pk int) (*Response, error) {
	url := fmt.Sprintf("export/download/file/%v/", pk)

	req, err := s.client.NewRequest("GET", url, nil)

	file, err := os.Create("temp")
	if err != nil {
		panic(err)
	}
	defer file.Close()
	resp, err := s.client.Do(req, file)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	cd := resp.Header.Get("Content-Disposition")
	_, params, err := mime.ParseMediaType(cd)
	if err != nil {
		panic(err)
	}
	// fname := params["filename"]
	os.Rename(file.Name(), params["filename"])
	// log.Println(fname)
	// file, err := os.Create(fname)
	// if err != nil {
	// 	panic(err)
	// }
	// defer file.Close()
	//
	// numBytes, err := io.Copy(file, resp.Body)
	// if err != nil {
	// 	panic(err)
	// }
	// log.Println("Downloaded", numBytes, "bytes to", fname)
	return resp, err
}
<commit_msg>:fire: old code<commit_after>package grid

import (
	"fmt"
	"mime"
	"os"
)

// ExportService handles communication with the Export related
// methods of the GRiD API.
//
// GRiD API docs: https://github.com/CRREL/GRiD-API/blob/v0.0/composed_api.rst#get-export-details
type ExportService struct {
	client *Client
}

type File struct {
	URL  string `json:"url"`
	Pk   int    `json:"pk"`
	Name string `json:"name"`
}

type ExportDetail struct {
	ExportFiles []File `json:"exportfiles"`
}

func (s *ExportService) ListByPk(pk int) ([]File, *Response, error) {
	url := fmt.Sprintf("api/v0/export/%v/", pk)

	req, err := s.client.NewRequest("GET", url, nil)

	exportDetail := new(ExportDetail)
	resp, err := s.client.Do(req, exportDetail)
	return exportDetail.ExportFiles, resp, err
}

func (s *ExportService) DownloadByPk(pk int) (*Response, error) {
	url := fmt.Sprintf("export/download/file/%v/", pk)

	req, err := s.client.NewRequest("GET", url, nil)

	file, err := os.Create("temp")
	if err != nil {
		panic(err)
	}
	defer file.Close()
	resp, err := s.client.Do(req, file)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	cd := resp.Header.Get("Content-Disposition")
	_, params, err := mime.ParseMediaType(cd)
	if err != nil {
		panic(err)
	}
	os.Rename(file.Name(), params["filename"])
	return resp, err
}
<|endoftext|>


<commit_before>package web

import (
	"fmt"
	"net/http"

	"github.com/opentracing/opentracing-go"

	"echo-web/model"
	"echo-web/model/orm"
	"echo-web/module/log"
	. "echo-web/conf"
)

func HomeHandler(c *Context) error {
	// OpenTracing层级监控示例，API层通过中间件已支持
	span := c.OpenTracingSpan()
	if span != nil {
		// Since we have to inject our span into the HTTP headers, we create a request
		asyncReq, _ := http.NewRequest("GET", Conf.Server.DomainApi+"/login", nil)
		// Inject the span context into the header
		err := span.Tracer().Inject(span.Context(),
			opentracing.TextMap,
			opentracing.HTTPHeadersCarrier(asyncReq.Header))
		if err != nil {
			log.Debugf("Could not inject span context into header: %v", err)
		}
		go func() {
			if _, err := http.DefaultClient.Do(asyncReq); err != nil {
				span.SetTag("error", true)
				span.LogEvent(fmt.Sprintf("GET /login error: %v", err))
			}
		}()
	} else {
		log.Debugf("opentracing span nil")
	}

	User := model.User{
		Model: orm.Model{Context: c},
		Id:    1,
	}
	User.TraceGetUserById(1)

	c.Set("tmpl", "web/home")
	c.Set("data", map[string]interface{}{
		"title": "Home",
	})

	return nil
}
<commit_msg>unsupported protocol scheme<commit_after>package web

import (
	"fmt"
	"net/http"

	"github.com/opentracing/opentracing-go"

	"echo-web/model"
	"echo-web/model/orm"
	"echo-web/module/log"
	. "echo-web/conf"
)

func HomeHandler(c *Context) error {
	// OpenTracing层级监控示例，API层通过中间件已支持
	span := c.OpenTracingSpan()
	if span != nil {
		// Since we have to inject our span into the HTTP headers, we create a request
		asyncReq, _ := http.NewRequest("GET", "http://"+Conf.Server.DomainApi+"/login", nil)
		// Inject the span context into the header
		err := span.Tracer().Inject(span.Context(),
			opentracing.TextMap,
			opentracing.HTTPHeadersCarrier(asyncReq.Header))
		if err != nil {
			log.Debugf("Could not inject span context into header: %v", err)
		}
		go func() {
			if _, err := http.DefaultClient.Do(asyncReq); err != nil {
				span.SetTag("error", true)
				span.LogEvent(fmt.Sprintf("GET /login error: %v", err))
			}
		}()
	} else {
		log.Debugf("opentracing span nil")
	}

	User := model.User{
		Model: orm.Model{Context: c},
		Id:    1,
	}
	User.TraceGetUserById(1)

	c.Set("tmpl", "web/home")
	c.Set("data", map[string]interface{}{
		"title": "Home",
	})

	return nil
}
<|endoftext|>


<commit_before>package uatparse

import ()

const (
	BLOCK_WIDTH      = float64(48.0 / 60.0)
	WIDE_BLOCK_WIDTH = float64(96.0 / 60.0)
	BLOCK_HEIGHT     = float64(4.0 / 60.0)
	BLOCK_THRESHOLD  = 405000
	BLOCKS_PER_RING  = 450
)

type NEXRADBlock struct {
	Radar_Type uint32
	Scale      int
	LatNorth   float64
	LonWest    float64
	Height     float64
	Width      float64
	Intensity  []uint8 // Really only 4-bit values.
}

func block_location(block_num int, ns_flag bool, scale_factor int) (float64, float64, float64, float64) {
	var realScale float64
	if scale_factor == 1 {
		realScale = float64(5.0)
	} else if scale_factor == 2 {
		realScale = float64(9.0)
	} else {
		realScale = float64(1.0)
	}

	if block_num >= BLOCK_THRESHOLD {
		block_num = block_num & ^1
	}

	raw_lat := float64(BLOCK_HEIGHT * float64(int(float64(block_num)/float64(BLOCKS_PER_RING))))
	raw_lon := float64(block_num%BLOCKS_PER_RING) * BLOCK_WIDTH

	var lonSize float64
	if block_num >= BLOCK_THRESHOLD {
		lonSize = WIDE_BLOCK_WIDTH * realScale
	} else {
		lonSize = BLOCK_WIDTH * realScale
	}

	latSize := BLOCK_HEIGHT * realScale

	if ns_flag { // Southern hemisphere.
		raw_lat = 0 - raw_lat
	} else {
		raw_lat = raw_lat + BLOCK_HEIGHT
	}

	if raw_lon > 180.0 {
		raw_lon = raw_lon - 360.0
	}

	return raw_lat, raw_lon, latSize, lonSize

}

func (f *UATFrame) decodeNexradFrame() {
	if len(f.FISB_data) < 4 { // Short read.
		return
	}

	rle_flag := (uint32(f.FISB_data[0]) & 0x80) != 0
	ns_flag := (uint32(f.FISB_data[0]) & 0x40) != 0
	block_num := ((int(f.FISB_data[0]) & 0x0f) << 16) | (int(f.FISB_data[1]) << 8) | (int(f.FISB_data[2]))
	scale_factor := (int(f.FISB_data[0]) & 0x30) >> 4

	if rle_flag { // Single bin, RLE encoded.
		lat, lon, h, w := block_location(block_num, ns_flag, scale_factor)
		var tmp NEXRADBlock
		tmp.Radar_Type = f.Product_id
		tmp.Scale = scale_factor
		tmp.LatNorth = lat
		tmp.LonWest = lon
		tmp.Height = h
		tmp.Width = w
		tmp.Intensity = make([]uint8, 0)

		intensityData := f.FISB_data[3:]
		for _, v := range intensityData {
			intensity := uint8(v) & 0x7
			runlength := (uint8(v) >> 3) + 1
			for runlength > 0 {
				tmp.Intensity = append(tmp.Intensity, intensity)
				runlength--
			}
		}
		f.NEXRAD = []NEXRADBlock{tmp}
	} else {
		var row_start int
		var row_size int
		if block_num >= 405000 {
			row_start = block_num - ((block_num - 405000) % 225)
			row_size = 225
		} else {
			row_start = block_num - (block_num % 450)
			row_size = 450
		}

		row_offset := block_num - row_start

		L := int(f.FISB_data[3] & 15)

		if len(f.FISB_data) < L+3 { // Short read.
			return
		}

		for i := 0; i < L; i++ {
			var bb int
			if i == 0 {
				bb = (int(f.FISB_data[3]) & 0xF0) | 0x08
			} else {
				bb = int(f.FISB_data[i+3])
			}

			for j := 0; j < 8; j++ {
				if bb&(1<<uint(j)) != 0 {
					row_x := (row_offset + 8*i + j - 3) % row_size
					bn := row_start + row_x
					lat, lon, h, w := block_location(bn, ns_flag, scale_factor)
					var tmp NEXRADBlock
					tmp.Radar_Type = f.Product_id
					tmp.Scale = scale_factor
					tmp.LatNorth = lat
					tmp.LonWest = lon
					tmp.Height = h
					tmp.Width = w
					tmp.Intensity = make([]uint8, 0)
					for k := 0; k < 128; k++ {
						z := uint8(0)
						if f.Product_id == 64 {
							z = 1
						}
						tmp.Intensity = append(tmp.Intensity, z)
					}
					f.NEXRAD = append(f.NEXRAD, tmp)
				}
			}
		}
	}

}
<commit_msg>Change Intensity encoding.<commit_after>package uatparse

import ()

const (
	BLOCK_WIDTH      = float64(48.0 / 60.0)
	WIDE_BLOCK_WIDTH = float64(96.0 / 60.0)
	BLOCK_HEIGHT     = float64(4.0 / 60.0)
	BLOCK_THRESHOLD  = 405000
	BLOCKS_PER_RING  = 450
)

type NEXRADBlock struct {
	Radar_Type uint32
	Scale      int
	LatNorth   float64
	LonWest    float64
	Height     float64
	Width      float64
	Intensity  []uint16 // Really only 4-bit values, but using this as a hack for the JSON encoding.
}

func block_location(block_num int, ns_flag bool, scale_factor int) (float64, float64, float64, float64) {
	var realScale float64
	if scale_factor == 1 {
		realScale = float64(5.0)
	} else if scale_factor == 2 {
		realScale = float64(9.0)
	} else {
		realScale = float64(1.0)
	}

	if block_num >= BLOCK_THRESHOLD {
		block_num = block_num & ^1
	}

	raw_lat := float64(BLOCK_HEIGHT * float64(int(float64(block_num)/float64(BLOCKS_PER_RING))))
	raw_lon := float64(block_num%BLOCKS_PER_RING) * BLOCK_WIDTH

	var lonSize float64
	if block_num >= BLOCK_THRESHOLD {
		lonSize = WIDE_BLOCK_WIDTH * realScale
	} else {
		lonSize = BLOCK_WIDTH * realScale
	}

	latSize := BLOCK_HEIGHT * realScale

	if ns_flag { // Southern hemisphere.
		raw_lat = 0 - raw_lat
	} else {
		raw_lat = raw_lat + BLOCK_HEIGHT
	}

	if raw_lon > 180.0 {
		raw_lon = raw_lon - 360.0
	}

	return raw_lat, raw_lon, latSize, lonSize

}

func (f *UATFrame) decodeNexradFrame() {
	if len(f.FISB_data) < 4 { // Short read.
		return
	}

	rle_flag := (uint32(f.FISB_data[0]) & 0x80) != 0
	ns_flag := (uint32(f.FISB_data[0]) & 0x40) != 0
	block_num := ((int(f.FISB_data[0]) & 0x0f) << 16) | (int(f.FISB_data[1]) << 8) | (int(f.FISB_data[2]))
	scale_factor := (int(f.FISB_data[0]) & 0x30) >> 4

	if rle_flag { // Single bin, RLE encoded.
		lat, lon, h, w := block_location(block_num, ns_flag, scale_factor)
		var tmp NEXRADBlock
		tmp.Radar_Type = f.Product_id
		tmp.Scale = scale_factor
		tmp.LatNorth = lat
		tmp.LonWest = lon
		tmp.Height = h
		tmp.Width = w
		tmp.Intensity = make([]uint16, 0)

		intensityData := f.FISB_data[3:]
		for _, v := range intensityData {
			intensity := uint16(v) & 0x7
			runlength := (uint16(v) >> 3) + 1
			for runlength > 0 {
				tmp.Intensity = append(tmp.Intensity, intensity)
				runlength--
			}
		}
		f.NEXRAD = []NEXRADBlock{tmp}
	} else {
		var row_start int
		var row_size int
		if block_num >= 405000 {
			row_start = block_num - ((block_num - 405000) % 225)
			row_size = 225
		} else {
			row_start = block_num - (block_num % 450)
			row_size = 450
		}

		row_offset := block_num - row_start

		L := int(f.FISB_data[3] & 15)

		if len(f.FISB_data) < L+3 { // Short read.
			return
		}

		for i := 0; i < L; i++ {
			var bb int
			if i == 0 {
				bb = (int(f.FISB_data[3]) & 0xF0) | 0x08
			} else {
				bb = int(f.FISB_data[i+3])
			}

			for j := 0; j < 8; j++ {
				if bb&(1<<uint(j)) != 0 {
					row_x := (row_offset + 8*i + j - 3) % row_size
					bn := row_start + row_x
					lat, lon, h, w := block_location(bn, ns_flag, scale_factor)
					var tmp NEXRADBlock
					tmp.Radar_Type = f.Product_id
					tmp.Scale = scale_factor
					tmp.LatNorth = lat
					tmp.LonWest = lon
					tmp.Height = h
					tmp.Width = w
					tmp.Intensity = make([]uint16, 0)
					for k := 0; k < 128; k++ {
						z := uint16(0)
						if f.Product_id == 64 {
							z = 1
						}
						tmp.Intensity = append(tmp.Intensity, z)
					}
					f.NEXRAD = append(f.NEXRAD, tmp)
				}
			}
		}
	}

}
<|endoftext|>