<commit_before>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"fmt"
	"strings"
	"reflect"
	"strconv"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"labix.org/v2/mgo/bson"
	"github.com/gorilla/mux"
)

type HttpParamDataType int8
type HttpParamType int8

const(
	HttpIntParam = HttpParamDataType(0)
	HttpStringParam = HttpParamDataType(1)
	HttpFloatParam = HttpParamDataType(2)
	HttpBoolParam = HttpParamDataType(3) // Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
	HttpObjectIdParam = HttpParamDataType(4)

	// All of the param types only support single values (i.e., no slices). If multiple values are present, the
	// first is taken.
	HttpParamPost = HttpParamType(0)
	HttpParamJsonPost = HttpParamType(1) // When the content body is posted in json format. This only supports one level
	HttpParamQuery = HttpParamType(2)
	HttpParamHeader = HttpParamType(3)
	HttpParamPath = HttpParamType(4) // This must be declared as {someName} in the path mapping
)

type HttpContext struct {
	Response http.ResponseWriter
	Request *http.Request
	Params map[string]*HttpParam
	ErrorCodes []string
	Errors []error

	postJson map[string]interface{}

	body []byte
}

type HttpParam struct {
	Name string
	InvalidErrorCode string
	DataType HttpParamDataType
	Type HttpParamType
	Required bool
	MinLength int
	MaxLength int
	Post bool
	Value interface{}
	Raw string
	Valid bool
	Present bool // If value is present and parsed properly
}

// Make sure your params are present and valid before trying to access.
func (self *HttpParam) Int() int { return self.Value.(int) }

func (self *HttpParam) Float() float64 { return self.Value.(float64) }

func (self *HttpParam) String() string { return self.Value.(string) }

func (self *HttpParam) Bool() bool { return self.Value.(bool) }

// Set a valid value for a param. Missing can be valid, but not present.
func (self *HttpParam) setPresentValue(value interface{}) {
	self.Present = true
	self.Value = value
}

// Validate the params. If any of the params are invalid, false is returned. You must call
// this first before calling the ErrorCodes []string. If not params are defined, this always
// returns "true". If there are raw data extraction errors, this is always false (e.g., body missing or incorrect).
func (self *HttpContext) ParamsAreValid() bool {

	if len(self.Errors) != 0 { return false }

	if len(self.Params) == 0 { return true }

	for _, param := range self.Params {
		switch param.DataType {
			case HttpIntParam: validateIntParam(self, param)
			case HttpStringParam: validateStringParam(self, param)
			case HttpFloatParam: validateFloatParam(self, param)
			case HttpBoolParam: validateBoolParam(self, param)
			case HttpObjectIdParam: validateObjectIdParam(self, param)
		}
	}

	return len(self.ErrorCodes) == 0
}

func (self *HttpContext) ParamInt(name string) int { return self.Params[name].Int() }

func (self *HttpContext) ParamFloat(name string) float64 { return self.Params[name].Float() }

func (self *HttpContext) ParamString(name string) string { return self.Params[name].String() }

func (self *HttpContext) ParamBool(name string) bool { return self.Params[name].Bool() }

func (self *HttpContext) HasRawErrors() bool { return len(self.Errors) > 0 }

// This returns the param value as a string. If the param is missing or empty,
// the string will be len == 0.
func retrieveParamValue(ctx *HttpContext, param *HttpParam) string {
	switch param.Type {
		case HttpParamPost: return strings.TrimSpace(ctx.Request.PostFormValue(param.Name))
		case HttpParamJsonPost: return retrieveJsonParamValue(ctx, param)
		case HttpParamQuery: return strings.TrimSpace(ctx.Request.FormValue(param.Name))
		case HttpParamHeader: return strings.TrimSpace(ctx.Request.Header.Get(param.Name))
		case HttpParamPath: return strings.TrimSpace(mux.Vars(ctx.Request)[param.Name])
	}
	return nadaStr
}

func retrieveJsonParamValue(ctx *HttpContext, param *HttpParam) string {

	if len(ctx.Errors) > 0 {
		return nadaStr
	}

	// If this is the first access, read the body
	if len(ctx.body) == 0 {
		var err error
		ctx.body, err = ioutil.ReadAll(ctx.Request.Body)
		if err != nil {
			ctx.Errors = append(ctx.Errors, NewStackError("Error in raw data extraction - error: %v", err))
			return nadaStr
		}
	}

	if ctx.postJson == nil {
		var genJson interface{}
		err := json.Unmarshal(ctx.body, &genJson)
		if err != nil {
			ctx.Errors = append(ctx.Errors, NewStackError("Error in raw json data extraction - error: %v", err))
			return nadaStr
		}

		ctx.postJson = genJson.(map[string]interface{})
	}

	// Look for the value in the json. The json may hold the data in a variety
	// of formats. Convert back to a string to deal with the other data types :-(
	val, found := ctx.postJson[param.Name]
	if !found {
		return nadaStr
	}

	valType := reflect.TypeOf(val)

	if valType == nil {
		return nadaStr
	}

	switch valType.Kind() {
		case reflect.Invalid: return nadaStr
		case reflect.Bool: return fmt.Sprintf("%t", val.(bool))
		case reflect.Float64: return fmt.Sprintf("%g", val.(float64))
		case reflect.String: return val.(string)
		default: return nadaStr
	}

	return nadaStr
}

func appendInvalidErrorCode(ctx *HttpContext, param *HttpParam) {
	// Do not dupplicate error codes.
	for i := range ctx.ErrorCodes { if ctx.ErrorCodes[i] == param.InvalidErrorCode { return } }
	ctx.ErrorCodes = append(ctx.ErrorCodes, param.InvalidErrorCode)
	param.Valid = false
}

func validateIntParam(ctx *HttpContext, param *HttpParam) {
	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 {
		return
	}

	if val, err := strconv.Atoi(param.Raw); err != nil {
		appendInvalidErrorCode(ctx, param)
	} else {
		param.setPresentValue(val)
	}
}

func validateStringParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if param.Required && param.MinLength > 0 && len(param.Raw) < param.MinLength {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if param.Required && param.MaxLength > 0 && len(param.Raw) > param.MaxLength {
		appendInvalidErrorCode(ctx, param)
		return
	}

	param.setPresentValue(param.Raw)
}

func validateFloatParam(ctx *HttpContext, param *HttpParam) {
	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 {
		return
	}

	if val, err := strconv.ParseFloat(param.Raw, 64); err != nil {
		appendInvalidErrorCode(ctx, param)
	} else {
		param.setPresentValue(val)
	}
}

func validateObjectIdParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 { return }

	if !bson.IsObjectIdHex(param.Raw) {
		appendInvalidErrorCode(ctx, param)
		return
	}

	param.setPresentValue(bson.ObjectIdHex(param.Raw))
}

// Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
func validateBoolParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 {
		return
	}

	if val, err := strconv.ParseBool(param.Raw); err != nil {
		appendInvalidErrorCode(ctx, param)
	} else {
		param.setPresentValue(val)
	}
}

func (self *HttpContext) DefineIntParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpIntParam, Required: required, Type: paramType, Valid: true }
}

// Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
func (self *HttpContext) DefineBoolParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpBoolParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineFloatParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpFloatParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineObjectIdParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpObjectIdParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineStringParam(name, invalidErrorCode string, paramType HttpParamType, required bool, minLength, maxLength int) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpStringParam, Required: required, Type: paramType, Valid: true }
}

// Call this method to init the http context struct.
func NewHttpContext(response http.ResponseWriter, request *http.Request) *HttpContext {
	return &HttpContext{ Response: response, Request: request, Params: make(map[string]*HttpParam) }
}

// This method returns true if the http request method is a HTTP post. If the
// field missing or incorrect, false is returned. This method will panic if
// the request is nil.
func IsHttpMethodPost(request *http.Request) bool {
	if request == nil {
		panic("request param is nil")
	}
	return len(request.Method) > 0 && strings.ToUpper(request.Method) == HttpPostMethod
}

// Write an http ok response string. The content type is text/plain.
func WriteOkResponseString(response http.ResponseWriter, msg string) error {
	if response == nil {
		return NewStackError("response param is nil")
	}

	msgLength := len(msg)

	if msgLength == 0 {
		return NewStackError("Response message is an empty string")
	}

	response.Header().Set(ContentTypeHeader, ContentTypeTextPlain)

	written, err := response.Write([]byte(msg))

	if err != nil {
		return err
	}

	if written != msgLength {
		return NewStackError("Did not write full message - bytes written %d - expected %d", written, msgLength)
	}

	return nil
}

// Encode and write a json response. If there is a problem encoding an http 500 is sent and an
// error is returned. If there are problems writting the response an error is returned.
func JsonEncodeAndWriteResponse(response http.ResponseWriter, value interface{}) error {

	if value == nil {
		return NewStackError("Nil value passed")
	}

	rawJson, err := json.Marshal(value)
	if err != nil {
		http.Error(response, "Error", 500)
		return NewStackError("Unable to marshal json: %v", err)
	}

	response.Header().Set(ContentTypeHeader, ContentTypeJson)

	written, err := response.Write(rawJson)
	if err != nil {
		return NewStackError("Unable to write response: %v", err)
	}

	if written != len(rawJson) {
		return NewStackError("Unable to write full response - wrote: %d - expected: %d", written, len(rawJson))
	}

	return nil
}

<commit_msg>added the rest of the object id impl<commit_after>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"fmt"
	"strings"
	"reflect"
	"strconv"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"labix.org/v2/mgo/bson"
	"github.com/gorilla/mux"
)

type HttpParamDataType int8
type HttpParamType int8

const(
	HttpIntParam = HttpParamDataType(0)
	HttpStringParam = HttpParamDataType(1)
	HttpFloatParam = HttpParamDataType(2)
	HttpBoolParam = HttpParamDataType(3) // Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
	HttpObjectIdParam = HttpParamDataType(4)

	// All of the param types only support single values (i.e., no slices). If multiple values are present, the
	// first is taken.
	HttpParamPost = HttpParamType(0)
	HttpParamJsonPost = HttpParamType(1) // When the content body is posted in json format. This only supports one level
	HttpParamQuery = HttpParamType(2)
	HttpParamHeader = HttpParamType(3)
	HttpParamPath = HttpParamType(4) // This must be declared as {someName} in the path mapping
)

type HttpContext struct {
	Response http.ResponseWriter
	Request *http.Request
	Params map[string]*HttpParam
	ErrorCodes []string
	Errors []error

	postJson map[string]interface{}

	body []byte
}

type HttpParam struct {
	Name string
	InvalidErrorCode string
	DataType HttpParamDataType
	Type HttpParamType
	Required bool
	MinLength int
	MaxLength int
	Post bool
	Value interface{}
	Raw string
	Valid bool
	Present bool // If value is present and parsed properly
}

// Make sure your params are present and valid before trying to access.
func (self *HttpParam) Int() int { return self.Value.(int) }

func (self *HttpParam) Float() float64 { return self.Value.(float64) }

func (self *HttpParam) String() string { return self.Value.(string) }

func (self *HttpParam) Bool() bool { return self.Value.(bool) }

func (self *HttpParam) ObjectId() *bson.ObjectId { return self.Value.(*bson.ObjectId) }

// Set a valid value for a param. Missing can be valid, but not present.
func (self *HttpParam) setPresentValue(value interface{}) {
	self.Present = true
	self.Value = value
}

// Validate the params. If any of the params are invalid, false is returned. You must call
// this first before calling the ErrorCodes []string. If not params are defined, this always
// returns "true". If there are raw data extraction errors, this is always false (e.g., body missing or incorrect).
func (self *HttpContext) ParamsAreValid() bool {

	if len(self.Errors) != 0 { return false }

	if len(self.Params) == 0 { return true }

	for _, param := range self.Params {
		switch param.DataType {
			case HttpIntParam: validateIntParam(self, param)
			case HttpStringParam: validateStringParam(self, param)
			case HttpFloatParam: validateFloatParam(self, param)
			case HttpBoolParam: validateBoolParam(self, param)
			case HttpObjectIdParam: validateObjectIdParam(self, param)
		}
	}

	return len(self.ErrorCodes) == 0
}

func (self *HttpContext) ParamInt(name string) int { return self.Params[name].Int() }

func (self *HttpContext) ParamFloat(name string) float64 { return self.Params[name].Float() }

func (self *HttpContext) ParamString(name string) string { return self.Params[name].String() }

func (self *HttpContext) ParamBool(name string) bool { return self.Params[name].Bool() }

func (self *HttpContext) ParamObjectId(name string) *bson.ObjectId { return self.Params[name].ObjectId() }

func (self *HttpContext) HasRawErrors() bool { return len(self.Errors) > 0 }

// This returns the param value as a string. If the param is missing or empty,
// the string will be len == 0.
func retrieveParamValue(ctx *HttpContext, param *HttpParam) string {
	switch param.Type {
		case HttpParamPost: return strings.TrimSpace(ctx.Request.PostFormValue(param.Name))
		case HttpParamJsonPost: return retrieveJsonParamValue(ctx, param)
		case HttpParamQuery: return strings.TrimSpace(ctx.Request.FormValue(param.Name))
		case HttpParamHeader: return strings.TrimSpace(ctx.Request.Header.Get(param.Name))
		case HttpParamPath: return strings.TrimSpace(mux.Vars(ctx.Request)[param.Name])
	}
	return nadaStr
}

func retrieveJsonParamValue(ctx *HttpContext, param *HttpParam) string {

	if len(ctx.Errors) > 0 {
		return nadaStr
	}

	// If this is the first access, read the body
	if len(ctx.body) == 0 {
		var err error
		ctx.body, err = ioutil.ReadAll(ctx.Request.Body)
		if err != nil {
			ctx.Errors = append(ctx.Errors, NewStackError("Error in raw data extraction - error: %v", err))
			return nadaStr
		}
	}

	if ctx.postJson == nil {
		var genJson interface{}
		err := json.Unmarshal(ctx.body, &genJson)
		if err != nil {
			ctx.Errors = append(ctx.Errors, NewStackError("Error in raw json data extraction - error: %v", err))
			return nadaStr
		}

		ctx.postJson = genJson.(map[string]interface{})
	}

	// Look for the value in the json. The json may hold the data in a variety
	// of formats. Convert back to a string to deal with the other data types :-(
	val, found := ctx.postJson[param.Name]
	if !found {
		return nadaStr
	}

	valType := reflect.TypeOf(val)

	if valType == nil { return nadaStr }

	switch valType.Kind() {
		case reflect.Invalid: return nadaStr
		case reflect.Bool: return fmt.Sprintf("%t", val.(bool))
		case reflect.Float64: return fmt.Sprintf("%g", val.(float64))
		case reflect.String: return val.(string)
		default: return nadaStr
	}

	return nadaStr
}

func appendInvalidErrorCode(ctx *HttpContext, param *HttpParam) {
	// Do not dupplicate error codes.
	for i := range ctx.ErrorCodes { if ctx.ErrorCodes[i] == param.InvalidErrorCode { return } }
	ctx.ErrorCodes = append(ctx.ErrorCodes, param.InvalidErrorCode)
	param.Valid = false
}

func validateIntParam(ctx *HttpContext, param *HttpParam) {
	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 {
		return
	}

	if val, err := strconv.Atoi(param.Raw); err != nil {
		appendInvalidErrorCode(ctx, param)
	} else {
		param.setPresentValue(val)
	}
}

func validateStringParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if param.Required && param.MinLength > 0 && len(param.Raw) < param.MinLength {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if param.Required && param.MaxLength > 0 && len(param.Raw) > param.MaxLength {
		appendInvalidErrorCode(ctx, param)
		return
	}

	param.setPresentValue(param.Raw)
}

func validateFloatParam(ctx *HttpContext, param *HttpParam) {
	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 {
		return
	}

	if val, err := strconv.ParseFloat(param.Raw, 64); err != nil {
		appendInvalidErrorCode(ctx, param)
	} else {
		param.setPresentValue(val)
	}
}

func validateObjectIdParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 { return }

	if !bson.IsObjectIdHex(param.Raw) {
		appendInvalidErrorCode(ctx, param)
		return
	}

	value := bson.ObjectIdHex(param.Raw)
	param.setPresentValue(&value)
}

// Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
func validateBoolParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 {
		return
	}

	if val, err := strconv.ParseBool(param.Raw); err != nil {
		appendInvalidErrorCode(ctx, param)
	} else {
		param.setPresentValue(val)
	}
}

func (self *HttpContext) DefineIntParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpIntParam, Required: required, Type: paramType, Valid: true }
}

// Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
func (self *HttpContext) DefineBoolParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpBoolParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineFloatParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpFloatParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineObjectIdParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpObjectIdParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineStringParam(name, invalidErrorCode string, paramType HttpParamType, required bool, minLength, maxLength int) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpStringParam, Required: required, Type: paramType, Valid: true }
}

// Call this method to init the http context struct.
func NewHttpContext(response http.ResponseWriter, request *http.Request) *HttpContext {
	return &HttpContext{ Response: response, Request: request, Params: make(map[string]*HttpParam) }
}

// This method returns true if the http request method is a HTTP post. If the
// field missing or incorrect, false is returned. This method will panic if
// the request is nil.
func IsHttpMethodPost(request *http.Request) bool {
	if request == nil {
		panic("request param is nil")
	}
	return len(request.Method) > 0 && strings.ToUpper(request.Method) == HttpPostMethod
}

// Write an http ok response string. The content type is text/plain.
func WriteOkResponseString(response http.ResponseWriter, msg string) error {
	if response == nil {
		return NewStackError("response param is nil")
	}

	msgLength := len(msg)

	if msgLength == 0 {
		return NewStackError("Response message is an empty string")
	}

	response.Header().Set(ContentTypeHeader, ContentTypeTextPlain)

	written, err := response.Write([]byte(msg))

	if err != nil {
		return err
	}

	if written != msgLength {
		return NewStackError("Did not write full message - bytes written %d - expected %d", written, msgLength)
	}

	return nil
}

// Encode and write a json response. If there is a problem encoding an http 500 is sent and an
// error is returned. If there are problems writting the response an error is returned.
func JsonEncodeAndWriteResponse(response http.ResponseWriter, value interface{}) error {

	if value == nil {
		return NewStackError("Nil value passed")
	}

	rawJson, err := json.Marshal(value)
	if err != nil {
		http.Error(response, "Error", 500)
		return NewStackError("Unable to marshal json: %v", err)
	}

	response.Header().Set(ContentTypeHeader, ContentTypeJson)

	written, err := response.Write(rawJson)
	if err != nil {
		return NewStackError("Unable to write response: %v", err)
	}

	if written != len(rawJson) {
		return NewStackError("Unable to write full response - wrote: %d - expected: %d", written, len(rawJson))
	}

	return nil
}

<|endoftext|>


<commit_before>// Copyright 2012, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*  Filename:    lexer.go
 *  Author:      Bryan Matsuo <bryan.matsuo [at] gmail.com>
 *  Created:     2012-11-02 22:10:59.782356 -0700 PDT
 *  Description: Main source file in go-lexer
 */

/*
Package lexer provides a simple scanner and types for handrolling lexers.
The implementation is based on Rob Pike's talk.

	http://www.youtube.com/watch?v=HxaD_trXwRE

Two APIs

The Lexer type has two APIs, one is used byte StateFn types.  The other is
called by the parser. These APIs are called the scanner and the parser APIs
here.

The parser API

The only function the parser calls on the lexer is Next to retreive the next
token from the input stream.  Eventually an item with type ItemEOF is returned
at which point there are no more tokens in the stream.

The scanner API

The lexer uses Emit to construct complete lexemes to return from
future/concurrent calls to Next by the parser.  The scanner uses a combination
of methods to manipulate its position and and prepare lexemes to be emitted.
Lexer errors are emitted to the parser using the Errorf method which keeps the
scanner-parser interface uniform.

Common lexer methods used in a scanner are the Accept[Run][Range] family of
methods.  Accept* methods take a set and advance the lexer if incoming runes
are in the set. The AcceptRun* subfamily advance the lexer as far as possible.

For scanning known sequences of bytes (e.g. keywords) the AcceptString method
avoids a lot of branching that would be incurred using methods that match
character classes.

The remaining methods provide low level functionality that can be combined to
address corner cases.
*/
package lexer

import (
	"container/list"
	"fmt"
	"math"
	"strings"
	"unicode"
	"unicode/utf8"
)

const EOF rune = 0x04

// IsEOF returns true if n is zero.
func IsEOF(c rune, n int) bool {
	return n == 0
}

// IsInvalid returns true if c is utf8.RuneError and n is 1.
func IsInvalid(c rune, n int) bool {
	return c == utf8.RuneError && n == 1
}

// StateFn functions scan runes from the lexer's input and emit items.  StateFn
// implementations responsible for emitting ItemEOF.
type StateFn func(*Lexer) StateFn

// Lexer contains an input string and state associate with the lexing the
// input.
type Lexer struct {
	input string     // string being scanned
	start int        // start position for the current lexeme
	pos   int        // current position
	width int        // length of the last rune read
	last  rune       // the last rune read
	state StateFn    // the current state
	items *list.List // Buffer of lexed items
}

// Create a new lexer. Must be given a non-nil state.
func New(start StateFn, input string) *Lexer {
	if start == nil {
		panic("nil start state")
	}
	return &Lexer{
		state: start,
		input: input,
		items: list.New(),
	}
}

// Input returns the input string being lexed by the l.
func (l *Lexer) Input() string {
	return l.input
}

// Start marks the first byte of item currently being lexed.
func (l *Lexer) Start() int {
	return l.start
}

// Pos marks the next byte to be read in the input string.  The behavior of Pos
// is unspecified if an error previously occurred or if all input has been
// consumed.
func (l *Lexer) Pos() int {
	return l.pos
}

// Current returns the contents of the item currently being lexed.
func (l *Lexer) Current() string {
	return l.input[l.start:l.pos]
}

// Last return the last rune read from the input stream.
func (l *Lexer) Last() (r rune, width int) {
	return l.last, l.width
}

// Advance adds one rune of input to the current lexeme, increments the lexer's
// position, and returns the input rune with its size in bytes (encoded as
// UTF-8).  Invalid UTF-8 codepoints cause the current call and all subsequent
// calls to return (utf8.RuneError, 1).  If there is no input the returned size
// is zero.
func (l *Lexer) Advance() (rune, int) {
	if l.pos >= len(l.input) {
		l.width = 0
		return EOF, l.width
	}
	l.last, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
	if l.last == utf8.RuneError && l.width == 1 {
		return l.last, l.width
	}
	l.pos += l.width
	return l.last, l.width
}

// Backup removes the last rune from the current lexeme and moves l's position
// back in the input string accordingly. Backup should only be called after a
// call to Advance.
func (l *Lexer) Backup() {
	l.pos -= l.width
}

// Peek returns the next rune in the input stream without adding it to the
// current lexeme.
func (l *Lexer) Peek() (rune, int) {
	c, n := l.Advance()
	l.Backup()
	return c, n
}

// Ignore throws away the current lexeme.
func (l *Lexer) Ignore() {
	l.start = l.pos
}

// Accept advances the lexer if the next rune is in valid.
func (l *Lexer) Accept(valid string) (ok bool) {
	r, _ := l.Advance()
	ok = strings.IndexRune(valid, r) >= 0
	if !ok {
		l.Backup()
	}
	return
}

// AcceptFunc advances the lexer if fn return true for the next rune.
func (l *Lexer) AcceptFunc(fn func(rune) bool) (ok bool) {
	switch r, n := l.Advance(); {
	case IsEOF(r, n):
		return false
	case IsInvalid(r, n):
		return false
	case fn(r):
		return true
	default:
		l.Backup()
		return false
	}
}

// AcceptRange advances l's position if the current rune is in tab.
func (l *Lexer) AcceptRange(tab *unicode.RangeTable) (ok bool) {
	r, _ := l.Advance()
	ok = unicode.Is(tab, r)
	if !ok {
		l.Backup()
	}
	return
}

// AcceptRun advances l's position as long as the current rune is in valid.
func (l *Lexer) AcceptRun(valid string) (n int) {
	for l.Accept(valid) {
		n++
	}
	return
}

// AcceptRunFunc advances l's position as long as fn returns true for the next
// input rune.
func (l *Lexer) AcceptRunFunc(fn func(rune) bool) int {
	var n int
	for l.AcceptFunc(fn) {
		n++
	}
	return n
}

// AcceptRunRange advances l's possition as long as the current rune is in tab.
func (l *Lexer) AcceptRunRange(tab *unicode.RangeTable) (n int) {
	for l.AcceptRange(tab) {
		n++
	}
	return
}

// AcceptString advances the lexer len(s) bytes if the next len(s) bytes equal
// s. AcceptString returns true if l advanced.
func (l *Lexer) AcceptString(s string) (ok bool) {
	if strings.HasPrefix(l.input[l.pos:], s) {
		l.pos += len(s)
		return true
	}
	return false
}

// Errorf causes an error item to be emitted from l.Next().  The item's value
// (and its error message) are the result of evaluating format and vs with
// fmt.Sprintf.
func (l *Lexer) Errorf(format string, vs ...interface{}) StateFn {
	l.enqueue(&Item{
		ItemError,
		l.start,
		fmt.Sprintf(format, vs...),
	})
	return nil
}

// Emit the current value as an Item with the specified type.
func (l *Lexer) Emit(t ItemType) {
	l.enqueue(&Item{
		t,
		l.start,
		l.input[l.start:l.pos],
	})
	l.start = l.pos
}

// The method by which items are extracted from the input.
// Returns nil if the lexer has entered a nil state.
func (l *Lexer) Next() (i *Item) {
	for {
		if head := l.dequeue(); head != nil {
			return head
		}
		if l.state == nil {
			return &Item{ItemEOF, l.start, ""}
		}
		l.state = l.state(l)
	}
	panic("unreachable")
}

func (l *Lexer) enqueue(i *Item) {
	l.items.PushBack(i)
}

func (l *Lexer) dequeue() *Item {
	head := l.items.Front()
	if head == nil {
		return nil
	}
	return l.items.Remove(head).(*Item)
}

// A type for all the types of items in the language being lexed.
type ItemType uint16

// Special item types.
const (
	ItemEOF ItemType = math.MaxUint16 - iota
	ItemError
)

// An individual scanned item (a lexeme).
type Item struct {
	Type  ItemType
	Pos   int
	Value string
}

// Err returns the error corresponding to i, if one exists.
func (i *Item) Err() error {
	if i.Type == ItemError {
		return (*Error)(i)
	}
	return nil
}

// String returns the raw lexeme of i.
func (i *Item) String() string {
	switch i.Type {
	case ItemError:
		return i.Value
	case ItemEOF:
		return "EOF"
	}
	if len(i.Value) > 10 {
		return fmt.Sprintf("%.10q...", i.Value)
	}
	return i.Value
}

// Error is an item of type ItemError
type Error Item

func (err *Error) Error() string {
	return (*Item)(err).String()
}
<commit_msg>StateFn godoc tweak<commit_after>// Copyright 2012, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*  Filename:    lexer.go
 *  Author:      Bryan Matsuo <bryan.matsuo [at] gmail.com>
 *  Created:     2012-11-02 22:10:59.782356 -0700 PDT
 *  Description: Main source file in go-lexer
 */

/*
Package lexer provides a simple scanner and types for handrolling lexers.
The implementation is based on Rob Pike's talk.

	http://www.youtube.com/watch?v=HxaD_trXwRE

Two APIs

The Lexer type has two APIs, one is used byte StateFn types.  The other is
called by the parser. These APIs are called the scanner and the parser APIs
here.

The parser API

The only function the parser calls on the lexer is Next to retreive the next
token from the input stream.  Eventually an item with type ItemEOF is returned
at which point there are no more tokens in the stream.

The scanner API

The lexer uses Emit to construct complete lexemes to return from
future/concurrent calls to Next by the parser.  The scanner uses a combination
of methods to manipulate its position and and prepare lexemes to be emitted.
Lexer errors are emitted to the parser using the Errorf method which keeps the
scanner-parser interface uniform.

Common lexer methods used in a scanner are the Accept[Run][Range] family of
methods.  Accept* methods take a set and advance the lexer if incoming runes
are in the set. The AcceptRun* subfamily advance the lexer as far as possible.

For scanning known sequences of bytes (e.g. keywords) the AcceptString method
avoids a lot of branching that would be incurred using methods that match
character classes.

The remaining methods provide low level functionality that can be combined to
address corner cases.
*/
package lexer

import (
	"container/list"
	"fmt"
	"math"
	"strings"
	"unicode"
	"unicode/utf8"
)

const EOF rune = 0x04

// IsEOF returns true if n is zero.
func IsEOF(c rune, n int) bool {
	return n == 0
}

// IsInvalid returns true if c is utf8.RuneError and n is 1.
func IsInvalid(c rune, n int) bool {
	return c == utf8.RuneError && n == 1
}

// StateFn functions scan runes from the lexer's input and emit items.  A StateFn
// is responsible for emitting ItemEOF after input has been consumed.
type StateFn func(*Lexer) StateFn

// Lexer contains an input string and state associate with the lexing the
// input.
type Lexer struct {
	input string     // string being scanned
	start int        // start position for the current lexeme
	pos   int        // current position
	width int        // length of the last rune read
	last  rune       // the last rune read
	state StateFn    // the current state
	items *list.List // Buffer of lexed items
}

// Create a new lexer. Must be given a non-nil state.
func New(start StateFn, input string) *Lexer {
	if start == nil {
		panic("nil start state")
	}
	return &Lexer{
		state: start,
		input: input,
		items: list.New(),
	}
}

// Input returns the input string being lexed by the l.
func (l *Lexer) Input() string {
	return l.input
}

// Start marks the first byte of item currently being lexed.
func (l *Lexer) Start() int {
	return l.start
}

// Pos marks the next byte to be read in the input string.  The behavior of Pos
// is unspecified if an error previously occurred or if all input has been
// consumed.
func (l *Lexer) Pos() int {
	return l.pos
}

// Current returns the contents of the item currently being lexed.
func (l *Lexer) Current() string {
	return l.input[l.start:l.pos]
}

// Last return the last rune read from the input stream.
func (l *Lexer) Last() (r rune, width int) {
	return l.last, l.width
}

// Advance adds one rune of input to the current lexeme, increments the lexer's
// position, and returns the input rune with its size in bytes (encoded as
// UTF-8).  Invalid UTF-8 codepoints cause the current call and all subsequent
// calls to return (utf8.RuneError, 1).  If there is no input the returned size
// is zero.
func (l *Lexer) Advance() (rune, int) {
	if l.pos >= len(l.input) {
		l.width = 0
		return EOF, l.width
	}
	l.last, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
	if l.last == utf8.RuneError && l.width == 1 {
		return l.last, l.width
	}
	l.pos += l.width
	return l.last, l.width
}

// Backup removes the last rune from the current lexeme and moves l's position
// back in the input string accordingly. Backup should only be called after a
// call to Advance.
func (l *Lexer) Backup() {
	l.pos -= l.width
}

// Peek returns the next rune in the input stream without adding it to the
// current lexeme.
func (l *Lexer) Peek() (rune, int) {
	c, n := l.Advance()
	l.Backup()
	return c, n
}

// Ignore throws away the current lexeme.
func (l *Lexer) Ignore() {
	l.start = l.pos
}

// Accept advances the lexer if the next rune is in valid.
func (l *Lexer) Accept(valid string) (ok bool) {
	r, _ := l.Advance()
	ok = strings.IndexRune(valid, r) >= 0
	if !ok {
		l.Backup()
	}
	return
}

// AcceptFunc advances the lexer if fn return true for the next rune.
func (l *Lexer) AcceptFunc(fn func(rune) bool) (ok bool) {
	switch r, n := l.Advance(); {
	case IsEOF(r, n):
		return false
	case IsInvalid(r, n):
		return false
	case fn(r):
		return true
	default:
		l.Backup()
		return false
	}
}

// AcceptRange advances l's position if the current rune is in tab.
func (l *Lexer) AcceptRange(tab *unicode.RangeTable) (ok bool) {
	r, _ := l.Advance()
	ok = unicode.Is(tab, r)
	if !ok {
		l.Backup()
	}
	return
}

// AcceptRun advances l's position as long as the current rune is in valid.
func (l *Lexer) AcceptRun(valid string) (n int) {
	for l.Accept(valid) {
		n++
	}
	return
}

// AcceptRunFunc advances l's position as long as fn returns true for the next
// input rune.
func (l *Lexer) AcceptRunFunc(fn func(rune) bool) int {
	var n int
	for l.AcceptFunc(fn) {
		n++
	}
	return n
}

// AcceptRunRange advances l's possition as long as the current rune is in tab.
func (l *Lexer) AcceptRunRange(tab *unicode.RangeTable) (n int) {
	for l.AcceptRange(tab) {
		n++
	}
	return
}

// AcceptString advances the lexer len(s) bytes if the next len(s) bytes equal
// s. AcceptString returns true if l advanced.
func (l *Lexer) AcceptString(s string) (ok bool) {
	if strings.HasPrefix(l.input[l.pos:], s) {
		l.pos += len(s)
		return true
	}
	return false
}

// Errorf causes an error item to be emitted from l.Next().  The item's value
// (and its error message) are the result of evaluating format and vs with
// fmt.Sprintf.
func (l *Lexer) Errorf(format string, vs ...interface{}) StateFn {
	l.enqueue(&Item{
		ItemError,
		l.start,
		fmt.Sprintf(format, vs...),
	})
	return nil
}

// Emit the current value as an Item with the specified type.
func (l *Lexer) Emit(t ItemType) {
	l.enqueue(&Item{
		t,
		l.start,
		l.input[l.start:l.pos],
	})
	l.start = l.pos
}

// The method by which items are extracted from the input.
// Returns nil if the lexer has entered a nil state.
func (l *Lexer) Next() (i *Item) {
	for {
		if head := l.dequeue(); head != nil {
			return head
		}
		if l.state == nil {
			return &Item{ItemEOF, l.start, ""}
		}
		l.state = l.state(l)
	}
	panic("unreachable")
}

func (l *Lexer) enqueue(i *Item) {
	l.items.PushBack(i)
}

func (l *Lexer) dequeue() *Item {
	head := l.items.Front()
	if head == nil {
		return nil
	}
	return l.items.Remove(head).(*Item)
}

// A type for all the types of items in the language being lexed.
type ItemType uint16

// Special item types.
const (
	ItemEOF ItemType = math.MaxUint16 - iota
	ItemError
)

// An individual scanned item (a lexeme).
type Item struct {
	Type  ItemType
	Pos   int
	Value string
}

// Err returns the error corresponding to i, if one exists.
func (i *Item) Err() error {
	if i.Type == ItemError {
		return (*Error)(i)
	}
	return nil
}

// String returns the raw lexeme of i.
func (i *Item) String() string {
	switch i.Type {
	case ItemError:
		return i.Value
	case ItemEOF:
		return "EOF"
	}
	if len(i.Value) > 10 {
		return fmt.Sprintf("%.10q...", i.Value)
	}
	return i.Value
}

// Error is an item of type ItemError
type Error Item

func (err *Error) Error() string {
	return (*Item)(err).String()
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package newtutil

import (
	"fmt"
	"os/user"
	"sort"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cast"

	"mynewt.apache.org/newt/newt/interfaces"
	"mynewt.apache.org/newt/util"
	"mynewt.apache.org/newt/viper"
)

var NewtVersion Version = Version{1, 0, 0}
var NewtVersionStr string = "Apache Newt (incubating) version: 1.0.1"
var NewtBlinkyTag string = "develop"
var NewtNumJobs int
var NewtForce bool

const NEWTRC_DIR string = ".newt"
const REPOS_FILENAME string = "repos.yml"

const CORE_REPO_NAME string = "apache-mynewt-core"
const ARDUINO_ZERO_REPO_NAME string = "mynewt_arduino_zero"

type Version struct {
	Major    int64
	Minor    int64
	Revision int64
}

func ParseVersion(s string) (Version, error) {
	v := Version{}
	parseErr := util.FmtNewtError("Invalid version string: %s", s)

	parts := strings.Split(s, ".")
	if len(parts) != 3 {
		return v, parseErr
	}

	var err error

	v.Major, err = strconv.ParseInt(parts[0], 10, 64)
	if err != nil {
		return v, parseErr
	}

	v.Minor, err = strconv.ParseInt(parts[1], 10, 64)
	if err != nil {
		return v, parseErr
	}

	v.Revision, err = strconv.ParseInt(parts[2], 10, 64)
	if err != nil {
		return v, parseErr
	}

	return v, nil
}

func (v *Version) String() string {
	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Revision)
}

func VerCmp(v1 Version, v2 Version) int64 {
	if r := v1.Major - v2.Major; r != 0 {
		return r
	}

	if r := v1.Minor - v2.Minor; r != 0 {
		return r
	}

	if r := v1.Revision - v2.Revision; r != 0 {
		return r
	}

	return 0
}

// Contains general newt settings read from $HOME/.newt
var newtrc *viper.Viper

func readNewtrc() *viper.Viper {
	usr, err := user.Current()
	if err != nil {
		log.Warn("Failed to obtain user name")
		return viper.New()
	}

	dir := usr.HomeDir + "/" + NEWTRC_DIR
	v, err := util.ReadConfig(dir, strings.TrimSuffix(REPOS_FILENAME, ".yml"))
	if err != nil {
		log.Debugf("Failed to read %s/%s file", dir, REPOS_FILENAME)
		return viper.New()
	}

	return v
}

func Newtrc() *viper.Viper {
	if newtrc != nil {
		return newtrc
	}

	newtrc = readNewtrc()
	return newtrc
}

func GetSliceFeatures(v *viper.Viper, features map[string]bool,
	key string) []interface{} {

	val := v.Get(key)
	vals := []interface{}{val}

	// Process the features in alphabetical order to ensure consistent
	// results across repeated runs.
	featureKeys := make([]string, 0, len(features))
	for feature, _ := range features {
		featureKeys = append(featureKeys, feature)
	}
	sort.Strings(featureKeys)

	for _, feature := range featureKeys {
		overwriteVal := v.Get(key + "." + feature + ".OVERWRITE")
		if overwriteVal != nil {
			return []interface{}{overwriteVal}
		}

		appendVal := v.Get(key + "." + feature)
		if appendVal != nil {
			vals = append(vals, appendVal)
		}
	}

	return vals
}

func GetStringMapFeatures(v *viper.Viper, features map[string]bool,
	key string) map[string]interface{} {

	result := map[string]interface{}{}

	slice := GetSliceFeatures(v, features, key)
	for _, itf := range slice {
		sub := cast.ToStringMap(itf)
		for k, v := range sub {
			result[k] = v
		}
	}

	return result
}

func GetStringFeatures(v *viper.Viper, features map[string]bool,
	key string) string {
	val := v.GetString(key)

	// Process the features in alphabetical order to ensure consistent
	// results across repeated runs.
	var featureKeys []string
	for feature, _ := range features {
		featureKeys = append(featureKeys, feature)
	}
	sort.Strings(featureKeys)

	for _, feature := range featureKeys {
		overwriteVal := v.GetString(key + "." + feature + ".OVERWRITE")
		if overwriteVal != "" {
			val = strings.Trim(overwriteVal, "\n")
			break
		}

		appendVal := v.GetString(key + "." + feature)
		if appendVal != "" {
			val += " " + strings.Trim(appendVal, "\n")
		}
	}
	return strings.TrimSpace(val)
}

func GetBoolFeaturesDflt(v *viper.Viper, features map[string]bool,
	key string, dflt bool) (bool, error) {

	s := GetStringFeatures(v, features, key)
	if s == "" {
		return dflt, nil
	}

	b, err := strconv.ParseBool(s)
	if err != nil {
		return dflt, util.FmtNewtError("invalid bool value for %s: %s",
			key, s)
	}

	return b, nil
}

func GetBoolFeatures(v *viper.Viper, features map[string]bool,
	key string) (bool, error) {

	return GetBoolFeaturesDflt(v, features, key, false)
}

func GetStringSliceFeatures(v *viper.Viper, features map[string]bool,
	key string) []string {

	vals := GetSliceFeatures(v, features, key)

	strVals := []string{}
	for _, v := range vals {
		subVals := cast.ToStringSlice(v)
		strVals = append(strVals, subVals...)
	}

	return strVals
}

// Parses a string of the following form:
//     [@repo]<path/to/package>
//
// @return string               repo name ("" if no repo)
//         string               package name
//         error                if invalid package string
func ParsePackageString(pkgStr string) (string, string, error) {
	// remove possible trailing '/'
	pkgStr = strings.TrimSuffix(pkgStr, "/")

	if strings.HasPrefix(pkgStr, "@") {
		nameParts := strings.SplitN(pkgStr[1:], "/", 2)
		if len(nameParts) == 1 {
			return "", "", util.NewNewtError(fmt.Sprintf("Invalid package "+
				"string; contains repo but no package name: %s", pkgStr))
		} else {
			return nameParts[0], nameParts[1], nil
		}
	} else {
		return "", pkgStr, nil
	}
}

func FindRepoDesignator(s string) (int, int) {
	start := strings.Index(s, "@")
	if start == -1 {
		return -1, -1
	}

	len := strings.Index(s[start:], "/")
	if len == -1 {
		return -1, -1
	}

	return start, len
}

func ReplaceRepoDesignators(s string) (string, bool) {
	start, len := FindRepoDesignator(s)
	if start == -1 {
		return s, false
	}
	repoName := s[start+1 : start+len]

	proj := interfaces.GetProject()
	repoPath := proj.FindRepoPath(repoName)
	if repoPath == "" {
		return s, false
	}

	// Trim common project base from repo path.
	relRepoPath := strings.TrimPrefix(repoPath, proj.Path()+"/")

	return s[:start] + relRepoPath + s[start+len:], true
}

func BuildPackageString(repoName string, pkgName string) string {
	if repoName != "" {
		return "@" + repoName + "/" + pkgName
	} else {
		return pkgName
	}
}

func GeneratedPreamble() string {
	return fmt.Sprintf("/**\n * This file was generated by %s\n */\n\n",
		NewtVersionStr)
}
<commit_msg>Update newt version number: 1.0.1-dev.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package newtutil

import (
	"fmt"
	"os/user"
	"sort"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cast"

	"mynewt.apache.org/newt/newt/interfaces"
	"mynewt.apache.org/newt/util"
	"mynewt.apache.org/newt/viper"
)

var NewtVersion Version = Version{1, 0, 0}
var NewtVersionStr string = "Apache Newt (incubating) version: 1.0.1-dev"
var NewtBlinkyTag string = "develop"
var NewtNumJobs int
var NewtForce bool

const NEWTRC_DIR string = ".newt"
const REPOS_FILENAME string = "repos.yml"

const CORE_REPO_NAME string = "apache-mynewt-core"
const ARDUINO_ZERO_REPO_NAME string = "mynewt_arduino_zero"

type Version struct {
	Major    int64
	Minor    int64
	Revision int64
}

func ParseVersion(s string) (Version, error) {
	v := Version{}
	parseErr := util.FmtNewtError("Invalid version string: %s", s)

	parts := strings.Split(s, ".")
	if len(parts) != 3 {
		return v, parseErr
	}

	var err error

	v.Major, err = strconv.ParseInt(parts[0], 10, 64)
	if err != nil {
		return v, parseErr
	}

	v.Minor, err = strconv.ParseInt(parts[1], 10, 64)
	if err != nil {
		return v, parseErr
	}

	v.Revision, err = strconv.ParseInt(parts[2], 10, 64)
	if err != nil {
		return v, parseErr
	}

	return v, nil
}

func (v *Version) String() string {
	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Revision)
}

func VerCmp(v1 Version, v2 Version) int64 {
	if r := v1.Major - v2.Major; r != 0 {
		return r
	}

	if r := v1.Minor - v2.Minor; r != 0 {
		return r
	}

	if r := v1.Revision - v2.Revision; r != 0 {
		return r
	}

	return 0
}

// Contains general newt settings read from $HOME/.newt
var newtrc *viper.Viper

func readNewtrc() *viper.Viper {
	usr, err := user.Current()
	if err != nil {
		log.Warn("Failed to obtain user name")
		return viper.New()
	}

	dir := usr.HomeDir + "/" + NEWTRC_DIR
	v, err := util.ReadConfig(dir, strings.TrimSuffix(REPOS_FILENAME, ".yml"))
	if err != nil {
		log.Debugf("Failed to read %s/%s file", dir, REPOS_FILENAME)
		return viper.New()
	}

	return v
}

func Newtrc() *viper.Viper {
	if newtrc != nil {
		return newtrc
	}

	newtrc = readNewtrc()
	return newtrc
}

func GetSliceFeatures(v *viper.Viper, features map[string]bool,
	key string) []interface{} {

	val := v.Get(key)
	vals := []interface{}{val}

	// Process the features in alphabetical order to ensure consistent
	// results across repeated runs.
	featureKeys := make([]string, 0, len(features))
	for feature, _ := range features {
		featureKeys = append(featureKeys, feature)
	}
	sort.Strings(featureKeys)

	for _, feature := range featureKeys {
		overwriteVal := v.Get(key + "." + feature + ".OVERWRITE")
		if overwriteVal != nil {
			return []interface{}{overwriteVal}
		}

		appendVal := v.Get(key + "." + feature)
		if appendVal != nil {
			vals = append(vals, appendVal)
		}
	}

	return vals
}

func GetStringMapFeatures(v *viper.Viper, features map[string]bool,
	key string) map[string]interface{} {

	result := map[string]interface{}{}

	slice := GetSliceFeatures(v, features, key)
	for _, itf := range slice {
		sub := cast.ToStringMap(itf)
		for k, v := range sub {
			result[k] = v
		}
	}

	return result
}

func GetStringFeatures(v *viper.Viper, features map[string]bool,
	key string) string {
	val := v.GetString(key)

	// Process the features in alphabetical order to ensure consistent
	// results across repeated runs.
	var featureKeys []string
	for feature, _ := range features {
		featureKeys = append(featureKeys, feature)
	}
	sort.Strings(featureKeys)

	for _, feature := range featureKeys {
		overwriteVal := v.GetString(key + "." + feature + ".OVERWRITE")
		if overwriteVal != "" {
			val = strings.Trim(overwriteVal, "\n")
			break
		}

		appendVal := v.GetString(key + "." + feature)
		if appendVal != "" {
			val += " " + strings.Trim(appendVal, "\n")
		}
	}
	return strings.TrimSpace(val)
}

func GetBoolFeaturesDflt(v *viper.Viper, features map[string]bool,
	key string, dflt bool) (bool, error) {

	s := GetStringFeatures(v, features, key)
	if s == "" {
		return dflt, nil
	}

	b, err := strconv.ParseBool(s)
	if err != nil {
		return dflt, util.FmtNewtError("invalid bool value for %s: %s",
			key, s)
	}

	return b, nil
}

func GetBoolFeatures(v *viper.Viper, features map[string]bool,
	key string) (bool, error) {

	return GetBoolFeaturesDflt(v, features, key, false)
}

func GetStringSliceFeatures(v *viper.Viper, features map[string]bool,
	key string) []string {

	vals := GetSliceFeatures(v, features, key)

	strVals := []string{}
	for _, v := range vals {
		subVals := cast.ToStringSlice(v)
		strVals = append(strVals, subVals...)
	}

	return strVals
}

// Parses a string of the following form:
//     [@repo]<path/to/package>
//
// @return string               repo name ("" if no repo)
//         string               package name
//         error                if invalid package string
func ParsePackageString(pkgStr string) (string, string, error) {
	// remove possible trailing '/'
	pkgStr = strings.TrimSuffix(pkgStr, "/")

	if strings.HasPrefix(pkgStr, "@") {
		nameParts := strings.SplitN(pkgStr[1:], "/", 2)
		if len(nameParts) == 1 {
			return "", "", util.NewNewtError(fmt.Sprintf("Invalid package "+
				"string; contains repo but no package name: %s", pkgStr))
		} else {
			return nameParts[0], nameParts[1], nil
		}
	} else {
		return "", pkgStr, nil
	}
}

func FindRepoDesignator(s string) (int, int) {
	start := strings.Index(s, "@")
	if start == -1 {
		return -1, -1
	}

	len := strings.Index(s[start:], "/")
	if len == -1 {
		return -1, -1
	}

	return start, len
}

func ReplaceRepoDesignators(s string) (string, bool) {
	start, len := FindRepoDesignator(s)
	if start == -1 {
		return s, false
	}
	repoName := s[start+1 : start+len]

	proj := interfaces.GetProject()
	repoPath := proj.FindRepoPath(repoName)
	if repoPath == "" {
		return s, false
	}

	// Trim common project base from repo path.
	relRepoPath := strings.TrimPrefix(repoPath, proj.Path()+"/")

	return s[:start] + relRepoPath + s[start+len:], true
}

func BuildPackageString(repoName string, pkgName string) string {
	if repoName != "" {
		return "@" + repoName + "/" + pkgName
	} else {
		return pkgName
	}
}

func GeneratedPreamble() string {
	return fmt.Sprintf("/**\n * This file was generated by %s\n */\n\n",
		NewtVersionStr)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package channels

import (
	"context"
	"encoding/json"
	"fmt"

	"go.uber.org/multierr"
	"k8s.io/apimachinery/pkg/api/meta"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/restmapper"
	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/upup/pkg/fi"
)

type Applier struct {
	Client     dynamic.Interface
	RESTMapper *restmapper.DeferredDiscoveryRESTMapper
}

// Apply applies the manifest to the cluster.
func (p *Applier) Apply(ctx context.Context, manifest []byte) error {
	objects, err := kubemanifest.LoadObjectsFrom(manifest)
	if err != nil {
		return fmt.Errorf("failed to parse objects: %w", err)
	}

	objectsByGVK := make(map[schema.GroupVersionKind][]*kubemanifest.Object)
	for _, object := range objects {
		key := object.GetNamespace() + "/" + object.GetName()
		gv, err := schema.ParseGroupVersion(object.APIVersion())
		if err != nil || gv.Version == "" {
			return fmt.Errorf("failed to parse apiVersion %q in object %s", object.APIVersion(), key)
		}
		kind := object.Kind()
		if kind == "" {
			return fmt.Errorf("failed to find kind in object %s", key)
		}

		gvk := gv.WithKind(kind)
		objectsByGVK[gvk] = append(objectsByGVK[gvk], object)
	}

	var applyErrors error
	for gvk := range objectsByGVK {
		if err := p.applyObjectsOfKind(ctx, gvk, objectsByGVK[gvk]); err != nil {
			applyErrors = multierr.Append(applyErrors, fmt.Errorf("failed to apply objects of kind %s: %w", gvk, err))
		}
	}
	return applyErrors
}

func (p *Applier) applyObjectsOfKind(ctx context.Context, gvk schema.GroupVersionKind, expectedObjects []*kubemanifest.Object) error {
	klog.V(2).Infof("applying objects of kind: %v", gvk)

	restMapping, err := p.RESTMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
	if err != nil {
		return fmt.Errorf("unable to find resource for %s: %w", gvk, err)
	}

	if err := p.applyObjects(ctx, restMapping, expectedObjects); err != nil {
		return err
	}

	return nil
}

func (p *Applier) applyObjects(ctx context.Context, restMapping *meta.RESTMapping, expectedObjects []*kubemanifest.Object) error {
	var merr error

	for _, expectedObject := range expectedObjects {
		err := p.patchObject(ctx, restMapping, expectedObject)
		merr = multierr.Append(merr, err)
	}

	return merr
}

func (p *Applier) patchObject(ctx context.Context, restMapping *meta.RESTMapping, expectedObject *kubemanifest.Object) error {
	gvr := restMapping.Resource
	name := expectedObject.GetName()
	namespace := expectedObject.GetNamespace()
	key := namespace + "/" + name

	var resource dynamic.ResourceInterface

	if restMapping.Scope.Name() == meta.RESTScopeNameNamespace {
		if namespace == "" {
			return fmt.Errorf("namespace not set for namespace-scoped object %q", key)
		}
		resource = p.Client.Resource(gvr).Namespace(namespace)
	} else {
		if namespace != "" {
			return fmt.Errorf("namespace was set for cluster-scoped object %q", key)
		}
		resource = p.Client.Resource(gvr)
	}

	obj := expectedObject.ToUnstructured()

	jsonData, err := json.Marshal(obj)
	if err != nil {
		return fmt.Errorf("failed to marsal %q into json: %w", obj.GetName(), err)
	}

	{
		_, err := resource.Patch(ctx, obj.GetName(), types.ApplyPatchType, jsonData, v1.PatchOptions{FieldManager: "kops", Force: fi.Bool(true)})
		if err != nil {
			return fmt.Errorf("failed to path object %q: %w", obj.GetName(), err)
		}
	}
	return nil
}
<commit_msg>Fix typo in channels error message<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package channels

import (
	"context"
	"encoding/json"
	"fmt"

	"go.uber.org/multierr"
	"k8s.io/apimachinery/pkg/api/meta"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/restmapper"
	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/upup/pkg/fi"
)

type Applier struct {
	Client     dynamic.Interface
	RESTMapper *restmapper.DeferredDiscoveryRESTMapper
}

// Apply applies the manifest to the cluster.
func (p *Applier) Apply(ctx context.Context, manifest []byte) error {
	objects, err := kubemanifest.LoadObjectsFrom(manifest)
	if err != nil {
		return fmt.Errorf("failed to parse objects: %w", err)
	}

	objectsByGVK := make(map[schema.GroupVersionKind][]*kubemanifest.Object)
	for _, object := range objects {
		key := object.GetNamespace() + "/" + object.GetName()
		gv, err := schema.ParseGroupVersion(object.APIVersion())
		if err != nil || gv.Version == "" {
			return fmt.Errorf("failed to parse apiVersion %q in object %s", object.APIVersion(), key)
		}
		kind := object.Kind()
		if kind == "" {
			return fmt.Errorf("failed to find kind in object %s", key)
		}

		gvk := gv.WithKind(kind)
		objectsByGVK[gvk] = append(objectsByGVK[gvk], object)
	}

	var applyErrors error
	for gvk := range objectsByGVK {
		if err := p.applyObjectsOfKind(ctx, gvk, objectsByGVK[gvk]); err != nil {
			applyErrors = multierr.Append(applyErrors, fmt.Errorf("failed to apply objects of kind %s: %w", gvk, err))
		}
	}
	return applyErrors
}

func (p *Applier) applyObjectsOfKind(ctx context.Context, gvk schema.GroupVersionKind, expectedObjects []*kubemanifest.Object) error {
	klog.V(2).Infof("applying objects of kind: %v", gvk)

	restMapping, err := p.RESTMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
	if err != nil {
		return fmt.Errorf("unable to find resource for %s: %w", gvk, err)
	}

	if err := p.applyObjects(ctx, restMapping, expectedObjects); err != nil {
		return err
	}

	return nil
}

func (p *Applier) applyObjects(ctx context.Context, restMapping *meta.RESTMapping, expectedObjects []*kubemanifest.Object) error {
	var merr error

	for _, expectedObject := range expectedObjects {
		err := p.patchObject(ctx, restMapping, expectedObject)
		merr = multierr.Append(merr, err)
	}

	return merr
}

func (p *Applier) patchObject(ctx context.Context, restMapping *meta.RESTMapping, expectedObject *kubemanifest.Object) error {
	gvr := restMapping.Resource
	name := expectedObject.GetName()
	namespace := expectedObject.GetNamespace()
	key := namespace + "/" + name

	var resource dynamic.ResourceInterface

	if restMapping.Scope.Name() == meta.RESTScopeNameNamespace {
		if namespace == "" {
			return fmt.Errorf("namespace not set for namespace-scoped object %q", key)
		}
		resource = p.Client.Resource(gvr).Namespace(namespace)
	} else {
		if namespace != "" {
			return fmt.Errorf("namespace was set for cluster-scoped object %q", key)
		}
		resource = p.Client.Resource(gvr)
	}

	obj := expectedObject.ToUnstructured()

	jsonData, err := json.Marshal(obj)
	if err != nil {
		return fmt.Errorf("failed to marsal %q into json: %w", obj.GetName(), err)
	}

	{
		_, err := resource.Patch(ctx, obj.GetName(), types.ApplyPatchType, jsonData, v1.PatchOptions{FieldManager: "kops", Force: fi.Bool(true)})
		if err != nil {
			return fmt.Errorf("failed to patch object %q: %w", obj.GetName(), err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package secret

import (
	"bytes"
	"log"
	"testing"
)

var (
	testMessage = []byte("Do not go gentle into that good night.")
	testKey     *[32]byte
)

func TestGenerateKey(t *testing.T) {
	var err error
	testKey, err = GenerateKey()
	if err != nil {
		t.Fatalf("%v", err)
	}
}

func TestEncrypt(t *testing.T) {
	ct, err := Encrypt(testKey, testMessage)
	if err != nil {
		t.Fatalf("%v", err)
	}

	pt, err := Decrypt(testKey, ct)
	if err != nil {
		t.Fatalf("%v", err)
	}

	if !bytes.Equal(testMessage, pt) {
		log.Fatalf("messages don't match")
	}
}
<commit_msg>Full test coverage<commit_after>package secret

import (
	"bytes"
	"crypto/rand"
	"testing"

	"golang.org/x/crypto/nacl/secretbox"
)

var (
	testMessage = []byte("Do not go gentle into that good night.")
	testKey     *[32]byte
)

/*
 * The following tests verify the positive functionality of this package:
 * can an encrypted message be decrypted?
 */

func TestGenerateKey(t *testing.T) {
	var err error
	testKey, err = GenerateKey()
	if err != nil {
		t.Fatalf("%v", err)
	}
}

func TestEncrypt(t *testing.T) {
	ct, err := Encrypt(testKey, testMessage)
	if err != nil {
		t.Fatalf("%v", err)
	}

	pt, err := Decrypt(testKey, ct)
	if err != nil {
		t.Fatalf("%v", err)
	}

	if !bytes.Equal(testMessage, pt) {
		t.Fatalf("messages don't match")
	}
}

/*
 * The following tests verify the negative functionality of this package:
 * does it fail when it should?
 */

func prngTester(size int, testFunc func()) {
	prng := rand.Reader
	buf := &bytes.Buffer{}

	rand.Reader = buf
	defer func() { rand.Reader = prng }()

	for i := 0; i < size; i++ {
		tmp := make([]byte, i)
		buf.Write(tmp)
		testFunc()
	}
}

func TestPRNGFailures(t *testing.T) {
	testFunc := func() {
		_, err := GenerateKey()
		if err == nil {
			t.Fatal("expected key generation failure with bad PRNG")
		}
	}
	prngTester(32, testFunc)

	testFunc = func() {
		_, err := GenerateNonce()
		if err == nil {
			t.Fatal("expected nonce generation failure with bad PRNG")
		}
	}
	prngTester(24, testFunc)

	testFunc = func() {
		_, err := Encrypt(testKey, testMessage)
		if err == nil {
			t.Fatal("expected encryption failure with bad PRNG")
		}
	}
	prngTester(24, testFunc)
}

func TestDecryptFailures(t *testing.T) {
	targetLength := 24 + secretbox.Overhead

	for i := 0; i < targetLength; i++ {
		buf := make([]byte, i)
		if _, err := Decrypt(testKey, buf); err == nil {
			t.Fatal("expected decryption failure with bad message length")
		}
	}

	otherKey, err := GenerateKey()
	if err != nil {
		t.Fatalf("%v", err)
	}

	ct, err := Encrypt(testKey, testMessage)
	if err != nil {
		t.Fatalf("%v", err)
	}

	if _, err = Decrypt(otherKey, ct); err == nil {
		t.Fatal("decrypt should fail with wrong key")
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"errors"
	"fmt"
	"io"

	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
	k "github.com/ipfs/go-ipfs/blocks/key"
	cmds "github.com/ipfs/go-ipfs/commands"
	"github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/filestore"
	"github.com/ipfs/go-ipfs/repo/fsrepo"
)

type chanWriter struct {
	ch     <-chan *filestore.ListRes
	buf    string
	offset int
}

func (w *chanWriter) Read(p []byte) (int, error) {
	if w.offset >= len(w.buf) {
		w.offset = 0
		res, more := <-w.ch
		if !more {
			return 0, io.EOF
		}
		w.buf = res.Format()
	}
	sz := copy(p, w.buf[w.offset:])
	w.offset += sz
	return sz, nil
}

var FileStoreCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Interact with filestore objects",
	},
	Subcommands: map[string]*cmds.Command{
		"ls":                 lsFileStore,
		"verify":             verifyFileStore,
		"find-dangling-pins": findDanglingPins,
	},
}

var lsFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List objects on filestore",
	},

	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch := make(chan *filestore.ListRes)
		go func() {
			defer close(ch)
			filestore.List(fs, ch)
		}()
		res.SetOutput(&chanWriter{ch, "", 0})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

var verifyFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Verify objects in filestore",
	},

	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch := make(chan *filestore.ListRes)
		go func() {
			defer close(ch)
			filestore.Verify(fs, ch)
		}()
		res.SetOutput(&chanWriter{ch, "", 0})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

func extractFilestore(req cmds.Request) (node *core.IpfsNode, fs *filestore.Datastore, err error) {
	node, err = req.InvocContext().GetNode()
	if err != nil {
		return
	}
	repo, ok := node.Repo.Self().(*fsrepo.FSRepo)
	if !ok {
		err = errors.New("Not a FSRepo")
		return
	}
	fs = repo.Filestore()
	return
}

var findDanglingPins = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List pinned objects that no longer exists",
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			return
		}
		r, w := io.Pipe()
		go func() {
			defer w.Close()
			err := listDanglingPins(n.Pinning.DirectKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			err = listDanglingPins(n.Pinning.RecursiveKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
		}()
		res.SetOutput(r)
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

func listDanglingPins(keys []k.Key, out io.Writer, d bs.Blockstore) error {
	for _, k := range keys {
		exists, err := d.Has(k)
		if err != nil {
			return err
		}
		if !exists {
			fmt.Fprintln(out, k.B58String())
		}
	}
	return nil
}
<commit_msg>Add "filestore rm" command.<commit_after>package commands

import (
	"errors"
	"fmt"
	"io"

	//ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
	k "github.com/ipfs/go-ipfs/blocks/key"
	cmds "github.com/ipfs/go-ipfs/commands"
	"github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/filestore"
	"github.com/ipfs/go-ipfs/repo/fsrepo"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)

type chanWriter struct {
	ch     <-chan *filestore.ListRes
	buf    string
	offset int
}

func (w *chanWriter) Read(p []byte) (int, error) {
	if w.offset >= len(w.buf) {
		w.offset = 0
		res, more := <-w.ch
		if !more {
			return 0, io.EOF
		}
		w.buf = res.Format()
	}
	sz := copy(p, w.buf[w.offset:])
	w.offset += sz
	return sz, nil
}

var FileStoreCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Interact with filestore objects",
	},
	Subcommands: map[string]*cmds.Command{
		"ls":                 lsFileStore,
		"verify":             verifyFileStore,
		"rm":                 rmFilestoreObjs,
		"find-dangling-pins": findDanglingPins,
	},
}

var lsFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List objects on filestore",
	},

	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch := make(chan *filestore.ListRes)
		go func() {
			defer close(ch)
			filestore.List(fs, ch)
		}()
		res.SetOutput(&chanWriter{ch, "", 0})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

var verifyFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Verify objects in filestore",
	},

	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch := make(chan *filestore.ListRes)
		go func() {
			defer close(ch)
			filestore.Verify(fs, ch)
		}()
		res.SetOutput(&chanWriter{ch, "", 0})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

var rmFilestoreObjs = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Remove objects from the Filestore",
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("hash", true, true, "Multi-hashes to remove.").EnableStdin(),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		node, fs, err := extractFilestore(req)
		_ = fs
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		hashes := req.Arguments()
		serr := res.Stderr()
		numErrors := 0
		for _, mhash := range hashes {
			err = delFilestoreObj(req, node, fs, mhash)
			if err != nil {
				fmt.Fprintf(serr, "Error deleting %s: %s\n", mhash, err.Error())
				numErrors += 1
			}
		}
		if numErrors > 0 {
			res.SetError(errors.New("Could not delete some keys"), cmds.ErrNormal)
			return
		}
		return
	},
}

func delFilestoreObj(req cmds.Request, node *core.IpfsNode, fs *filestore.Datastore, mhash string) error {
	key := k.B58KeyDecode(mhash)
	err := fs.DeleteDirect(key.DsKey())
	if err != nil {
		return err
	}
	stillExists, err := node.Blockstore.Has(key)
	if err != nil {
		return err
	}
	if stillExists {
		return nil
	}
	_, pinned1, err := node.Pinning.IsPinnedWithType(key, "recursive")
	if err != nil {
		return err
	}
	_, pinned2, err := node.Pinning.IsPinnedWithType(key, "direct")
	if err != nil {
		return err
	}
	if pinned1 || pinned2 {
		println("unpinning")
		ctx, cancel := context.WithCancel(req.Context())
		defer cancel()
		err = node.Pinning.Unpin(ctx, key, true)
		if err != nil {
			return err
		}
		err := node.Pinning.Flush()
		if err != nil {
			return err
		}
	}
	return nil
}

func extractFilestore(req cmds.Request) (node *core.IpfsNode, fs *filestore.Datastore, err error) {
	node, err = req.InvocContext().GetNode()
	if err != nil {
		return
	}
	repo, ok := node.Repo.Self().(*fsrepo.FSRepo)
	if !ok {
		err = errors.New("Not a FSRepo")
		return
	}
	fs = repo.Filestore()
	return
}

var findDanglingPins = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List pinned objects that no longer exists",
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			return
		}
		r, w := io.Pipe()
		go func() {
			defer w.Close()
			err := listDanglingPins(n.Pinning.DirectKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			err = listDanglingPins(n.Pinning.RecursiveKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
		}()
		res.SetOutput(r)
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

func listDanglingPins(keys []k.Key, out io.Writer, d bs.Blockstore) error {
	for _, k := range keys {
		exists, err := d.Has(k)
		if err != nil {
			return err
		}
		if !exists {
			fmt.Fprintln(out, k.B58String())
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package expvar

import (
	"bytes"
	"encoding/json"
	"net/http/httptest"
	"strconv"
	"testing"
)

// RemoveAll removes all exported variables.
// This is for tests only.
func RemoveAll() {
	mutex.Lock()
	defer mutex.Unlock()
	vars = make(map[string]Var)
	varKeys = nil
}

func TestInt(t *testing.T) {
	RemoveAll()
	reqs := NewInt("requests")
	if reqs.i != 0 {
		t.Errorf("reqs.i = %v, want 0", reqs.i)
	}
	if reqs != Get("requests").(*Int) {
		t.Errorf("Get() failed.")
	}

	reqs.Add(1)
	reqs.Add(3)
	if reqs.i != 4 {
		t.Errorf("reqs.i = %v, want 4", reqs.i)
	}

	if s := reqs.String(); s != "4" {
		t.Errorf("reqs.String() = %q, want \"4\"", s)
	}

	reqs.Set(-2)
	if reqs.i != -2 {
		t.Errorf("reqs.i = %v, want -2", reqs.i)
	}
}

func TestFloat(t *testing.T) {
	RemoveAll()
	reqs := NewFloat("requests-float")
	if reqs.f != 0.0 {
		t.Errorf("reqs.f = %v, want 0", reqs.f)
	}
	if reqs != Get("requests-float").(*Float) {
		t.Errorf("Get() failed.")
	}

	reqs.Add(1.5)
	reqs.Add(1.25)
	if reqs.f != 2.75 {
		t.Errorf("reqs.f = %v, want 2.75", reqs.f)
	}

	if s := reqs.String(); s != "2.75" {
		t.Errorf("reqs.String() = %q, want \"4.64\"", s)
	}

	reqs.Add(-2)
	if reqs.f != 0.75 {
		t.Errorf("reqs.f = %v, want 0.75", reqs.f)
	}
}

func TestString(t *testing.T) {
	RemoveAll()
	name := NewString("my-name")
	if name.s != "" {
		t.Errorf("name.s = %q, want \"\"", name.s)
	}

	name.Set("Mike")
	if name.s != "Mike" {
		t.Errorf("name.s = %q, want \"Mike\"", name.s)
	}

	if s := name.String(); s != "\"Mike\"" {
		t.Errorf("reqs.String() = %q, want \"\"Mike\"\"", s)
	}
}

func TestMapCounter(t *testing.T) {
	RemoveAll()
	colors := NewMap("bike-shed-colors")

	colors.Add("red", 1)
	colors.Add("red", 2)
	colors.Add("blue", 4)
	colors.AddFloat(`green "midori"`, 4.125)
	if x := colors.m["red"].(*Int).i; x != 3 {
		t.Errorf("colors.m[\"red\"] = %v, want 3", x)
	}
	if x := colors.m["blue"].(*Int).i; x != 4 {
		t.Errorf("colors.m[\"blue\"] = %v, want 4", x)
	}
	if x := colors.m[`green "midori"`].(*Float).f; x != 4.125 {
		t.Errorf("colors.m[`green \"midori\"] = %v, want 3.14", x)
	}

	// colors.String() should be '{"red":3, "blue":4}',
	// though the order of red and blue could vary.
	s := colors.String()
	var j interface{}
	err := json.Unmarshal([]byte(s), &j)
	if err != nil {
		t.Errorf("colors.String() isn't valid JSON: %v", err)
	}
	m, ok := j.(map[string]interface{})
	if !ok {
		t.Error("colors.String() didn't produce a map.")
	}
	red := m["red"]
	x, ok := red.(float64)
	if !ok {
		t.Error("red.Kind() is not a number.")
	}
	if x != 3 {
		t.Errorf("red = %v, want 3", x)
	}
}

func TestFunc(t *testing.T) {
	RemoveAll()
	var x interface{} = []string{"a", "b"}
	f := Func(func() interface{} { return x })
	if s, exp := f.String(), `["a","b"]`; s != exp {
		t.Errorf(`f.String() = %q, want %q`, s, exp)
	}

	x = 17
	if s, exp := f.String(), `17`; s != exp {
		t.Errorf(`f.String() = %q, want %q`, s, exp)
	}
}

func TestHandler(t *testing.T) {
	RemoveAll()
	m := NewMap("map1")
	m.Add("a", 1)
	m.Add("z", 2)
	m2 := NewMap("map2")
	for i := 0; i < 9; i++ {
		m2.Add(strconv.Itoa(i), int64(i))
	}
	rr := httptest.NewRecorder()
	rr.Body = new(bytes.Buffer)
	expvarHandler(rr, nil)
	want := `{
"map1": {"a": 1, "z": 2},
"map2": {"0": 0, "1": 1, "2": 2, "3": 3, "4": 4, "5": 5, "6": 6, "7": 7, "8": 8}
}
`
	if got := rr.Body.String(); got != want {
		t.Errorf("HTTP handler wrote:\n%s\nWant:\n%s", got, want)
	}
}
<commit_msg>expvar: Add benchmarks for perf sensitive operations<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package expvar

import (
	"bytes"
	"encoding/json"
	"net"
	"net/http/httptest"
	"runtime"
	"strconv"
	"sync"
	"testing"
)

// RemoveAll removes all exported variables.
// This is for tests only.
func RemoveAll() {
	mutex.Lock()
	defer mutex.Unlock()
	vars = make(map[string]Var)
	varKeys = nil
}

func TestInt(t *testing.T) {
	RemoveAll()
	reqs := NewInt("requests")
	if reqs.i != 0 {
		t.Errorf("reqs.i = %v, want 0", reqs.i)
	}
	if reqs != Get("requests").(*Int) {
		t.Errorf("Get() failed.")
	}

	reqs.Add(1)
	reqs.Add(3)
	if reqs.i != 4 {
		t.Errorf("reqs.i = %v, want 4", reqs.i)
	}

	if s := reqs.String(); s != "4" {
		t.Errorf("reqs.String() = %q, want \"4\"", s)
	}

	reqs.Set(-2)
	if reqs.i != -2 {
		t.Errorf("reqs.i = %v, want -2", reqs.i)
	}
}

func BenchmarkIntAdd(b *testing.B) {
	var v Int

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			v.Add(1)
		}
	})
}

func BenchmarkIntSet(b *testing.B) {
	var v Int

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			v.Set(1)
		}
	})
}

func TestFloat(t *testing.T) {
	RemoveAll()
	reqs := NewFloat("requests-float")
	if reqs.f != 0.0 {
		t.Errorf("reqs.f = %v, want 0", reqs.f)
	}
	if reqs != Get("requests-float").(*Float) {
		t.Errorf("Get() failed.")
	}

	reqs.Add(1.5)
	reqs.Add(1.25)
	if reqs.f != 2.75 {
		t.Errorf("reqs.f = %v, want 2.75", reqs.f)
	}

	if s := reqs.String(); s != "2.75" {
		t.Errorf("reqs.String() = %q, want \"4.64\"", s)
	}

	reqs.Add(-2)
	if reqs.f != 0.75 {
		t.Errorf("reqs.f = %v, want 0.75", reqs.f)
	}
}

func BenchmarkFloatAdd(b *testing.B) {
	var f Float

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			f.Add(1.0)
		}
	})
}

func BenchmarkFloatSet(b *testing.B) {
	var f Float

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			f.Set(1.0)
		}
	})
}

func TestString(t *testing.T) {
	RemoveAll()
	name := NewString("my-name")
	if name.s != "" {
		t.Errorf("name.s = %q, want \"\"", name.s)
	}

	name.Set("Mike")
	if name.s != "Mike" {
		t.Errorf("name.s = %q, want \"Mike\"", name.s)
	}

	if s := name.String(); s != "\"Mike\"" {
		t.Errorf("reqs.String() = %q, want \"\"Mike\"\"", s)
	}
}

func BenchmarkStringSet(b *testing.B) {
	var s String

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			s.Set("red")
		}
	})
}

func TestMapCounter(t *testing.T) {
	RemoveAll()
	colors := NewMap("bike-shed-colors")

	colors.Add("red", 1)
	colors.Add("red", 2)
	colors.Add("blue", 4)
	colors.AddFloat(`green "midori"`, 4.125)
	if x := colors.m["red"].(*Int).i; x != 3 {
		t.Errorf("colors.m[\"red\"] = %v, want 3", x)
	}
	if x := colors.m["blue"].(*Int).i; x != 4 {
		t.Errorf("colors.m[\"blue\"] = %v, want 4", x)
	}
	if x := colors.m[`green "midori"`].(*Float).f; x != 4.125 {
		t.Errorf("colors.m[`green \"midori\"] = %v, want 3.14", x)
	}

	// colors.String() should be '{"red":3, "blue":4}',
	// though the order of red and blue could vary.
	s := colors.String()
	var j interface{}
	err := json.Unmarshal([]byte(s), &j)
	if err != nil {
		t.Errorf("colors.String() isn't valid JSON: %v", err)
	}
	m, ok := j.(map[string]interface{})
	if !ok {
		t.Error("colors.String() didn't produce a map.")
	}
	red := m["red"]
	x, ok := red.(float64)
	if !ok {
		t.Error("red.Kind() is not a number.")
	}
	if x != 3 {
		t.Errorf("red = %v, want 3", x)
	}
}

func BenchmarkMapSet(b *testing.B) {
	m := new(Map).Init()

	v := new(Int)

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			m.Set("red", v)
		}
	})
}

func BenchmarkMapAddSame(b *testing.B) {
	for i := 0; i < b.N; i++ {
		m := new(Map).Init()
		m.Add("red", 1)
		m.Add("red", 1)
		m.Add("red", 1)
		m.Add("red", 1)
	}
}

func BenchmarkMapAddDifferent(b *testing.B) {
	for i := 0; i < b.N; i++ {
		m := new(Map).Init()
		m.Add("red", 1)
		m.Add("blue", 1)
		m.Add("green", 1)
		m.Add("yellow", 1)
	}
}

func TestFunc(t *testing.T) {
	RemoveAll()
	var x interface{} = []string{"a", "b"}
	f := Func(func() interface{} { return x })
	if s, exp := f.String(), `["a","b"]`; s != exp {
		t.Errorf(`f.String() = %q, want %q`, s, exp)
	}

	x = 17
	if s, exp := f.String(), `17`; s != exp {
		t.Errorf(`f.String() = %q, want %q`, s, exp)
	}
}

func TestHandler(t *testing.T) {
	RemoveAll()
	m := NewMap("map1")
	m.Add("a", 1)
	m.Add("z", 2)
	m2 := NewMap("map2")
	for i := 0; i < 9; i++ {
		m2.Add(strconv.Itoa(i), int64(i))
	}
	rr := httptest.NewRecorder()
	rr.Body = new(bytes.Buffer)
	expvarHandler(rr, nil)
	want := `{
"map1": {"a": 1, "z": 2},
"map2": {"0": 0, "1": 1, "2": 2, "3": 3, "4": 4, "5": 5, "6": 6, "7": 7, "8": 8}
}
`
	if got := rr.Body.String(); got != want {
		t.Errorf("HTTP handler wrote:\n%s\nWant:\n%s", got, want)
	}
}

func BenchmarkRealworldExpvarUsage(b *testing.B) {
	var (
		bytesSent Int
		bytesRead Int
	)

	// The benchmark creates GOMAXPROCS client/server pairs.
	// Each pair creates 4 goroutines: client reader/writer and server reader/writer.
	// The benchmark stresses concurrent reading and writing to the same connection.
	// Such pattern is used in net/http and net/rpc.

	b.StopTimer()

	P := runtime.GOMAXPROCS(0)
	N := b.N / P
	W := 1000

	// Setup P client/server connections.
	clients := make([]net.Conn, P)
	servers := make([]net.Conn, P)
	ln, err := net.Listen("tcp", laddr)
	if err != nil {
		b.Fatalf("Listen failed: %v", err)
	}
	defer ln.Close()
	done := make(chan bool)
	go func() {
		for p := 0; p < P; p++ {
			s, err := ln.Accept()
			if err != nil {
				b.Errorf("Accept failed: %v", err)
				return
			}
			servers[p] = s
		}
		done <- true
	}()
	for p := 0; p < P; p++ {
		c, err := net.Dial("tcp", ln.Addr().String())
		if err != nil {
			b.Fatalf("Dial failed: %v", err)
		}
		clients[p] = c
	}
	<-done

	b.StartTimer()

	var wg sync.WaitGroup
	wg.Add(4 * P)
	for p := 0; p < P; p++ {
		// Client writer.
		go func(c net.Conn) {
			defer wg.Done()
			var buf [1]byte
			for i := 0; i < N; i++ {
				v := byte(i)
				for w := 0; w < W; w++ {
					v *= v
				}
				buf[0] = v
				n, err := c.Write(buf[:])
				if err != nil {
					b.Errorf("Write failed: %v", err)
					return
				}

				bytesSent.Add(int64(n))
			}
		}(clients[p])

		// Pipe between server reader and server writer.
		pipe := make(chan byte, 128)

		// Server reader.
		go func(s net.Conn) {
			defer wg.Done()
			var buf [1]byte
			for i := 0; i < N; i++ {
				n, err := s.Read(buf[:])

				if err != nil {
					b.Errorf("Read failed: %v", err)
					return
				}

				bytesRead.Add(int64(n))
				pipe <- buf[0]
			}
		}(servers[p])

		// Server writer.
		go func(s net.Conn) {
			defer wg.Done()
			var buf [1]byte
			for i := 0; i < N; i++ {
				v := <-pipe
				for w := 0; w < W; w++ {
					v *= v
				}
				buf[0] = v
				n, err := s.Write(buf[:])
				if err != nil {
					b.Errorf("Write failed: %v", err)
					return
				}

				bytesSent.Add(int64(n))
			}
			s.Close()
		}(servers[p])

		// Client reader.
		go func(c net.Conn) {
			defer wg.Done()
			var buf [1]byte
			for i := 0; i < N; i++ {
				n, err := c.Read(buf[:])

				if err != nil {
					b.Errorf("Read failed: %v", err)
					return
				}

				bytesRead.Add(int64(n))
			}
			c.Close()
		}(clients[p])
	}
	wg.Wait()
}
<|endoftext|>