<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package main contains a simple command line tool for Timezone API
// Directions docs: https://developers.google.com/maps/documentation/timezone/
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/kr/pretty"
	"google.golang.org/maps"
)

var (
	apiKey    = flag.String("key", "", "API Key for using Google Maps API.")
	location  = flag.String("location", "", "a comma-separated lat,lng tuple (eg. location=-33.86,151.20), representing the location to look up.")
	timestamp = flag.String("timestamp", "", "specifies the desired time as seconds since midnight, January 1, 1970 UTC.")
	language  = flag.String("language", "", "The language in which to return results.")
)

func usageAndExit(msg string) {
	fmt.Fprintln(os.Stderr, msg)
	fmt.Println("Flags:")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Parse()
	client := &http.Client{}
	if *apiKey == "" {
		usageAndExit("Please specify an API Key.")
	}
	ctx := maps.NewContext(*apiKey, client)
	r := &maps.TimezoneRequest{
		Timestamp: *timestamp,
		Language:  *language,
	}

	parseLocation(*location, r)

	pretty.Println(r)

	resp, err := r.Get(ctx)
	if err != nil {
		log.Fatalf("error %v", err)
	}

	pretty.Println(resp)
}

func parseLocation(location string, r *maps.TimezoneRequest) {
	if location != "" {
		l := strings.Split(location, ",")
		lat, err := strconv.ParseFloat(l[0], 64)
		if err != nil {
			log.Fatalf("Couldn't parse latlng: %#v", err)
		}
		lng, err := strconv.ParseFloat(l[1], 64)
		if err != nil {
			log.Fatalf("Couldn't parse latlng: %#v", err)
		}
		r.Location = &maps.LatLng{
			Lat: lat,
			Lng: lng,
		}
	} else {
		usageAndExit("latlng is required")
	}
}
<commit_msg>Tidyup<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package main contains a simple command line tool for Timezone API
// Directions docs: https://developers.google.com/maps/documentation/timezone/
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/kr/pretty"
	"google.golang.org/maps"
)

var (
	apiKey    = flag.String("key", "", "API Key for using Google Maps API.")
	location  = flag.String("location", "", "a comma-separated lat,lng tuple (eg. location=-33.86,151.20), representing the location to look up.")
	timestamp = flag.String("timestamp", "", "specifies the desired time as seconds since midnight, January 1, 1970 UTC.")
	language  = flag.String("language", "", "The language in which to return results.")
)

func usageAndExit(msg string) {
	fmt.Fprintln(os.Stderr, msg)
	fmt.Println("Flags:")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Parse()
	client := &http.Client{}
	if *apiKey == "" {
		usageAndExit("Please specify an API Key.")
	}
	ctx := maps.NewContext(*apiKey, client)
	r := &maps.TimezoneRequest{
		Timestamp: *timestamp,
		Language:  *language,
	}

	parseLocation(*location, r)

	pretty.Println(r)

	resp, err := r.Get(ctx)
	if err != nil {
		log.Fatalf("error %v", err)
	}

	pretty.Println(resp)
}

func parseLocation(location string, r *maps.TimezoneRequest) {
	if location != "" {
		l := strings.Split(location, ",")
		lat, err := strconv.ParseFloat(l[0], 64)
		if err != nil {
			log.Fatalf("Couldn't parse latlng: %#v", err)
		}
		lng, err := strconv.ParseFloat(l[1], 64)
		if err != nil {
			log.Fatalf("Couldn't parse latlng: %#v", err)
		}
		r.Location = &maps.LatLng{
			Lat: lat,
			Lng: lng,
		}
	} else {
		usageAndExit("location is required")
	}
}
<|endoftext|>


<commit_before>package jwthelper

import (
	"errors"
	"fmt"
	"github.com/dgrijalva/jwt-go"
	"github.com/northbright/errorhelper"
	"github.com/northbright/pathhelper"
	"io/ioutil"
	"net/http"
	"sync"
)

// Key struct consists of algorithm, signning key and verifying key.
type Key struct {
	Method    jwt.SigningMethod // jwt.SigningMethod
	SignKey   interface{}       // Signing key. HMAC: []byte, RSA: *crypto/rsa.PrivateKey, ECDSA: *crypto/ecdsa.PrivateKey.
	VerifyKey interface{}       // Verifying key. HMAC: []byte, RSA: *crypto/rsa.PublicKey, ECDSA: *crypto/ecdsa.PublicKey.
}

//  KeyManger manages the keys by using kid(key id).
type KeyManager struct {
	Keys         map[string]*Key // Key map. Key: kid(key id), Value: Key Struct
	sync.RWMutex                 // Access map concurrently.
}

var (
	km KeyManager = KeyManager{} // Internal key manager.
)

// ReadKey() reads key bytes from the key file.
func ReadKey(keyFile string) (key []byte, err error) {
	if err := errorhelper.GenEmptyStringError(keyFile, "keyFile"); err != nil {
		return nil, err
	}

	// Make Abs key file path with current executable path if KeyFilePath is relative.
	p := ""
	if p, err = pathhelper.GetAbsPath(keyFile); err != nil {
		return nil, err
	}

	buf := []byte{}
	if buf, err = ioutil.ReadFile(p); err != nil {
		return nil, err
	}

	return buf, nil
}

// setKey() sets the kid - Key pair.
//
//   Params:
//       kid: Key id. It should be unique.
//       key: Key struct.
func setKey(kid string, key *Key) {
	km.Lock()
	km.Keys[kid] = key
	km.Unlock()
}

// GetKey() return the key struct by given kid.
func GetKey(kid string) (k *Key, err error) {
	if err := errorhelper.GenEmptyStringError(kid, "kid"); err != nil {
		return nil, err
	}

	km.RLock()
	k, ok := km.Keys[kid]
	km.RUnlock()

	if !ok {
		return nil, errorhelper.GenMapKeyNotFoundError(kid)
	}

	return k, nil
}

// SetKeyFromFile() reads the keys from the key files then stores the unique kid - Key struct pair.
//
//   Params:
//       kid: Key id(unique).
//       alg: JWT alg.
//       signKeyFile: Signing key file.
//       verifyKeyFile: Verifying key file.
//   Return:
//       err: error.
func SetKeyFromFile(kid, alg, signKeyFile, verifyKeyFile string) (err error) {
	key := &Key{}

	if err := errorhelper.GenEmptyStringError(kid, "kid"); err != nil {
		return err
	}

	if err := errorhelper.GenEmptyStringError(alg, "alg"); err != nil {
		return err
	}

	m := jwt.GetSigningMethod(alg)
	if m == nil {
		return errors.New(fmt.Sprintf("Incorrect alg: %s. Available algs: HS246,HS384,HS512,RS256,RS384,RS512,ES256,ES384,ES512", alg))
	}

	// Set Signing Method
	key.Method = m

	switch alg {
	case "HS256":
	case "HS384":
	case "HS512":
		if key.SignKey, err = ReadKey(signKeyFile); err != nil {
			return err
		}
		key.VerifyKey = key.SignKey

	case "RS256":
	case "RS384":
	case "RS512":
		buf := []byte{}
		if buf, err = ReadKey(signKeyFile); err != nil {
			return err
		}

		if key.SignKey, err = jwt.ParseRSAPrivateKeyFromPEM(buf); err != nil {
			return err
		}

		if buf, err = ReadKey(verifyKeyFile); err != nil {
			return err
		}

		if key.VerifyKey, err = jwt.ParseRSAPublicKeyFromPEM(buf); err != nil {
			return err
		}

	case "ES256":
	case "ES384":
	case "ES512":
		buf := []byte{}
		if buf, err = ReadKey(signKeyFile); err != nil {
			return err
		}

		if key.SignKey, err = jwt.ParseECPrivateKeyFromPEM(buf); err != nil {
			return err
		}

		if buf, err = ReadKey(verifyKeyFile); err != nil {
			return err
		}

		if key.VerifyKey, err = jwt.ParseECPublicKeyFromPEM(buf); err != nil {
			return err
		}
	default:
		return errors.New(fmt.Sprintf("Incorrect alg: %s. Available algs: HS246,HS384,HS512,RS256,RS384,RS512,ES256,ES384,ES512", alg))
	}

	setKey(kid, key)
	return nil
}

// SetHS256Key() reads the HMAC SHA-256 key from key file and stores it with unique kid(key id).
//
// Key generation: github.com/northbright/keygen to generate HMAC SHA-XX key file.
func SetHS256Key(kid, keyFile string) (err error) {
	return SetKeyFromFile(kid, "HS256", keyFile, keyFile)
}

// SetHS384Key() reads the HMAC SHA-384 key from key file and stores it with unique kid(key id).
//
// Key generation: github.com/northbright/keygen to generate HMAC SHA-XX key file.
func SetHS384Key(kid, keyFile string) (err error) {
	return SetKeyFromFile(kid, "HS384", keyFile, keyFile)
}

// SetHS512Key() reads the HMAC SHA-512 key from key file and stores it with unique kid(key id).
//
// Key generation: github.com/northbright/keygen to generate HMAC SHA-XX key file.
func SetHS512Key(kid, keyFile string) (err error) {
	return SetKeyFromFile(kid, "HS512", keyFile, keyFile)
}

// SetRS256Key() reads the RSA SHA-256 keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t rsa -b 2048
func SetRS256Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "RS256", privKeyFile, pubKeyFile)
}

// SetRS384Key() reads the RSA SHA-384 keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t rsa -b 2048
func SetRS384Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "RS384", privKeyFile, pubKeyFile)
}

// SetRS512Key() reads the RSA SHA-512 keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t rsa -b 2048
func SetRS512Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "RS512", privKeyFile, pubKeyFile)
}

// SetES256Key() reads the ECDSA(using P-256 curve and SHA-256) keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t ecdsa -b 256
func SetES256Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "ES256", privKeyFile, pubKeyFile)
}

// SetES384Key() reads the ECDSA(using P-384 curve and SHA-384) keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t ecdsa -b 384
func SetES384Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "ES384", privKeyFile, pubKeyFile)
}

// SetES512Key() reads the ECDSA(using P-521 curve and SHA-512) keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t ecdsa -b 521(NOT 512)
func SetES512Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "ES512", privKeyFile, pubKeyFile)
}

// CreateTokenString() creates a new JWT token string.
//
//   Params:
//       kid: Key id.
//       claims: map[string]interface{} to fill the jwt.Token[Claims].
//   Return:
//       tokenString: new created JWT token string.
//       err: error.
func CreateTokenString(kid string, claims map[string]interface{}) (tokenString string, err error) {
	var k *Key

	if err = errorhelper.GenEmptyStringError(kid, "kid"); err != nil {
		return "", err
	}

	if k, err = GetKey(kid); err != nil {
		return "", err
	}

	t := jwt.New(k.Method)
	t.Header["kid"] = kid
	t.Claims = claims
	return t.SignedString(k.SignKey)
}

// jwt-go's KeyFunc type:
//
// type Keyfunc func(*Token) (interface{}, error)
func keyFunc(token *jwt.Token) (interface{}, error) {
	if _, ok := token.Header["kid"]; !ok {
		return nil, errorhelper.GenMapKeyNotFoundError("kid")
	}

	kid := ""
	if str, ok := token.Header["kid"].(string); !ok {
		msg := fmt.Sprintf("token.Header[\"kid\"]'s type is %T, but not string.", token.Header["kid"])
		return nil, errors.New(msg)
	} else {
		kid = str
	}

	key, err := GetKey(kid)
	if err != nil {
		return nil, err
	}

	// Check signing method
	if token.Method.Alg() != key.Method.Alg() {
		return nil, errors.New("Signing Method Error.")
	}

	return key.VerifyKey, nil
}

// Parse() parses and validates the input token string.
//
//   Params:
//       tokenString: input JWT token string.
//   Return:
//       kid: Key id.
//       claims: map[string]interface{} to fill the jwt.Token[Claims].
//       valid: token is valid or not.
//       err: error.
func Parse(tokenString string) (kid string, claims map[string]interface{}, valid bool, err error) {
	t, err := jwt.Parse(tokenString, keyFunc)
	if err != nil {
		return "", nil, false, err
	}

	return t.Header["kid"].(string), t.Claims, t.Valid, nil
}

// ParseFromRequest() parses and validates the input token string in an http.Request. It's a wrapper of jwt.ParseFromRequest().
//
//   Params:
//       r: http.Request may contain jwt token.
//   Return:
//       kid: Key id.
//       claims: map[string]interface{} to fill the jwt.Token[Claims].
//       valid: token is valid or not.
//       err: error.
func ParseFromRequest(r *http.Request) (kid string, claims map[string]interface{}, valid bool, err error) {
	t, err := jwt.ParseFromRequest(r, keyFunc)
	if err != nil {
		return "", nil, false, err
	}

	return t.Header["kid"].(string), t.Claims, t.Valid, nil
}
<commit_msg>Fix the issue that map of km's not initialized.<commit_after>package jwthelper

import (
	"errors"
	"fmt"
	"github.com/dgrijalva/jwt-go"
	"github.com/northbright/errorhelper"
	"github.com/northbright/pathhelper"
	"io/ioutil"
	"net/http"
	"sync"
)

// Key struct consists of algorithm, signning key and verifying key.
type Key struct {
	Method    jwt.SigningMethod // jwt.SigningMethod
	SignKey   interface{}       // Signing key. HMAC: []byte, RSA: *crypto/rsa.PrivateKey, ECDSA: *crypto/ecdsa.PrivateKey.
	VerifyKey interface{}       // Verifying key. HMAC: []byte, RSA: *crypto/rsa.PublicKey, ECDSA: *crypto/ecdsa.PublicKey.
}

//  KeyManger manages the keys by using kid(key id).
type KeyManager struct {
	Keys         map[string]*Key // Key map. Key: kid(key id), Value: Key Struct
	sync.RWMutex                 // Access map concurrently.
}

var (
	km KeyManager = KeyManager{Keys: make(map[string]*Key)} // Internal key manager.
)

// ReadKey() reads key bytes from the key file.
func ReadKey(keyFile string) (key []byte, err error) {
	if err := errorhelper.GenEmptyStringError(keyFile, "keyFile"); err != nil {
		return nil, err
	}

	// Make Abs key file path with current executable path if KeyFilePath is relative.
	p := ""
	if p, err = pathhelper.GetAbsPath(keyFile); err != nil {
		return nil, err
	}

	buf := []byte{}
	if buf, err = ioutil.ReadFile(p); err != nil {
		return nil, err
	}

	return buf, nil
}

// setKey() sets the kid - Key pair.
//
//   Params:
//       kid: Key id. It should be unique.
//       key: Key struct.
func setKey(kid string, key *Key) {
	km.Lock()
	km.Keys[kid] = key
	km.Unlock()
}

// GetKey() return the key struct by given kid.
func GetKey(kid string) (k *Key, err error) {
	if err := errorhelper.GenEmptyStringError(kid, "kid"); err != nil {
		return nil, err
	}

	km.RLock()
	k, ok := km.Keys[kid]
	km.RUnlock()

	if !ok {
		return nil, errorhelper.GenMapKeyNotFoundError(kid)
	}

	return k, nil
}

// SetKeyFromFile() reads the keys from the key files then stores the unique kid - Key struct pair.
//
//   Params:
//       kid: Key id(unique).
//       alg: JWT alg.
//       signKeyFile: Signing key file.
//       verifyKeyFile: Verifying key file.
//   Return:
//       err: error.
func SetKeyFromFile(kid, alg, signKeyFile, verifyKeyFile string) (err error) {
	key := &Key{}

	if err := errorhelper.GenEmptyStringError(kid, "kid"); err != nil {
		return err
	}

	if err := errorhelper.GenEmptyStringError(alg, "alg"); err != nil {
		return err
	}

	m := jwt.GetSigningMethod(alg)
	if m == nil {
		return errors.New(fmt.Sprintf("Incorrect alg: %s. Available algs: HS246,HS384,HS512,RS256,RS384,RS512,ES256,ES384,ES512", alg))
	}

	// Set Signing Method
	key.Method = m

	switch alg {
	case "HS256":
	case "HS384":
	case "HS512":
		if key.SignKey, err = ReadKey(signKeyFile); err != nil {
			return err
		}
		key.VerifyKey = key.SignKey

	case "RS256":
	case "RS384":
	case "RS512":
		buf := []byte{}
		if buf, err = ReadKey(signKeyFile); err != nil {
			return err
		}

		if key.SignKey, err = jwt.ParseRSAPrivateKeyFromPEM(buf); err != nil {
			return err
		}

		if buf, err = ReadKey(verifyKeyFile); err != nil {
			return err
		}

		if key.VerifyKey, err = jwt.ParseRSAPublicKeyFromPEM(buf); err != nil {
			return err
		}

	case "ES256":
	case "ES384":
	case "ES512":
		buf := []byte{}
		if buf, err = ReadKey(signKeyFile); err != nil {
			return err
		}

		if key.SignKey, err = jwt.ParseECPrivateKeyFromPEM(buf); err != nil {
			return err
		}

		if buf, err = ReadKey(verifyKeyFile); err != nil {
			return err
		}

		if key.VerifyKey, err = jwt.ParseECPublicKeyFromPEM(buf); err != nil {
			return err
		}
	default:
		return errors.New(fmt.Sprintf("Incorrect alg: %s. Available algs: HS246,HS384,HS512,RS256,RS384,RS512,ES256,ES384,ES512", alg))
	}

	setKey(kid, key)
	return nil
}

// SetHS256Key() reads the HMAC SHA-256 key from key file and stores it with unique kid(key id).
//
// Key generation: github.com/northbright/keygen to generate HMAC SHA-XX key file.
func SetHS256Key(kid, keyFile string) (err error) {
	return SetKeyFromFile(kid, "HS256", keyFile, keyFile)
}

// SetHS384Key() reads the HMAC SHA-384 key from key file and stores it with unique kid(key id).
//
// Key generation: github.com/northbright/keygen to generate HMAC SHA-XX key file.
func SetHS384Key(kid, keyFile string) (err error) {
	return SetKeyFromFile(kid, "HS384", keyFile, keyFile)
}

// SetHS512Key() reads the HMAC SHA-512 key from key file and stores it with unique kid(key id).
//
// Key generation: github.com/northbright/keygen to generate HMAC SHA-XX key file.
func SetHS512Key(kid, keyFile string) (err error) {
	return SetKeyFromFile(kid, "HS512", keyFile, keyFile)
}

// SetRS256Key() reads the RSA SHA-256 keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t rsa -b 2048
func SetRS256Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "RS256", privKeyFile, pubKeyFile)
}

// SetRS384Key() reads the RSA SHA-384 keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t rsa -b 2048
func SetRS384Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "RS384", privKeyFile, pubKeyFile)
}

// SetRS512Key() reads the RSA SHA-512 keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t rsa -b 2048
func SetRS512Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "RS512", privKeyFile, pubKeyFile)
}

// SetES256Key() reads the ECDSA(using P-256 curve and SHA-256) keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t ecdsa -b 256
func SetES256Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "ES256", privKeyFile, pubKeyFile)
}

// SetES384Key() reads the ECDSA(using P-384 curve and SHA-384) keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t ecdsa -b 384
func SetES384Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "ES384", privKeyFile, pubKeyFile)
}

// SetES512Key() reads the ECDSA(using P-521 curve and SHA-512) keys from key files and stores it with unique kid(key id).
//
// Key generation: ssh-keygen -t ecdsa -b 521(NOT 512)
func SetES512Key(kid, privKeyFile, pubKeyFile string) (err error) {
	return SetKeyFromFile(kid, "ES512", privKeyFile, pubKeyFile)
}

// CreateTokenString() creates a new JWT token string.
//
//   Params:
//       kid: Key id.
//       claims: map[string]interface{} to fill the jwt.Token[Claims].
//   Return:
//       tokenString: new created JWT token string.
//       err: error.
func CreateTokenString(kid string, claims map[string]interface{}) (tokenString string, err error) {
	var k *Key

	if err = errorhelper.GenEmptyStringError(kid, "kid"); err != nil {
		return "", err
	}

	if k, err = GetKey(kid); err != nil {
		return "", err
	}

	t := jwt.New(k.Method)
	t.Header["kid"] = kid
	t.Claims = claims
	return t.SignedString(k.SignKey)
}

// jwt-go's KeyFunc type:
//
// type Keyfunc func(*Token) (interface{}, error)
func keyFunc(token *jwt.Token) (interface{}, error) {
	if _, ok := token.Header["kid"]; !ok {
		return nil, errorhelper.GenMapKeyNotFoundError("kid")
	}

	kid := ""
	if str, ok := token.Header["kid"].(string); !ok {
		msg := fmt.Sprintf("token.Header[\"kid\"]'s type is %T, but not string.", token.Header["kid"])
		return nil, errors.New(msg)
	} else {
		kid = str
	}

	key, err := GetKey(kid)
	if err != nil {
		return nil, err
	}

	// Check signing method
	if token.Method.Alg() != key.Method.Alg() {
		return nil, errors.New("Signing Method Error.")
	}

	return key.VerifyKey, nil
}

// Parse() parses and validates the input token string.
//
//   Params:
//       tokenString: input JWT token string.
//   Return:
//       kid: Key id.
//       claims: map[string]interface{} to fill the jwt.Token[Claims].
//       valid: token is valid or not.
//       err: error.
func Parse(tokenString string) (kid string, claims map[string]interface{}, valid bool, err error) {
	t, err := jwt.Parse(tokenString, keyFunc)
	if err != nil {
		return "", nil, false, err
	}

	return t.Header["kid"].(string), t.Claims, t.Valid, nil
}

// ParseFromRequest() parses and validates the input token string in an http.Request. It's a wrapper of jwt.ParseFromRequest().
//
//   Params:
//       r: http.Request may contain jwt token.
//   Return:
//       kid: Key id.
//       claims: map[string]interface{} to fill the jwt.Token[Claims].
//       valid: token is valid or not.
//       err: error.
func ParseFromRequest(r *http.Request) (kid string, claims map[string]interface{}, valid bool, err error) {
	t, err := jwt.ParseFromRequest(r, keyFunc)
	if err != nil {
		return "", nil, false, err
	}

	return t.Header["kid"].(string), t.Claims, t.Valid, nil
}
<|endoftext|>


<commit_before>package seccomp

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/subgraph/oz"
	"golang.org/x/sys/unix"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"syscall"
	"unsafe"
)

const (
	STRINGARG = iota + 1
	PTRARG
	INTARG
)

const MAXARGS int = 6
const MAXSTRING int = 100

type SystemCallArgs [MAXARGS]int

type SystemCall struct {
	prefix string
	name   string
	num    int
	args   SystemCallArgs
}

func Tracer() {

	p := new(oz.Profile)
	if err := json.NewDecoder(os.Stdin).Decode(&p); err != nil {
		log.Error("unable to decode profile data: %v", err)
		os.Exit(1)
	}

	log := createLogger()

	var proc_attr syscall.ProcAttr
	var sys_attr syscall.SysProcAttr

	sys_attr.Ptrace = true
	done := false
	proc_attr.Sys = &sys_attr

	cmd := os.Args[1]
	cmdArgs := os.Args[2:]
	log.Info("Tracer running command (%v) arguments (%v)\n", cmd, cmdArgs)
	c := exec.Command(cmd)
	c.SysProcAttr = &syscall.SysProcAttr{Ptrace: true}
	c.Env = os.Environ()
	c.Args = append(c.Args, cmdArgs...)

	pi, err := c.StdinPipe()
	if err != nil {
		fmt.Errorf("error creating stdin pipe for tracer process: %v", err)
		os.Exit(1)
	}
	jdata, err := json.Marshal(p)
	if err != nil {
		fmt.Errorf("Unable to marshal seccomp state: %+v", err)
		os.Exit(1)
	}
	io.Copy(pi, bytes.NewBuffer(jdata))
	log.Info(string(jdata))
	pi.Close()

	children := make(map[int]bool)

	if err := c.Start(); err == nil {
		children[c.Process.Pid] = true
		var s syscall.WaitStatus
		pid, err := syscall.Wait4(-1, &s, syscall.WALL, nil)
		children[pid] = true
		if err != nil {
			log.Error("Error (wait4): %v", err)
		}
		log.Info("Tracing child pid: %v\n", pid)
		for done == false {
			syscall.PtraceSetOptions(pid, unix.PTRACE_O_TRACESECCOMP|unix.PTRACE_O_TRACEFORK|unix.PTRACE_O_TRACEVFORK|unix.PTRACE_O_TRACECLONE|unix.PTRACE_O_TRACEEXIT)
			syscall.PtraceCont(pid, 0)
			pid, err = syscall.Wait4(-1, &s, syscall.WALL, nil)
			if err != nil {
				log.Error("Error (wait4): %v\n", err)
				if len(children) == 0 {
					done = true
				}
				continue
			}
			children[pid] = true
			if s.Exited() == true {
				delete(children, pid)
				log.Info("Child pid %v finished.\n", pid)
				if len(children) == 0 {
					done = true
				}
				continue
			}
			if uint32(s)>>8 == (uint32(unix.SIGTRAP) | (unix.PTRACE_EVENT_SECCOMP << 8)) {
				if err != nil {
					log.Error("Error (ptrace): %v", err)
					continue
				}
				var regs syscall.PtraceRegs
				err = syscall.PtraceGetRegs(pid, &regs)
				if err != nil {
					log.Error("Error (ptrace): %v", err)
				}
				systemcall, err := syscallByNum(int(regs.Orig_rax))
				if err != nil {
					log.Error("Error: %v", err)
					continue
				}
				var callrep string = fmt.Sprintf("%s(", systemcall.name)
				var reg uint64 = 0

				for arg := range systemcall.args {

					if systemcall.args[arg] == 0 {
						break
					}

					if arg > 0 {
						callrep += fmt.Sprintf(",")
					}

					switch arg {
					case 0:
						reg = regs.Rdi
					case 1:
						reg = regs.Rsi
					case 2:
						reg = regs.Rdx
					case 3:
						reg = regs.Rcx
					case 4:
						reg = regs.R8
					case 5:
						reg = regs.R9
					}
					if systemcall.args[arg] == STRINGARG {
						str, err := readStringArg(pid, uintptr(reg))
						if err != nil {
							log.Error("Error: %v", err)
						} else {
							callrep += fmt.Sprintf("\"%s\"", str)
						}
					} else if systemcall.args[arg] == INTARG {
						callrep += fmt.Sprintf("%d", uint64(reg))
					} else {
						/* Stringify pointers in writes to stdout/stderr */
						write, err := syscallByName("write")
						if err != nil {
							log.Error("Error: %v", err)
						}
						if systemcall.num == write.num && (regs.Rdi == uint64(syscall.Stdout) || regs.Rdi == uint64(syscall.Stderr)) {
							str, err := readStringArg(pid, uintptr(reg))
							if err != nil {
								log.Error("Error %v", err)
							} else {
								if isPrintableASCII(str) == true {
									callrep += fmt.Sprintf("\"%s\"", str)
								} else {
									callrep += fmt.Sprintf("0x%X", uintptr(reg))
								}
							}
						} else {
							callrep += fmt.Sprintf("0x%X", uintptr(reg))
						}
					}

				}
				callrep += ")"
				log.Info("==============================================\nseccomp hit on sandbox pid %v (%v) syscall %v (%v): \n\n%s\nI ==============================================\n\n", pid, getProcessCmdLine(pid), systemcall.name, regs.Orig_rax, callrep)
			}
		}
	} else {
		log.Error("Error: %v", err)
	}

}

func readStringArg(pid int, addr uintptr) (s string, err error) {
	buf := make([]byte, unsafe.Sizeof(addr))
	done := false
	err = nil
	for done == false {
		_, err := syscall.PtracePeekText(pid, addr, buf)
		if err != nil {
			fmt.Printf("Error (ptrace): %v\n", err)
		} else {
			for b := range buf {
				if buf[b] == 0 {
					done = true
					break
				} else {
					s += string(buf[b])
					if len(s) > 90 {
						s += "..."
						break
					}
				}
			}
		}
		addr += unsafe.Sizeof(addr)
	}
	return s, nil
}

func readUIntArg(pid int, addr uintptr) (uint64, error) {
	buf := make([]byte, unsafe.Sizeof(addr))

	_, err := syscall.PtracePeekText(pid, addr, buf)
	if err != nil {
		fmt.Printf("Error (ptrace): %v\n", err)
		return 0, err
	} else {
		i := binary.LittleEndian.Uint64(buf)
		return i, nil
	}
	return 0, errors.New("Error.")
}

func readPtrArg(pid int, addr uintptr) (uintptr, error) {

	buf := make([]byte, unsafe.Sizeof(addr))

	_, err := syscall.PtracePeekText(pid, addr, buf)
	if err != nil {
		fmt.Printf("Error (ptrace): %v\n", err)
		return 0, err
	} else {
		i := binary.LittleEndian.Uint64(buf)
		return uintptr(i), nil
	}
	return 0, nil
}

func syscallByNum(num int) (s SystemCall, err error) {
	var q SystemCall = SystemCall{"", "", -1, [6]int{0, 0, 0, 0, 0, 0}}
	for i := range syscalls {
		if syscalls[i].num == num {
			q = syscalls[i]
			return q, nil
		}
	}
	return q, errors.New("System call not found.\n")
}

func syscallByName(name string) (s SystemCall, err error) {
	var q SystemCall = SystemCall{"", "", -1, [6]int{0, 0, 0, 0, 0, 0}}
	for i := range syscalls {
		if syscalls[i].name == name {
			q = syscalls[i]
			return q, nil
		}
	}
	return q, errors.New("System call not found.\n")
}

func isPrintableASCII(s string) bool {
	for _, x := range s {
		if x > 127 {
			return false
		}
	}
	return true
}

func getProcessCmdLine(pid int) string {
	path := "/proc/" + strconv.Itoa(pid) + "/cmdline"
	cmdline, err := ioutil.ReadFile(path)

	if err != nil {
		log.Error("Error (read): %v", err)
		return "unknown"
	}
	return string(cmdline)
}
<commit_msg>s/\0/ /g on /proc/pid/cmdline<commit_after>package seccomp

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/subgraph/oz"
	"golang.org/x/sys/unix"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"syscall"
	"unsafe"
)

const (
	STRINGARG = iota + 1
	PTRARG
	INTARG
)

const MAXARGS int = 6
const MAXSTRING int = 100

type SystemCallArgs [MAXARGS]int

type SystemCall struct {
	prefix string
	name   string
	num    int
	args   SystemCallArgs
}

func Tracer() {

	p := new(oz.Profile)
	if err := json.NewDecoder(os.Stdin).Decode(&p); err != nil {
		log.Error("unable to decode profile data: %v", err)
		os.Exit(1)
	}

	log := createLogger()

	var proc_attr syscall.ProcAttr
	var sys_attr syscall.SysProcAttr

	sys_attr.Ptrace = true
	done := false
	proc_attr.Sys = &sys_attr

	cmd := os.Args[1]
	cmdArgs := os.Args[2:]
	log.Info("Tracer running command (%v) arguments (%v)\n", cmd, cmdArgs)
	c := exec.Command(cmd)
	c.SysProcAttr = &syscall.SysProcAttr{Ptrace: true}
	c.Env = os.Environ()
	c.Args = append(c.Args, cmdArgs...)

	pi, err := c.StdinPipe()
	if err != nil {
		fmt.Errorf("error creating stdin pipe for tracer process: %v", err)
		os.Exit(1)
	}
	jdata, err := json.Marshal(p)
	if err != nil {
		fmt.Errorf("Unable to marshal seccomp state: %+v", err)
		os.Exit(1)
	}
	io.Copy(pi, bytes.NewBuffer(jdata))
	log.Info(string(jdata))
	pi.Close()

	children := make(map[int]bool)

	if err := c.Start(); err == nil {
		children[c.Process.Pid] = true
		var s syscall.WaitStatus
		pid, err := syscall.Wait4(-1, &s, syscall.WALL, nil)
		children[pid] = true
		if err != nil {
			log.Error("Error (wait4): %v", err)
		}
		log.Info("Tracing child pid: %v\n", pid)
		for done == false {
			syscall.PtraceSetOptions(pid, unix.PTRACE_O_TRACESECCOMP|unix.PTRACE_O_TRACEFORK|unix.PTRACE_O_TRACEVFORK|unix.PTRACE_O_TRACECLONE|unix.PTRACE_O_TRACEEXIT)
			syscall.PtraceCont(pid, 0)
			pid, err = syscall.Wait4(-1, &s, syscall.WALL, nil)
			if err != nil {
				log.Error("Error (wait4): %v\n", err)
				if len(children) == 0 {
					done = true
				}
				continue
			}
			children[pid] = true
			if s.Exited() == true {
				delete(children, pid)
				log.Info("Child pid %v finished.\n", pid)
				if len(children) == 0 {
					done = true
				}
				continue
			}
			if uint32(s)>>8 == (uint32(unix.SIGTRAP) | (unix.PTRACE_EVENT_SECCOMP << 8)) {
				if err != nil {
					log.Error("Error (ptrace): %v", err)
					continue
				}
				var regs syscall.PtraceRegs
				err = syscall.PtraceGetRegs(pid, &regs)
				if err != nil {
					log.Error("Error (ptrace): %v", err)
				}
				systemcall, err := syscallByNum(int(regs.Orig_rax))
				if err != nil {
					log.Error("Error: %v", err)
					continue
				}
				var callrep string = fmt.Sprintf("%s(", systemcall.name)
				var reg uint64 = 0

				for arg := range systemcall.args {

					if systemcall.args[arg] == 0 {
						break
					}

					if arg > 0 {
						callrep += fmt.Sprintf(",")
					}

					switch arg {
					case 0:
						reg = regs.Rdi
					case 1:
						reg = regs.Rsi
					case 2:
						reg = regs.Rdx
					case 3:
						reg = regs.Rcx
					case 4:
						reg = regs.R8
					case 5:
						reg = regs.R9
					}
					if systemcall.args[arg] == STRINGARG {
						str, err := readStringArg(pid, uintptr(reg))
						if err != nil {
							log.Error("Error: %v", err)
						} else {
							callrep += fmt.Sprintf("\"%s\"", str)
						}
					} else if systemcall.args[arg] == INTARG {
						callrep += fmt.Sprintf("%d", uint64(reg))
					} else {
						/* Stringify pointers in writes to stdout/stderr */
						write, err := syscallByName("write")
						if err != nil {
							log.Error("Error: %v", err)
						}
						if systemcall.num == write.num && (regs.Rdi == uint64(syscall.Stdout) || regs.Rdi == uint64(syscall.Stderr)) {
							str, err := readStringArg(pid, uintptr(reg))
							if err != nil {
								log.Error("Error %v", err)
							} else {
								if isPrintableASCII(str) == true {
									callrep += fmt.Sprintf("\"%s\"", str)
								} else {
									callrep += fmt.Sprintf("0x%X", uintptr(reg))
								}
							}
						} else {
							callrep += fmt.Sprintf("0x%X", uintptr(reg))
						}
					}

				}
				callrep += ")"
				log.Info("==============================================\nseccomp hit on sandbox pid %v (%v) syscall %v (%v): \n\n%s\nI ==============================================\n\n", pid, getProcessCmdLine(pid), systemcall.name, regs.Orig_rax, callrep)
			}
		}
	} else {
		log.Error("Error: %v", err)
	}

}

func readStringArg(pid int, addr uintptr) (s string, err error) {
	buf := make([]byte, unsafe.Sizeof(addr))
	done := false
	err = nil
	for done == false {
		_, err := syscall.PtracePeekText(pid, addr, buf)
		if err != nil {
			fmt.Printf("Error (ptrace): %v\n", err)
		} else {
			for b := range buf {
				if buf[b] == 0 {
					done = true
					break
				} else {
					s += string(buf[b])
					if len(s) > 90 {
						s += "..."
						break
					}
				}
			}
		}
		addr += unsafe.Sizeof(addr)
	}
	return s, nil
}

func readUIntArg(pid int, addr uintptr) (uint64, error) {
	buf := make([]byte, unsafe.Sizeof(addr))

	_, err := syscall.PtracePeekText(pid, addr, buf)
	if err != nil {
		fmt.Printf("Error (ptrace): %v\n", err)
		return 0, err
	} else {
		i := binary.LittleEndian.Uint64(buf)
		return i, nil
	}
	return 0, errors.New("Error.")
}

func readPtrArg(pid int, addr uintptr) (uintptr, error) {

	buf := make([]byte, unsafe.Sizeof(addr))

	_, err := syscall.PtracePeekText(pid, addr, buf)
	if err != nil {
		fmt.Printf("Error (ptrace): %v\n", err)
		return 0, err
	} else {
		i := binary.LittleEndian.Uint64(buf)
		return uintptr(i), nil
	}
	return 0, nil
}

func syscallByNum(num int) (s SystemCall, err error) {
	var q SystemCall = SystemCall{"", "", -1, [6]int{0, 0, 0, 0, 0, 0}}
	for i := range syscalls {
		if syscalls[i].num == num {
			q = syscalls[i]
			return q, nil
		}
	}
	return q, errors.New("System call not found.\n")
}

func syscallByName(name string) (s SystemCall, err error) {
	var q SystemCall = SystemCall{"", "", -1, [6]int{0, 0, 0, 0, 0, 0}}
	for i := range syscalls {
		if syscalls[i].name == name {
			q = syscalls[i]
			return q, nil
		}
	}
	return q, errors.New("System call not found.\n")
}

func isPrintableASCII(s string) bool {
	for _, x := range s {
		if x > 127 {
			return false
		}
	}
	return true
}

func getProcessCmdLine(pid int) string {
	path := "/proc/" + strconv.Itoa(pid) + "/cmdline"
	cmdline, err := ioutil.ReadFile(path)
	for b := range cmdline {
		if b < (len(cmdline) - 1) {
			if cmdline[b] == 0x00 {
				cmdline[b] = 0x20
			}
		}
	}
	if err != nil {
		log.Error("Error (read): %v", err)
		return "unknown"
	}
	return string(cmdline)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/jacobsa/aws/exp/sdb"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"sync"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type integrationTest struct {
	db sdb.SimpleDB
}

func (t *integrationTest) SetUp(i *TestInfo) {
	var err error

	// Open a connection.
	t.db, err = sdb.NewSimpleDB(g_region, g_accessKey)
	AssertEq(nil, err)
}

////////////////////////////////////////////////////////////////////////
// Domains
////////////////////////////////////////////////////////////////////////

type DomainsTest struct {
	integrationTest

	mutex           sync.Mutex
	domainsToDelete []sdb.Domain  // Protected by mutex
}

func init() { RegisterTestSuite(&DomainsTest{}) }

func (t *DomainsTest) ensureDeleted(d sdb.Domain) {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	t.domainsToDelete = append(t.domainsToDelete, d)
}

func (t *DomainsTest) TearDown() {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	// Delete each of the domains created during the test.
	for _, d := range t.domainsToDelete {
		ExpectEq(nil, t.db.DeleteDomain(d), "Domain: %s", d.Name())
	}
}

func (t *DomainsTest) InvalidAccessKey() {
	// Open a connection with an unknown key ID.
	wrongKey := g_accessKey
	wrongKey.Id += "taco"

	db, err := sdb.NewSimpleDB(g_region, wrongKey)
	AssertEq(nil, err)

	// Attempt to create a domain.
	_, err = db.OpenDomain("some_domain")

	ExpectThat(err, Error(HasSubstr("403")))
	ExpectThat(err, Error(HasSubstr("Key Id")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

func (t *DomainsTest) SeparatelyNamedDomainsHaveIndependentItems() {
	var err error

	// Open two domains.
	domain0, err := t.db.OpenDomain("taco")
	AssertEq(nil, err)
	t.ensureDeleted(domain0)

	domain1, err := t.db.OpenDomain("burrito")
	AssertEq(nil, err)
	t.ensureDeleted(domain1)

	// Set up an item in the first.
	itemName := sdb.ItemName("some_item")
	err = domain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in the other domain. There should be
	// none.
	attrs, err := domain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(attrs, ElementsAre())
}

func (t *DomainsTest) IdenticallyNamedDomainsHaveIdenticalItems() {
	var err error

	// Open two domains with the same name.
	domain0, err := t.db.OpenDomain("taco")
	AssertEq(nil, err)
	t.ensureDeleted(domain0)

	domain1, err := t.db.OpenDomain(domain0.Name())
	AssertEq(nil, err)
	t.ensureDeleted(domain1)

	// Set up an item in the first.
	itemName := sdb.ItemName("some_item")
	err = domain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in the other domain.
	attrs, err := domain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(
		attrs,
		ElementsAre(
			DeepEquals(
				[]sdb.Attribute{
					sdb.Attribute{Name: "enchilada", Value: "queso"},
				},
			),
		),
	)
}

func (t *DomainsTest) Delete() {
	ExpectEq("TODO", "")
}

func (t *DomainsTest) OpeningTwiceDoesntDeleteExistingItems() {
	ExpectEq("TODO", "")
}

func (t *DomainsTest) DeleteTwice() {
	ExpectEq("TODO", "")
}

////////////////////////////////////////////////////////////////////////
// Items
////////////////////////////////////////////////////////////////////////

type ItemsTest struct {
	integrationTest
}

func init() { RegisterTestSuite(&ItemsTest{}) }

func (t *ItemsTest) WrongAccessKeySecret() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8ItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8AttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8AttributeValue() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongAttributeValue() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) PutThenGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchPutThenGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchPutThenBatchGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetForNonExistentItem() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetForNonExistentItems() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedValuePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedNonExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SuccessfulPreconditions() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteAllAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchDelete() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidSelectQuery() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectAll() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectCount() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithPredicates() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithSortOrder() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithLimit() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectEmptyResultSet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectLargeResultSet() {
	ExpectEq("TODO", "")
}
<commit_msg>Fixed a test bug.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/jacobsa/aws/exp/sdb"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"sync"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type integrationTest struct {
	db sdb.SimpleDB
}

func (t *integrationTest) SetUp(i *TestInfo) {
	var err error

	// Open a connection.
	t.db, err = sdb.NewSimpleDB(g_region, g_accessKey)
	AssertEq(nil, err)
}

////////////////////////////////////////////////////////////////////////
// Domains
////////////////////////////////////////////////////////////////////////

type DomainsTest struct {
	integrationTest

	mutex           sync.Mutex
	domainsToDelete []sdb.Domain  // Protected by mutex
}

func init() { RegisterTestSuite(&DomainsTest{}) }

func (t *DomainsTest) ensureDeleted(d sdb.Domain) {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	t.domainsToDelete = append(t.domainsToDelete, d)
}

func (t *DomainsTest) TearDown() {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	// Delete each of the domains created during the test.
	for _, d := range t.domainsToDelete {
		ExpectEq(nil, t.db.DeleteDomain(d), "Domain: %s", d.Name())
	}
}

func (t *DomainsTest) InvalidAccessKey() {
	// Open a connection with an unknown key ID.
	wrongKey := g_accessKey
	wrongKey.Id += "taco"

	db, err := sdb.NewSimpleDB(g_region, wrongKey)
	AssertEq(nil, err)

	// Attempt to create a domain.
	_, err = db.OpenDomain("some_domain")

	ExpectThat(err, Error(HasSubstr("403")))
	ExpectThat(err, Error(HasSubstr("Key Id")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

func (t *DomainsTest) SeparatelyNamedDomainsHaveIndependentItems() {
	var err error

	// Open two domains.
	domain0, err := t.db.OpenDomain("taco")
	AssertEq(nil, err)
	t.ensureDeleted(domain0)

	domain1, err := t.db.OpenDomain("burrito")
	AssertEq(nil, err)
	t.ensureDeleted(domain1)

	// Set up an item in the first.
	itemName := sdb.ItemName("some_item")
	err = domain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in the other domain. There should be
	// none.
	attrs, err := domain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(attrs, ElementsAre())
}

func (t *DomainsTest) IdenticallyNamedDomainsHaveIdenticalItems() {
	var err error

	// Open two domains with the same name.
	domain0, err := t.db.OpenDomain("taco")
	AssertEq(nil, err)
	t.ensureDeleted(domain0)

	domain1, err := t.db.OpenDomain(domain0.Name())
	AssertEq(nil, err)
	t.ensureDeleted(domain1)

	// Set up an item in the first.
	itemName := sdb.ItemName("some_item")
	err = domain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in the other domain.
	attrs, err := domain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(
		attrs,
		ElementsAre(
			DeepEquals(sdb.Attribute{Name: "enchilada", Value: "queso"}),
		),
	)
}

func (t *DomainsTest) Delete() {
	ExpectEq("TODO", "")
}

func (t *DomainsTest) OpeningTwiceDoesntDeleteExistingItems() {
	ExpectEq("TODO", "")
}

func (t *DomainsTest) DeleteTwice() {
	ExpectEq("TODO", "")
}

////////////////////////////////////////////////////////////////////////
// Items
////////////////////////////////////////////////////////////////////////

type ItemsTest struct {
	integrationTest
}

func init() { RegisterTestSuite(&ItemsTest{}) }

func (t *ItemsTest) WrongAccessKeySecret() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8ItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8AttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8AttributeValue() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongAttributeValue() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) PutThenGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchPutThenGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchPutThenBatchGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetForNonExistentItem() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetForNonExistentItems() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedValuePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedNonExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SuccessfulPreconditions() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteAllAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchDelete() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidSelectQuery() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectAll() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectCount() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithPredicates() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithSortOrder() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithLimit() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectEmptyResultSet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectLargeResultSet() {
	ExpectEq("TODO", "")
}
<|endoftext|>


<commit_before>package forwarder

import (
	"bytes"
	"code.google.com/p/goprotobuf/proto"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/cep21/gohelpers/structdefaults"
	"github.com/cep21/gohelpers/workarounds"
	"github.com/golang/glog"
	"github.com/signalfuse/com_signalfuse_metrics_protobuf"
	"github.com/signalfuse/signalfxproxy/config"
	"github.com/signalfuse/signalfxproxy/core"
	"github.com/signalfuse/signalfxproxy/core/value"
	"github.com/signalfuse/signalfxproxy/protocoltypes"
	"io/ioutil"
	"net/http"
	"runtime"
	"sync"
	"time"
)

type metricCreationRequest struct {
	toCreate        map[string]com_signalfuse_metrics_protobuf.MetricType
	responseChannel chan (error)
}

type signalfxJSONConnector struct {
	*basicBufferedForwarder
	url                string
	metricCreationChan chan *metricCreationRequest
	connectionTimeout  time.Duration
	defaultAuthToken   string
	tr                 *http.Transport
	sendVersion        int
	client             *http.Client
	userAgent          string
	defaultSource      string
	// Map of all metric names to if we've created them with their metric type
	v1MetricLoadedCache      map[string]struct{}
	v1MetricLoadedCacheMutex sync.Mutex
	MetricCreationURL        string
}

// ValueToSend are values are sent from the proxy to a reciever for the datapoint
type ValueToSend interface {
}

// BodySendFormat is the JSON format signalfx datapoints are expected to be in
type BodySendFormat struct {
	Metric     string            `json:"metric"`
	Timestamp  int64             `json:"timestamp"`
	Value      ValueToSend       `json:"value"`
	Dimensions map[string]string `json:"dimensions"`
}

func (bodySendFormat *BodySendFormat) String() string {
	return fmt.Sprintf("DP[metric=%s|time=%d|val=%s|dimensions=%s]", bodySendFormat.Metric, bodySendFormat.Timestamp, bodySendFormat.Value, bodySendFormat.Dimensions)
}

var defaultConfig = &config.ForwardTo{
	URL:               workarounds.GolangDoesnotAllowPointerToStringLiteral("https://api.signalfuse.com/v1/datapoint"),
	DefaultSource:     workarounds.GolangDoesnotAllowPointerToStringLiteral(""),
	MetricCreationURL: workarounds.GolangDoesnotAllowPointerToStringLiteral("https://api.signalfuse.com/v1/metric?bulkupdate=true"),
	TimeoutDuration:   workarounds.GolangDoesnotAllowPointerToTimeLiteral(time.Second * 30),
	BufferSize:        workarounds.GolangDoesnotAllowPointerToUintLiteral(uint32(10000)),
	DrainingThreads:   workarounds.GolangDoesnotAllowPointerToUintLiteral(uint32(5)),
	Name:              workarounds.GolangDoesnotAllowPointerToStringLiteral("signalfx-forwarder"),
	MaxDrainSize:      workarounds.GolangDoesnotAllowPointerToUintLiteral(uint32(100)),
}

// SignalfxJSONForwarderLoader loads a json forwarder forwarding points from proxy to SignalFx
func SignalfxJSONForwarderLoader(forwardTo *config.ForwardTo) (core.StatKeepingStreamingAPI, error) {
	structdefaults.FillDefaultFrom(forwardTo, defaultConfig)
	glog.Infof("Creating signalfx forwarder using final config %s", forwardTo)
	return NewSignalfxJSONForwarer(*forwardTo.URL, *forwardTo.TimeoutDuration, *forwardTo.BufferSize,
		*forwardTo.DefaultAuthToken, *forwardTo.DrainingThreads, *forwardTo.Name, *forwardTo.MetricCreationURL,
		*forwardTo.MaxDrainSize, *forwardTo.DefaultSource)
}

// NewSignalfxJSONForwarer creates a new JSON forwarder
func NewSignalfxJSONForwarer(url string, timeout time.Duration, bufferSize uint32,
	defaultAuthToken string, drainingThreads uint32, name string, MetricCreationURL string,
	maxDrainSize uint32, defaultSource string) (core.StatKeepingStreamingAPI, error) {
	tr := &http.Transport{
		TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
		MaxIdleConnsPerHost: int(drainingThreads) * 2,
	}
	ret := &signalfxJSONConnector{
		basicBufferedForwarder: newBasicBufferedForwarder(bufferSize, maxDrainSize, name, drainingThreads),
		url:              url,
		defaultAuthToken: defaultAuthToken,
		sendVersion:      1,
		userAgent:        fmt.Sprintf("SignalfxProxy/0.1 (gover %s)", runtime.Version()),
		tr:               tr,
		client: &http.Client{
			Transport: tr,
		},
		connectionTimeout:   timeout,
		v1MetricLoadedCache: map[string]struct{}{},
		MetricCreationURL:   MetricCreationURL,
		defaultSource:       defaultSource,
		metricCreationChan:  make(chan *metricCreationRequest),
	}
	ret.start(ret.process)
	go ret.metricCreationLoop()
	return ret, nil
}

func (connector *signalfxJSONConnector) encodePostBodyV2(datapoints []core.Datapoint) ([]byte, string, error) {
	bodyToSend := make(map[string][]*BodySendFormat)
	for _, dp := range datapoints {
		bsf := &BodySendFormat{
			Metric:     dp.Metric(),
			Timestamp:  dp.Timestamp().UnixNano() / time.Millisecond.Nanoseconds(),
			Dimensions: dp.Dimensions(),
		}
		f, err := dp.Value().FloatValue()
		if err == nil {
			bsf.Value = f
		} else {
			bsf.Value = dp.Value().WireValue()
		}
		_, ok := bodyToSend[dp.MetricType().String()]
		if !ok {
			bodyToSend[dp.MetricType().String()] = make([]*BodySendFormat, 0)
		}
		bodyToSend[dp.MetricType().String()] = append(bodyToSend[dp.MetricType().String()], bsf)
	}
	jsonBytes, err := json.Marshal(&bodyToSend)
	glog.V(3).Infof("Posting %s from %s", jsonBytes, bodyToSend)

	// Now we can send datapoints
	return jsonBytes, "application/json", err
}

func datumForPoint(pv value.DatapointValue) *com_signalfuse_metrics_protobuf.Datum {
	i, err := pv.IntValue()
	if err == nil {
		return &com_signalfuse_metrics_protobuf.Datum{IntValue: &i}
	}
	f, err := pv.FloatValue()
	if err == nil {
		return &com_signalfuse_metrics_protobuf.Datum{DoubleValue: &f}
	}
	s := pv.WireValue()
	return &com_signalfuse_metrics_protobuf.Datum{StrValue: &s}
}

func (connector *signalfxJSONConnector) metricCreationLoop() {
	for {
		glog.V(3).Infof("Waiting for creation request")
		request := <-connector.metricCreationChan
		glog.V(3).Infof("Got creation request: %s", request)
		resp := connector.createMetricsOfType(request.toCreate)
		request.responseChannel <- resp
	}
}

func (connector *signalfxJSONConnector) createMetricsOfType(metricsToCreate map[string]com_signalfuse_metrics_protobuf.MetricType) error {
	if len(metricsToCreate) == 0 {
		return nil
	}
	postBody := []protocoltypes.SignalfxMetricCreationStruct{}
	for metricName, metricType := range metricsToCreate {
		postBody = append(postBody, protocoltypes.SignalfxMetricCreationStruct{
			MetricName: metricName,
			MetricType: metricType.String(),
		})
	}
	jsonBytes, err := json.Marshal(&postBody)
	if err != nil {
		glog.Warningf("Unable to marshal body: %s", err)
		return err
	}
	glog.V(3).Infof("Posting %s from %s", jsonBytes, postBody)

	req, _ := http.NewRequest("POST", connector.MetricCreationURL, bytes.NewBuffer(jsonBytes))
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-SF-TOKEN", connector.defaultAuthToken)
	req.Header.Set("User-Agent", connector.userAgent)

	req.Header.Set("Connection", "Keep-Alive")
	glog.V(3).Infof("Request is %s", req)
	resp, err := connector.client.Do(req)

	if err != nil {
		glog.Warningf("Unable to POST response: %s", err)
		return err
	}

	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		glog.Warningf("Unable to verify response body: %s", err)
		return err
	}
	if resp.StatusCode != 200 {
		glog.Warningf("Metric creation failed: %s", respBody)
		return fmt.Errorf("invalid status code: %d", resp.StatusCode)
	}
	var metricCreationBody []protocoltypes.SignalfxMetricCreationResponse
	err = json.Unmarshal(respBody, &metricCreationBody)
	if err != nil {
		glog.Warningf("body=(%s), err=(%s)", respBody, err)
		return err
	}
	connector.v1MetricLoadedCacheMutex.Lock()
	defer connector.v1MetricLoadedCacheMutex.Unlock()
	for index, resp := range metricCreationBody {
		metricName := postBody[index].MetricName
		if resp.Code == 0 || resp.Code == 409 {
			connector.v1MetricLoadedCache[metricName] = struct{}{}
		} else {
			glog.Warningf("Unable to create metric %s: %s", metricName, respBody)
		}
	}
	glog.V(3).Infof("Metric creation %s returned %s", jsonBytes, respBody)
	return nil
}

func (connector *signalfxJSONConnector) figureOutReasonableSource(point core.Datapoint) string {
	thisPointSource := point.Dimensions()["sf_source"]
	if thisPointSource != "" {
		return thisPointSource
	}
	return connector.defaultSource
}

func (connector *signalfxJSONConnector) encodePostBodyV1(datapoints []core.Datapoint) ([]byte, string, error) {
	var msgBody []byte
	metricsToBeCreated := make(map[string]com_signalfuse_metrics_protobuf.MetricType)
	for _, point := range datapoints {
		thisPointSource := connector.figureOutReasonableSource(point)
		if thisPointSource == "" {
			glog.Warningf("unable to figure out a reasonable source for %s, skipping", point)
			continue
		}
		if point.MetricType() != com_signalfuse_metrics_protobuf.MetricType_GAUGE {
			preCreated := func() (bool) {
				connector.v1MetricLoadedCacheMutex.Lock()
				defer connector.v1MetricLoadedCacheMutex.Unlock()
				_, ok := connector.v1MetricLoadedCache[point.Metric()]
				return ok
			}()

			if !preCreated {
				metricsToBeCreated[point.Metric()] = point.MetricType()
			}
		}
		m := point.Metric()
		ts := point.Timestamp().UnixNano() / time.Millisecond.Nanoseconds()
		relativeTimeDp, ok := point.(core.TimeRelativeDatapoint)
		if ok {
			ts = relativeTimeDp.RelativeTime()
		}
		v := &com_signalfuse_metrics_protobuf.DataPoint{
			Source:    &thisPointSource,
			Metric:    &m,
			Timestamp: &ts,
			Value:     datumForPoint(point.Value()),
		}
		glog.V(3).Infof("Single datapoint to signalfx: %s", v)
		encodedBytes, err := proto.Marshal(v)
		if err != nil {
			return nil, "", err
		}
		msgBody = append(msgBody, proto.EncodeVarint(uint64(len(encodedBytes)))...)
		msgBody = append(msgBody, encodedBytes...)
	}
	glog.V(3).Infof("Posting %s len %d", msgBody, len(msgBody))

	if len(metricsToBeCreated) > 0 {
		// Create metrics if we need to
		creationRequest := &metricCreationRequest{
			toCreate:        metricsToBeCreated,
			responseChannel: make(chan error),
		}
		glog.V(3).Infof("Trying to create metrics first")
		connector.metricCreationChan <- creationRequest
		err := <-creationRequest.responseChannel
		if err != nil {
			return nil, "", err
		}
		glog.V(3).Infof("Metric creation OK")
	}
	return msgBody, "application/x-protobuf", nil
}

func (connector *signalfxJSONConnector) GetStats() []core.Datapoint {
	return []core.Datapoint{}
}

func (connector *signalfxJSONConnector) encodePostBody(datapoints []core.Datapoint) ([]byte, string, error) {
	switch connector.sendVersion {
	case 2:
		return connector.encodePostBodyV2(datapoints)
	default:
		return connector.encodePostBodyV1(datapoints)
	}
}

func (connector *signalfxJSONConnector) process(datapoints []core.Datapoint) error {
	glog.V(2).Infof("Got %d points: %s\n", len(datapoints), datapoints)
	jsonBytes, bodyType, err := connector.encodePostBody(datapoints)

	if err != nil {
		glog.Warningf("Unable to marshal json: %s", err)
		return err
	}
	req, _ := http.NewRequest("POST", connector.url, bytes.NewBuffer(jsonBytes))
	glog.V(3).Infof("Request: %s from %s", req, string(jsonBytes))
	req.Header.Set("Content-Type", bodyType)
	req.Header.Set("X-SF-TOKEN", connector.defaultAuthToken)
	req.Header.Set("User-Agent", connector.userAgent)

	req.Header.Set("Connection", "Keep-Alive")
	resp, err := connector.client.Do(req)

	if err != nil {
		glog.Warningf("Unable to POST response: %s", err)
		return err
	}

	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		glog.Warningf("Unable to verify response body: %s", err)
		return err
	}
	if resp.StatusCode != 200 {
		glog.Warningf("Metric upload failed: %s", respBody)
		return fmt.Errorf("invalid status code: %d", resp.StatusCode)
	}
	var body string
	err = json.Unmarshal(respBody, &body)
	if err != nil {
		glog.Warningf("body=(%s), err=(%s)", respBody, err)
		return err
	}
	if body != "OK" {
		glog.Warningf("Response not OK: %s", body)
		return err
	}
	return nil
}
<commit_msg>Moved debug logging<commit_after>package forwarder

import (
	"bytes"
	"code.google.com/p/goprotobuf/proto"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/cep21/gohelpers/structdefaults"
	"github.com/cep21/gohelpers/workarounds"
	"github.com/golang/glog"
	"github.com/signalfuse/com_signalfuse_metrics_protobuf"
	"github.com/signalfuse/signalfxproxy/config"
	"github.com/signalfuse/signalfxproxy/core"
	"github.com/signalfuse/signalfxproxy/core/value"
	"github.com/signalfuse/signalfxproxy/protocoltypes"
	"io/ioutil"
	"net/http"
	"runtime"
	"sync"
	"time"
)

type metricCreationRequest struct {
	toCreate        map[string]com_signalfuse_metrics_protobuf.MetricType
	responseChannel chan (error)
}

type signalfxJSONConnector struct {
	*basicBufferedForwarder
	url                string
	metricCreationChan chan *metricCreationRequest
	connectionTimeout  time.Duration
	defaultAuthToken   string
	tr                 *http.Transport
	sendVersion        int
	client             *http.Client
	userAgent          string
	defaultSource      string
	// Map of all metric names to if we've created them with their metric type
	v1MetricLoadedCache      map[string]struct{}
	v1MetricLoadedCacheMutex sync.Mutex
	MetricCreationURL        string
}

// ValueToSend are values are sent from the proxy to a reciever for the datapoint
type ValueToSend interface {
}

// BodySendFormat is the JSON format signalfx datapoints are expected to be in
type BodySendFormat struct {
	Metric     string            `json:"metric"`
	Timestamp  int64             `json:"timestamp"`
	Value      ValueToSend       `json:"value"`
	Dimensions map[string]string `json:"dimensions"`
}

func (bodySendFormat *BodySendFormat) String() string {
	return fmt.Sprintf("DP[metric=%s|time=%d|val=%s|dimensions=%s]", bodySendFormat.Metric, bodySendFormat.Timestamp, bodySendFormat.Value, bodySendFormat.Dimensions)
}

var defaultConfig = &config.ForwardTo{
	URL:               workarounds.GolangDoesnotAllowPointerToStringLiteral("https://api.signalfuse.com/v1/datapoint"),
	DefaultSource:     workarounds.GolangDoesnotAllowPointerToStringLiteral(""),
	MetricCreationURL: workarounds.GolangDoesnotAllowPointerToStringLiteral("https://api.signalfuse.com/v1/metric?bulkupdate=true"),
	TimeoutDuration:   workarounds.GolangDoesnotAllowPointerToTimeLiteral(time.Second * 30),
	BufferSize:        workarounds.GolangDoesnotAllowPointerToUintLiteral(uint32(10000)),
	DrainingThreads:   workarounds.GolangDoesnotAllowPointerToUintLiteral(uint32(5)),
	Name:              workarounds.GolangDoesnotAllowPointerToStringLiteral("signalfx-forwarder"),
	MaxDrainSize:      workarounds.GolangDoesnotAllowPointerToUintLiteral(uint32(100)),
}

// SignalfxJSONForwarderLoader loads a json forwarder forwarding points from proxy to SignalFx
func SignalfxJSONForwarderLoader(forwardTo *config.ForwardTo) (core.StatKeepingStreamingAPI, error) {
	structdefaults.FillDefaultFrom(forwardTo, defaultConfig)
	glog.Infof("Creating signalfx forwarder using final config %s", forwardTo)
	return NewSignalfxJSONForwarer(*forwardTo.URL, *forwardTo.TimeoutDuration, *forwardTo.BufferSize,
		*forwardTo.DefaultAuthToken, *forwardTo.DrainingThreads, *forwardTo.Name, *forwardTo.MetricCreationURL,
		*forwardTo.MaxDrainSize, *forwardTo.DefaultSource)
}

// NewSignalfxJSONForwarer creates a new JSON forwarder
func NewSignalfxJSONForwarer(url string, timeout time.Duration, bufferSize uint32,
	defaultAuthToken string, drainingThreads uint32, name string, MetricCreationURL string,
	maxDrainSize uint32, defaultSource string) (core.StatKeepingStreamingAPI, error) {
	tr := &http.Transport{
		TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
		MaxIdleConnsPerHost: int(drainingThreads) * 2,
	}
	ret := &signalfxJSONConnector{
		basicBufferedForwarder: newBasicBufferedForwarder(bufferSize, maxDrainSize, name, drainingThreads),
		url:              url,
		defaultAuthToken: defaultAuthToken,
		sendVersion:      1,
		userAgent:        fmt.Sprintf("SignalfxProxy/0.1 (gover %s)", runtime.Version()),
		tr:               tr,
		client: &http.Client{
			Transport: tr,
		},
		connectionTimeout:   timeout,
		v1MetricLoadedCache: map[string]struct{}{},
		MetricCreationURL:   MetricCreationURL,
		defaultSource:       defaultSource,
		metricCreationChan:  make(chan *metricCreationRequest),
	}
	ret.start(ret.process)
	go ret.metricCreationLoop()
	return ret, nil
}

func (connector *signalfxJSONConnector) encodePostBodyV2(datapoints []core.Datapoint) ([]byte, string, error) {
	bodyToSend := make(map[string][]*BodySendFormat)
	for _, dp := range datapoints {
		bsf := &BodySendFormat{
			Metric:     dp.Metric(),
			Timestamp:  dp.Timestamp().UnixNano() / time.Millisecond.Nanoseconds(),
			Dimensions: dp.Dimensions(),
		}
		f, err := dp.Value().FloatValue()
		if err == nil {
			bsf.Value = f
		} else {
			bsf.Value = dp.Value().WireValue()
		}
		_, ok := bodyToSend[dp.MetricType().String()]
		if !ok {
			bodyToSend[dp.MetricType().String()] = make([]*BodySendFormat, 0)
		}
		bodyToSend[dp.MetricType().String()] = append(bodyToSend[dp.MetricType().String()], bsf)
	}
	jsonBytes, err := json.Marshal(&bodyToSend)
	glog.V(3).Infof("Posting %s from %s", jsonBytes, bodyToSend)

	// Now we can send datapoints
	return jsonBytes, "application/json", err
}

func datumForPoint(pv value.DatapointValue) *com_signalfuse_metrics_protobuf.Datum {
	i, err := pv.IntValue()
	if err == nil {
		return &com_signalfuse_metrics_protobuf.Datum{IntValue: &i}
	}
	f, err := pv.FloatValue()
	if err == nil {
		return &com_signalfuse_metrics_protobuf.Datum{DoubleValue: &f}
	}
	s := pv.WireValue()
	return &com_signalfuse_metrics_protobuf.Datum{StrValue: &s}
}

func (connector *signalfxJSONConnector) metricCreationLoop() {
	for {
		glog.V(3).Infof("Waiting for creation request")
		request := <-connector.metricCreationChan
		glog.V(3).Infof("Got creation request: %s", request)
		resp := connector.createMetricsOfType(request.toCreate)
		request.responseChannel <- resp
	}
}

func (connector *signalfxJSONConnector) createMetricsOfType(metricsToCreate map[string]com_signalfuse_metrics_protobuf.MetricType) error {
	if len(metricsToCreate) == 0 {
		return nil
	}
	postBody := []protocoltypes.SignalfxMetricCreationStruct{}
	for metricName, metricType := range metricsToCreate {
		postBody = append(postBody, protocoltypes.SignalfxMetricCreationStruct{
			MetricName: metricName,
			MetricType: metricType.String(),
		})
	}
	jsonBytes, err := json.Marshal(&postBody)
	if err != nil {
		glog.Warningf("Unable to marshal body: %s", err)
		return err
	}
	glog.V(3).Infof("Posting %s from %s", jsonBytes, postBody)

	req, _ := http.NewRequest("POST", connector.MetricCreationURL, bytes.NewBuffer(jsonBytes))
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-SF-TOKEN", connector.defaultAuthToken)
	req.Header.Set("User-Agent", connector.userAgent)

	req.Header.Set("Connection", "Keep-Alive")
	glog.V(3).Infof("Request is %s", req)
	resp, err := connector.client.Do(req)

	if err != nil {
		glog.Warningf("Unable to POST response: %s", err)
		return err
	}

	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		glog.Warningf("Unable to verify response body: %s", err)
		return err
	}
	if resp.StatusCode != 200 {
		glog.Warningf("Metric creation failed: %s", respBody)
		return fmt.Errorf("invalid status code: %d", resp.StatusCode)
	}
	var metricCreationBody []protocoltypes.SignalfxMetricCreationResponse
	err = json.Unmarshal(respBody, &metricCreationBody)
	if err != nil {
		glog.Warningf("body=(%s), err=(%s)", respBody, err)
		return err
	}
	connector.v1MetricLoadedCacheMutex.Lock()
	defer connector.v1MetricLoadedCacheMutex.Unlock()
	for index, resp := range metricCreationBody {
		metricName := postBody[index].MetricName
		if resp.Code == 0 || resp.Code == 409 {
			connector.v1MetricLoadedCache[metricName] = struct{}{}
		} else {
			glog.Warningf("Unable to create metric %s: %s", metricName, respBody)
		}
	}
	glog.V(3).Infof("Metric creation %s returned %s", jsonBytes, respBody)
	return nil
}

func (connector *signalfxJSONConnector) figureOutReasonableSource(point core.Datapoint) string {
	thisPointSource := point.Dimensions()["sf_source"]
	if thisPointSource != "" {
		return thisPointSource
	}
	return connector.defaultSource
}

func (connector *signalfxJSONConnector) encodePostBodyV1(datapoints []core.Datapoint) ([]byte, string, error) {
	var msgBody []byte
	metricsToBeCreated := make(map[string]com_signalfuse_metrics_protobuf.MetricType)
	for _, point := range datapoints {
		thisPointSource := connector.figureOutReasonableSource(point)
		if thisPointSource == "" {
			glog.Warningf("unable to figure out a reasonable source for %s, skipping", point)
			continue
		}
		if point.MetricType() != com_signalfuse_metrics_protobuf.MetricType_GAUGE {
			preCreated := func() (bool) {
				connector.v1MetricLoadedCacheMutex.Lock()
				defer connector.v1MetricLoadedCacheMutex.Unlock()
				_, ok := connector.v1MetricLoadedCache[point.Metric()]
				return ok
			}()

			if !preCreated {
				metricsToBeCreated[point.Metric()] = point.MetricType()
			}
		}
		m := point.Metric()
		ts := point.Timestamp().UnixNano() / time.Millisecond.Nanoseconds()
		relativeTimeDp, ok := point.(core.TimeRelativeDatapoint)
		if ok {
			ts = relativeTimeDp.RelativeTime()
		}
		v := &com_signalfuse_metrics_protobuf.DataPoint{
			Source:    &thisPointSource,
			Metric:    &m,
			Timestamp: &ts,
			Value:     datumForPoint(point.Value()),
		}
		glog.V(3).Infof("Single datapoint to signalfx: %s", v)
		encodedBytes, err := proto.Marshal(v)
		if err != nil {
			return nil, "", err
		}
		msgBody = append(msgBody, proto.EncodeVarint(uint64(len(encodedBytes)))...)
		msgBody = append(msgBody, encodedBytes...)
	}
	glog.V(3).Infof("Posting %s len %d", msgBody, len(msgBody))

	if len(metricsToBeCreated) > 0 {
		// Create metrics if we need to
		creationRequest := &metricCreationRequest{
			toCreate:        metricsToBeCreated,
			responseChannel: make(chan error),
		}
		glog.V(3).Infof("Trying to create metrics first")
		connector.metricCreationChan <- creationRequest
		err := <-creationRequest.responseChannel
		if err != nil {
			return nil, "", err
		}
		glog.V(3).Infof("Metric creation OK")
	}
	return msgBody, "application/x-protobuf", nil
}

func (connector *signalfxJSONConnector) GetStats() []core.Datapoint {
	return []core.Datapoint{}
}

func (connector *signalfxJSONConnector) encodePostBody(datapoints []core.Datapoint) ([]byte, string, error) {
	switch connector.sendVersion {
	case 2:
		return connector.encodePostBodyV2(datapoints)
	default:
		return connector.encodePostBodyV1(datapoints)
	}
}

func (connector *signalfxJSONConnector) process(datapoints []core.Datapoint) error {
	glog.V(2).Infof("Got %d points: %s\n", len(datapoints), datapoints)
	jsonBytes, bodyType, err := connector.encodePostBody(datapoints)

	if err != nil {
		glog.Warningf("Unable to marshal json: %s", err)
		return err
	}
	req, _ := http.NewRequest("POST", connector.url, bytes.NewBuffer(jsonBytes))
	req.Header.Set("Content-Type", bodyType)
	req.Header.Set("X-SF-TOKEN", connector.defaultAuthToken)
	req.Header.Set("User-Agent", connector.userAgent)

	req.Header.Set("Connection", "Keep-Alive")
	glog.V(3).Infof("Request: %s from %s", req, string(jsonBytes))
	resp, err := connector.client.Do(req)

	if err != nil {
		glog.Warningf("Unable to POST response: %s", err)
		return err
	}

	glog.V(3).Infof("Response: %s", resp)
	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		glog.Warningf("Unable to verify response body: %s", err)
		return err
	}
	if resp.StatusCode != 200 {
		glog.Warningf("Metric upload failed: %s", respBody)
		return fmt.Errorf("invalid status code: %d", resp.StatusCode)
	}
	var body string
	err = json.Unmarshal(respBody, &body)
	if err != nil {
		glog.Warningf("body=(%s), err=(%s)", respBody, err)
		return err
	}
	if body != "OK" {
		glog.Warningf("Response not OK: %s", body)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package schema

import (
	"time"
)

func (u *User) IdCol() int64 {
	return u.UserId
}

func (u *User) IdentCol() string {
	return u.Ident
}

func (u *User) SetCreatedTime(now time.Time) {
	u.CreatedTs = ToTspb(now)
}

func (u *User) SetModifiedTime(now time.Time) {
	u.ModifiedTs = ToTspb(now)
}

func (u *User) GetCreatedTime() time.Time {
	return ToTime(u.CreatedTs)
}

func (u *User) GetModifiedTime() time.Time {
	return ToTime(u.ModifiedTs)
}

func (u *User) Version() int64 {
	return ToTime(u.ModifiedTs).UnixNano()
}

func UserHasPerm(u *User, uc User_Capability) bool {
	for _, c := range u.Capability {
		if c == uc {
			return true
		}
	}
	return false
}

/**
 * The user id of the anonymous user.  Due to proto3, this is not distinguishable
 * from not being set, so bugs in the code will appear to set anonymous when they
 * shouldn't.  This seems okay, since tests can check most of this.  0 will mean
 * that "we don't know".  This means that either the user was actually anonymous,
 * or the data was created at a time when the user wasn't known, which are both
 * correct.  In the event of data corruption, we still don't know who the correct
 * user was, so 0 would be the unfortuantely correct answer.
 */
var AnonymousUserID int64 = 0

// TODO: make this configurable.
var (
	// Capabilities of Anonymous users
	UserAnonymousCap = []User_Capability{
		User_USER_CREATE,
	}

	// Capabilities of new users.
	UserNewCap = []User_Capability{
		User_PIC_READ,
		User_PIC_INDEX,
		User_PIC_UPDATE_VIEW_COUNTER,
		User_PIC_TAG_CREATE,
		User_PIC_COMMENT_CREATE,
		User_PIC_VOTE_CREATE,
		User_USER_READ_SELF,
	}
)

var AnonymousUser = &User{
	UserId:     AnonymousUserID,
	Capability: UserAnonymousCap,
}
<commit_msg>be/schema: add setter for last seen time<commit_after>package schema

import (
	"time"
)

func (u *User) IdCol() int64 {
	return u.UserId
}

func (u *User) IdentCol() string {
	return u.Ident
}

func (u *User) SetCreatedTime(now time.Time) {
	u.CreatedTs = ToTspb(now)
}

func (u *User) SetModifiedTime(now time.Time) {
	u.ModifiedTs = ToTspb(now)
}

func (u *User) SetLastSeenTime(now time.Time) {
	u.LastSeenTs = ToTspb(now)
}

func (u *User) GetCreatedTime() time.Time {
	return ToTime(u.CreatedTs)
}

func (u *User) GetModifiedTime() time.Time {
	return ToTime(u.ModifiedTs)
}

func (u *User) GetLastSeenTime() *time.Time {
	if u.LastSeenTs == nil {
		return nil
	}
	t := ToTime(u.LastSeenTs)
	return &t
}

func (u *User) Version() int64 {
	return ToTime(u.ModifiedTs).UnixNano()
}

func UserHasPerm(u *User, uc User_Capability) bool {
	for _, c := range u.Capability {
		if c == uc {
			return true
		}
	}
	return false
}

/**
 * The user id of the anonymous user.  Due to proto3, this is not distinguishable
 * from not being set, so bugs in the code will appear to set anonymous when they
 * shouldn't.  This seems okay, since tests can check most of this.  0 will mean
 * that "we don't know".  This means that either the user was actually anonymous,
 * or the data was created at a time when the user wasn't known, which are both
 * correct.  In the event of data corruption, we still don't know who the correct
 * user was, so 0 would be the unfortuantely correct answer.
 */
var AnonymousUserID int64 = 0

// TODO: make this configurable.
var (
	// Capabilities of Anonymous users
	UserAnonymousCap = []User_Capability{
		User_USER_CREATE,
	}

	// Capabilities of new users.
	UserNewCap = []User_Capability{
		User_PIC_READ,
		User_PIC_INDEX,
		User_PIC_UPDATE_VIEW_COUNTER,
		User_PIC_TAG_CREATE,
		User_PIC_COMMENT_CREATE,
		User_PIC_VOTE_CREATE,
		User_USER_READ_SELF,
	}
)

var AnonymousUser = &User{
	UserId:     AnonymousUserID,
	Capability: UserAnonymousCap,
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"time"

	"github.com/davecheney/profile"
	"github.com/dustin/go-humanize"
	"github.com/h2so5/utp"
)

type RandReader struct{}

func (r RandReader) Read(p []byte) (n int, err error) {
	for i := range p {
		p[i] = byte(rand.Int())
	}
	return len(p), nil
}

func main() {
	var l = flag.Int("c", 10485760, "Payload length (bytes)")
	var h = flag.Bool("h", false, "Human readable")
	flag.Parse()

	defer profile.Start(profile.CPUProfile).Stop()

	if *h {
		fmt.Printf("Payload: %s\n", humanize.IBytes(uint64(*l)))
	} else {
		fmt.Printf("Payload: %d\n", *l)
	}

	c2s := c2s(int64(*l))
	n, p := humanize.ComputeSI(c2s)
	if *h {
		fmt.Printf("C2S: %f%sbps\n", n, p)
	} else {
		fmt.Printf("C2S: %f\n", c2s)
	}

	s2c := s2c(int64(*l))
	n, p = humanize.ComputeSI(s2c)
	if *h {
		fmt.Printf("S2C: %f%sbps\n", n, p)
	} else {
		fmt.Printf("S2C: %f\n", s2c)
	}

	avg := (c2s + s2c) / 2.0
	n, p = humanize.ComputeSI(avg)

	if *h {
		fmt.Printf("AVG: %f%sbps\n", n, p)
	} else {
		fmt.Printf("AVG: %f\n", avg)
	}
}

func c2s(l int64) float64 {
	ln, err := utp.Listen("utp", "127.0.0.1:0")
	if err != nil {
		log.Fatal(err)
	}

	raddr, err := utp.ResolveUTPAddr("utp", ln.Addr().String())
	if err != nil {
		log.Fatal(err)
	}

	c, err := utp.DialUTPTimeout("utp", nil, raddr, 1000*time.Millisecond)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	if err != nil {
		log.Fatal(err)
	}

	s, err := ln.Accept()
	if err != nil {
		log.Fatal(err)
	}
	defer s.Close()
	ln.Close()

	rch := make(chan int)

	sendHash := md5.New()
	go func() {
		defer c.Close()
		io.Copy(io.MultiWriter(c, sendHash), io.LimitReader(RandReader{}, l))
	}()

	readHash := md5.New()
	go func() {
		io.Copy(readHash, s)
		rch <- 0
	}()

	start := time.Now()
	<-rch
	bps := float64(l*8) / (float64(time.Now().Sub(start)) / float64(time.Second))

	if !bytes.Equal(sendHash.Sum(nil), readHash.Sum(nil)) {
		log.Fatal("Broken payload")
	}

	return bps
}

func s2c(l int64) float64 {
	ln, err := utp.Listen("utp", "127.0.0.1:0")
	if err != nil {
		log.Fatal(err)
	}

	raddr, err := utp.ResolveUTPAddr("utp", ln.Addr().String())
	if err != nil {
		log.Fatal(err)
	}

	c, err := utp.DialUTPTimeout("utp", nil, raddr, 1000*time.Millisecond)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	if err != nil {
		log.Fatal(err)
	}

	s, err := ln.Accept()
	if err != nil {
		log.Fatal(err)
	}
	defer s.Close()
	ln.Close()

	rch := make(chan int)

	sendHash := md5.New()
	go func() {
		defer s.Close()
		io.Copy(io.MultiWriter(s, sendHash), io.LimitReader(RandReader{}, l))
	}()

	readHash := md5.New()
	go func() {
		io.Copy(readHash, c)
		rch <- 0
	}()

	start := time.Now()
	<-rch
	bps := float64(l*8) / (float64(time.Now().Sub(start)) / float64(time.Second))

	if !bytes.Equal(sendHash.Sum(nil), readHash.Sum(nil)) {
		log.Fatal("Broken payload")
	}

	return bps
}
<commit_msg>Add -s option to benchmark<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"time"

	"github.com/davecheney/profile"
	"github.com/dustin/go-humanize"
	"github.com/h2so5/utp"
)

type RandReader struct{}

func (r RandReader) Read(p []byte) (n int, err error) {
	for i := range p {
		p[i] = byte(rand.Int())
	}
	return len(p), nil
}

func main() {
	var l = flag.Int("c", 10485760, "Payload length (bytes)")
	var h = flag.Bool("h", false, "Human readable")
	var s = flag.Bool("s", false, "Stream mode(Low memory usage, but Slow)")
	flag.Parse()

	defer profile.Start(profile.CPUProfile).Stop()

	if *h {
		fmt.Printf("Payload: %s\n", humanize.IBytes(uint64(*l)))
	} else {
		fmt.Printf("Payload: %d\n", *l)
	}

	c2s := c2s(int64(*l), *s)
	n, p := humanize.ComputeSI(c2s)
	if *h {
		fmt.Printf("C2S: %f%sbps\n", n, p)
	} else {
		fmt.Printf("C2S: %f\n", c2s)
	}

	s2c := s2c(int64(*l), *s)
	n, p = humanize.ComputeSI(s2c)
	if *h {
		fmt.Printf("S2C: %f%sbps\n", n, p)
	} else {
		fmt.Printf("S2C: %f\n", s2c)
	}

	avg := (c2s + s2c) / 2.0
	n, p = humanize.ComputeSI(avg)

	if *h {
		fmt.Printf("AVG: %f%sbps\n", n, p)
	} else {
		fmt.Printf("AVG: %f\n", avg)
	}
}

func c2s(l int64, stream bool) float64 {
	ln, err := utp.Listen("utp", "127.0.0.1:0")
	if err != nil {
		log.Fatal(err)
	}

	raddr, err := utp.ResolveUTPAddr("utp", ln.Addr().String())
	if err != nil {
		log.Fatal(err)
	}

	c, err := utp.DialUTPTimeout("utp", nil, raddr, 1000*time.Millisecond)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	if err != nil {
		log.Fatal(err)
	}

	s, err := ln.Accept()
	if err != nil {
		log.Fatal(err)
	}
	defer s.Close()
	ln.Close()

	rch := make(chan int)

	sendHash := md5.New()
	readHash := md5.New()

	var bps float64
	if stream {
		go func() {
			defer c.Close()
			io.Copy(io.MultiWriter(c, sendHash), io.LimitReader(RandReader{}, l))
		}()

		go func() {
			io.Copy(readHash, s)
			rch <- 0
		}()

		start := time.Now()
		<-rch
		bps = float64(l*8) / (float64(time.Now().Sub(start)) / float64(time.Second))

	} else {
		var sendBuf, readBuf bytes.Buffer
		io.Copy(io.MultiWriter(&sendBuf, sendHash), io.LimitReader(RandReader{}, l))

		go func() {
			defer c.Close()
			io.Copy(c, &sendBuf)
		}()

		go func() {
			io.Copy(&readBuf, s)
			rch <- 0
		}()

		start := time.Now()
		<-rch
		bps = float64(l*8) / (float64(time.Now().Sub(start)) / float64(time.Second))

		io.Copy(sendHash, &sendBuf)
		io.Copy(readHash, &readBuf)
	}

	if !bytes.Equal(sendHash.Sum(nil), readHash.Sum(nil)) {
		log.Fatal("Broken payload")
	}

	return bps
}

func s2c(l int64, stream bool) float64 {
	ln, err := utp.Listen("utp", "127.0.0.1:0")
	if err != nil {
		log.Fatal(err)
	}

	raddr, err := utp.ResolveUTPAddr("utp", ln.Addr().String())
	if err != nil {
		log.Fatal(err)
	}

	c, err := utp.DialUTPTimeout("utp", nil, raddr, 1000*time.Millisecond)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	if err != nil {
		log.Fatal(err)
	}

	s, err := ln.Accept()
	if err != nil {
		log.Fatal(err)
	}
	defer s.Close()
	ln.Close()

	rch := make(chan int)

	sendHash := md5.New()
	readHash := md5.New()

	var bps float64

	if stream {
		go func() {
			defer s.Close()
			io.Copy(io.MultiWriter(s, sendHash), io.LimitReader(RandReader{}, l))
		}()

		go func() {
			io.Copy(readHash, c)
			rch <- 0
		}()

		start := time.Now()
		<-rch
		bps = float64(l*8) / (float64(time.Now().Sub(start)) / float64(time.Second))

	} else {
		var sendBuf, readBuf bytes.Buffer
		io.Copy(io.MultiWriter(&sendBuf, sendHash), io.LimitReader(RandReader{}, l))

		go func() {
			defer s.Close()
			io.Copy(s, &sendBuf)
		}()

		go func() {
			io.Copy(&readBuf, c)
			rch <- 0
		}()

		start := time.Now()
		<-rch
		bps = float64(l*8) / (float64(time.Now().Sub(start)) / float64(time.Second))

		io.Copy(sendHash, &sendBuf)
		io.Copy(readHash, &readBuf)
	}

	if !bytes.Equal(sendHash.Sum(nil), readHash.Sum(nil)) {
		log.Fatal("Broken payload")
	}

	return bps
}
<|endoftext|>


<commit_before>package nobu

import (
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"

	"zetsuboushita.net/vc_file_grouper/vc"
)

// Skill skill as known by nobu bot
type Skill struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

// NewSkills generates a skill array for a given card
func newSkills(c *vc.Card) []Skill {
	log.Printf("looking for skills on card %d:%s", c.ID, c.Name)
	skills := make([]Skill, 0)

	// skill 1
	s := c.Skill1()
	if s != nil {
		log.Printf("Found skill-1 on card %d:%s", c.ID, c.Name)
		sPrefix := "Skill"
		if len(c.Rarity()) == 3 {
			if c.Rarity()[0] == 'G' {
				sPrefix = "Awoken"
			} else if c.Rarity()[0] == 'X' {
				sPrefix = "Reborn"
			}
		}
		skills = append(skills, newSkill(sPrefix, s))
	}
	// skill 2
	s = c.Skill2()
	if s != nil && !s.Expires() {
		log.Printf("Found skill-2 on card %d:%s", c.ID, c.Name)
		sPrefix := "Second Skill"
		if len(c.Rarity()) == 3 {
			if c.Rarity()[0] == 'G' {
				sPrefix = "Awoken " + sPrefix
			} else if c.Rarity()[0] == 'X' {
				sPrefix = "Reborn " + sPrefix
			}
		}
		skills = append(skills, newSkill(sPrefix, s))
	}

	if a := c.LastEvo().AwakensTo(); a != nil {
		log.Printf("Found awakening on card %d:%s -> %d:%s", c.ID, c.Name, a.ID, a.Name)
		tmp := newSkills(a)
		skills = append(skills, tmp...)
		// the recursion will catch any rebirths
	}
	// will only pick up rebirths if we are looking at the awoken card.
	if r := c.RebirthsTo(); r != nil {
		log.Printf("Found rebirth on card %d:%s -> %d:%s", c.ID, c.Name, r.ID, r.Name)
		tmp := newSkills(r)
		skills = append(skills, tmp...)
	}

	log.Printf("Found %d skills total on card %d:%s", len(skills), c.ID, c.Name)
	return skills
}

func newSkill(sPrefix string, s *vc.Skill) Skill {
	activations := getActivations(s)
	sMin := cleanSkill(s.SkillMin())
	sMax := cleanSkill(s.SkillMax())
	skill := ""
	if sMin == sMax {
		skill = fmt.Sprintf("Activations: %s\nEffect: %s", activations, sMin)
	} else {
		skill = fmt.Sprintf("Activations: %s\nMin Level Effect: %s\nMax Level Effect: %s",
			activations,
			sMin,
			sMax,
		)
	}
	return Skill{
		Name:  fmt.Sprintf("%s: %s", sPrefix, s.Name), // Skill, Second Skill, Awoken, Awoken Second Skill
		Value: skill,
	}
}

func getActivations(s *vc.Skill) string {
	if s.MaxCount > 0 {
		return strconv.Itoa(s.MaxCount)
	} else if strings.Contains(s.SkillMin(), "【Autoskill】") {
		return "Always On"
	} else {
		return "Infinite"
	}
}

func cleanSkill(name string) string {
	name = strings.Replace(strings.Replace(name, "{{", "", -1), "}}", "", -1)
	re := regexp.MustCompile(`\s+[/\\]\s+Max \d+ time(s)?`)
	name = re.ReplaceAllString(name, "")
	return name
}
<commit_msg>add element icons for bot skills<commit_after>package nobu

import (
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"

	"zetsuboushita.net/vc_file_grouper/vc"
)

// Skill skill as known by nobu bot
type Skill struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

// NewSkills generates a skill array for a given card
func newSkills(c *vc.Card) []Skill {
	log.Printf("looking for skills on card %d:%s", c.ID, c.Name)
	skills := make([]Skill, 0)

	// skill 1
	s := c.Skill1()
	if s != nil {
		log.Printf("Found skill-1 on card %d:%s", c.ID, c.Name)
		sPrefix := "Skill"
		if len(c.Rarity()) == 3 {
			if c.Rarity()[0] == 'G' {
				sPrefix = "Awoken"
			} else if c.Rarity()[0] == 'X' {
				sPrefix = "Reborn"
			}
		}
		skills = append(skills, newSkill(sPrefix, s))
	}
	// skill 2
	s = c.Skill2()
	if s != nil && !s.Expires() {
		log.Printf("Found skill-2 on card %d:%s", c.ID, c.Name)
		sPrefix := "Second Skill"
		if len(c.Rarity()) == 3 {
			if c.Rarity()[0] == 'G' {
				sPrefix = "Awoken " + sPrefix
			} else if c.Rarity()[0] == 'X' {
				sPrefix = "Reborn " + sPrefix
			}
		}
		skills = append(skills, newSkill(sPrefix, s))
	}

	if a := c.LastEvo().AwakensTo(); a != nil {
		log.Printf("Found awakening on card %d:%s -> %d:%s", c.ID, c.Name, a.ID, a.Name)
		tmp := newSkills(a)
		skills = append(skills, tmp...)
		// the recursion will catch any rebirths
	}
	// will only pick up rebirths if we are looking at the awoken card.
	if r := c.RebirthsTo(); r != nil {
		log.Printf("Found rebirth on card %d:%s -> %d:%s", c.ID, c.Name, r.ID, r.Name)
		tmp := newSkills(r)
		skills = append(skills, tmp...)
	}

	log.Printf("Found %d skills total on card %d:%s", len(skills), c.ID, c.Name)
	return skills
}

func newSkill(sPrefix string, s *vc.Skill) Skill {
	activations := getActivations(s)
	sMin := cleanSkill(s.SkillMin())
	sMax := cleanSkill(s.SkillMax())
	skill := ""
	if sMin == sMax {
		skill = fmt.Sprintf("Activations: %s\nEffect: %s", activations, sMin)
	} else {
		skill = fmt.Sprintf("Activations: %s\nMin Level Effect: %s\nMax Level Effect: %s",
			activations,
			sMin,
			sMax,
		)
	}
	return Skill{
		Name:  fmt.Sprintf("%s: %s", sPrefix, s.Name), // Skill, Second Skill, Awoken, Awoken Second Skill
		Value: skill,
	}
}

func getActivations(s *vc.Skill) string {
	if s.MaxCount > 0 {
		return strconv.Itoa(s.MaxCount)
	} else if strings.Contains(s.SkillMin(), "【Autoskill】") {
		return "Always On"
	} else {
		return "Infinite"
	}
}

func cleanSkill(name string) string {
	//bot icons:
	//'<:Passion:534375884480577536>'
	//'<:Cool:534375884598018049>'
	//'<:Light:534375885541998602>'
	//'<:Dark:534375884279382016>'
	//'<:Special:534375884493291530>'
	name = strings.Replace(name, "{{Passion}}", "<:Passion:534375884480577536>", -1)
	name = strings.Replace(name, "{{Cool}}", "<:Cool:534375884598018049", -1)
	name = strings.Replace(name, "{{Light}}", "<:Light:534375885541998602>", -1)
	name = strings.Replace(name, "{{Dark}}", "<:Dark:534375884279382016>", -1)
	name = strings.Replace(name, "{{Special}}", "<:Special:534375884493291530>", -1)
	name = strings.Replace(strings.Replace(name, "{{", "", -1), "}}", "", -1)
	re := regexp.MustCompile(`\s+[/\\]\s+Max \d+ time(s)?`)
	name = re.ReplaceAllString(name, "")
	return name
}
<|endoftext|>