<commit_before>package moon

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"testing"

	"google.golang.org/appengine"

	"github.com/goji/httpauth"
	"golang.org/x/net/context"
)

func assertEquals(t *testing.T, e interface{}, o interface{}) {
	if e != o {
		t.Errorf("\n...expected = %v\n...obtained = %v", e, o)
	}
}

func serveAndRequest(h http.Handler, auth bool) string {
	//auth := base64.StdEncoding.EncodeToString([]byte("user:pass"))
	//r.Header.Set("Authorization", "Basic "+auth)

	ts := httptest.NewServer(h)
	defer ts.Close()

	req, _ := http.NewRequest("GET", ts.URL, nil)

	if auth {
		req.SetBasicAuth("user", "pass")
	}

	res, err := http.DefaultClient.Do(req)
	//res, err := http.Client.Do(req)

	// res, err := http.Get(ts.URL)
	if err != nil {
		log.Fatal(err)
	}
	resBody, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}
	//	log.Print(resBody)
	return string(resBody)
}

func tokenMiddlewareA(ctx context.Context, next HandlerWithContext) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ctx = context.WithValue(ctx, "tokenA", "123")
		next.ServeHTTP(ctx, w, r)
	})
}

func tokenMiddlewareB(ctx context.Context, next HandlerWithContext) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ctx = context.WithValue(ctx, "tokenB", "456")
		next.ServeHTTP(ctx, w, r)
	})
}

func tokenHandler(ctx context.Context) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Tokens are: %v, %v", ctx.Value("tokenA"), ctx.Value("tokenB"))
	})
}

func TestContext(t *testing.T) {
	Context = func(r *http.Request) context.Context {
		return appengine.NewContext(r)
	}
	st := New(tokenMiddlewareA, tokenMiddlewareB).Then(tokenHandler)
	res := serveAndRequest(st, false)
	assertEquals(t, "Tokens are: 123, 456", res)
}

func authHandler(ctx context.Context) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		u, p, _ := r.BasicAuth()
		fmt.Fprintf(w, "Hello: %v, %v", u, p)
	})
}

func TestGojiBasicAuthUnauthorized(t *testing.T) {
	st := New(Adapt(httpauth.SimpleBasicAuth("user", "pass"))).Then(authHandler)
	res := serveAndRequest(st, false)
	assertEquals(t, "Unauthorized\n", res)
}

func TestGojiBasicAuthAuthorized(t *testing.T) {
	st := New(Adapt(httpauth.SimpleBasicAuth("user", "pass"))).Then(authHandler)
	res := serveAndRequest(st, true)
	assertEquals(t, "Hello: user, pass", res)
}
<commit_msg>Remove appengine context from test<commit_after>package moon

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/goji/httpauth"
	"golang.org/x/net/context"
)

func assertEquals(t *testing.T, e interface{}, o interface{}) {
	if e != o {
		t.Errorf("\n...expected = %v\n...obtained = %v", e, o)
	}
}

func serveAndRequest(h http.Handler, auth bool) string {
	//auth := base64.StdEncoding.EncodeToString([]byte("user:pass"))
	//r.Header.Set("Authorization", "Basic "+auth)

	ts := httptest.NewServer(h)
	defer ts.Close()

	req, _ := http.NewRequest("GET", ts.URL, nil)

	if auth {
		req.SetBasicAuth("user", "pass")
	}

	res, err := http.DefaultClient.Do(req)
	//res, err := http.Client.Do(req)

	// res, err := http.Get(ts.URL)
	if err != nil {
		log.Fatal(err)
	}
	resBody, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}
	//	log.Print(resBody)
	return string(resBody)
}

func tokenMiddlewareA(ctx context.Context, next HandlerWithContext) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ctx = context.WithValue(ctx, "tokenA", "123")
		next.ServeHTTP(ctx, w, r)
	})
}

func tokenMiddlewareB(ctx context.Context, next HandlerWithContext) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ctx = context.WithValue(ctx, "tokenB", "456")
		next.ServeHTTP(ctx, w, r)
	})
}

func tokenHandler(ctx context.Context) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Tokens are: %v, %v", ctx.Value("tokenA"), ctx.Value("tokenB"))
	})
}

func TestContext(t *testing.T) {
	st := New(tokenMiddlewareA, tokenMiddlewareB).Then(tokenHandler)
	res := serveAndRequest(st, false)
	assertEquals(t, "Tokens are: 123, 456", res)
}

func TestContextRoot(t *testing.T) {
	Context = func(r *http.Request) context.Context {
		ctx := context.TODO()
		return context.WithValue(ctx, "tokenA", "789")
	}
	st := New(tokenMiddlewareB).Then(tokenHandler)
	res := serveAndRequest(st, false)
	assertEquals(t, "Tokens are: 789, 456", res)
}

func authHandler(ctx context.Context) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		u, p, _ := r.BasicAuth()
		fmt.Fprintf(w, "Hello: %v, %v", u, p)
	})
}

func TestGojiBasicAuthUnauthorized(t *testing.T) {
	st := New(Adapt(httpauth.SimpleBasicAuth("user", "pass"))).Then(authHandler)
	res := serveAndRequest(st, false)
	assertEquals(t, "Unauthorized\n", res)
}

func TestGojiBasicAuthAuthorized(t *testing.T) {
	st := New(Adapt(httpauth.SimpleBasicAuth("user", "pass"))).Then(authHandler)
	res := serveAndRequest(st, true)
	assertEquals(t, "Hello: user, pass", res)
}
<|endoftext|>


<commit_before>package handlers

import (
	"crypto/sha512"
	"encoding/hex"
	"fmt"

	"github.com/Hucaru/Valhalla/common/connection"
	"github.com/Hucaru/Valhalla/common/constants"
	"github.com/Hucaru/Valhalla/common/packet"
	"github.com/Hucaru/Valhalla/loginServer/loginConn"
)

// HandlePacket -
func HandlePacket(conn *loginConn.Connection, buffer packet.Packet) {

	// Handle data packet
	pos := 0

	opcode := buffer.ReadByte(&pos)

	switch opcode {
	case constants.LOGIN_REQUEST:
		handleLoginRequest(buffer, &pos, conn)
	case constants.LOGIN_CHECK_LOGIN:
		handleCheckLogin(conn)
	default:
		fmt.Println("UNKNOWN LOGIN PACKET:", buffer)
	}

}

func handleLoginRequest(p packet.Packet, pos *int, conn *loginConn.Connection) {
	usernameLength := p.ReadInt16(pos)
	username := p.ReadString(pos, int(usernameLength))

	passwordLength := p.ReadInt16(pos)
	password := p.ReadString(pos, int(passwordLength))

	// hash the password
	hasher := sha512.New()
	hasher.Write([]byte(password))
	hashedPassword := hex.EncodeToString(hasher.Sum(nil))

	var userID uint32
	var user string
	var databasePassword string
	var isLogedIn bool
	var isBanned int
	var isAdmin bool

	err := connection.Db.QueryRow("SELECT userID, username, password, isLogedIn, isBanned, isAdmin FROM users WHERE username=?", username).
		Scan(&userID, &user, &databasePassword, &isLogedIn, &isBanned, &isAdmin)

	result := byte(0x00)

	if err != nil {
		result = 0x05
	} else if hashedPassword != databasePassword {
		result = 0x04
	} else if isLogedIn {
		result = 0x07
	} else if isBanned > 0 {
		result = 0x02
	}

	// -Banned- = 2
	// Deleted or Blocked = 3
	// Invalid Password = 4
	// Not Registered = 5
	// Sys Error = 6
	// Already online = 7
	// System error = 9
	// Too many requests = 10
	// Older than 20 = 11
	// Master cannot login on this IP = 13

	pac := packet.NewPacket()
	pac.WriteByte(constants.LOGIN_RESPONCE)
	pac.WriteByte(result)
	pac.WriteByte(0x00)
	pac.WriteInt32(0)

	if result <= 0x01 {

		pac.WriteUint32(userID)
		pac.WriteByte(0x00)
		if isAdmin {
			pac.WriteByte(0x01)
		} else {
			pac.WriteByte(0x00)
		}
		pac.WriteByte(0x01)
		pac.WriteString(username)

		conn.SetUserID(userID)
		conn.SetIsLogedIn(true)
		_, err = connection.Db.Query("UPDATE users set isLogedIn=1 WHERE userID=?", userID)
	} else if result == 0x02 {
		pac.WriteByte(byte(isBanned))
		pac.WriteInt64(0) // Expire time, for now let set this to epoch
	}

	pac.WriteInt64(0)
	pac.WriteInt64(0)
	pac.WriteInt64(0)
	conn.Write(pac)

	if err != nil {
		fmt.Println(err)
	}
}

func handleCheckLogin(conn *loginConn.Connection) {
	// No idea what this packet is for
	pac := packet.NewPacket()
	pac.WriteByte(0x03)
	pac.WriteByte(0x04)
	pac.WriteByte(0x00)
	conn.Write(pac)

	var username string

	userID := conn.GetUserID()

	err := connection.Db.QueryRow("SELECT username FROM users WHERE userID=?", userID).
		Scan(&username)

	if err != nil {
		fmt.Println("handleCheckLogin database retrieval issue for userID:", userID, err)
	}

	hasher := sha512.New()
	hasher.Write([]byte(username)) // Username should be unique so might as well use this
	hashedUsername := fmt.Sprintf("%x02", hasher.Sum(nil))

	conn.SetSessionHash(hashedUsername)

	pac = packet.NewPacket()
	pac.WriteByte(constants.LOGIN_SEND_SESSION_HASH)
	pac.WriteString(hashedUsername)
	conn.Write(pac)
}
<commit_msg>Comment on packet I don't understand. Also previous commit fixed loging user in even if bad connect<commit_after>package handlers

import (
	"crypto/sha512"
	"encoding/hex"
	"fmt"

	"github.com/Hucaru/Valhalla/common/connection"
	"github.com/Hucaru/Valhalla/common/constants"
	"github.com/Hucaru/Valhalla/common/packet"
	"github.com/Hucaru/Valhalla/loginServer/loginConn"
)

// HandlePacket -
func HandlePacket(conn *loginConn.Connection, buffer packet.Packet) {

	// Handle data packet
	pos := 0

	opcode := buffer.ReadByte(&pos)

	switch opcode {
	case constants.LOGIN_REQUEST:
		handleLoginRequest(buffer, &pos, conn)
	case constants.LOGIN_CHECK_LOGIN:
		handleCheckLogin(conn)
	default:
		fmt.Println("UNKNOWN LOGIN PACKET:", buffer)
	}

}

func handleLoginRequest(p packet.Packet, pos *int, conn *loginConn.Connection) {
	usernameLength := p.ReadInt16(pos)
	username := p.ReadString(pos, int(usernameLength))

	passwordLength := p.ReadInt16(pos)
	password := p.ReadString(pos, int(passwordLength))

	// hash the password
	hasher := sha512.New()
	hasher.Write([]byte(password))
	hashedPassword := hex.EncodeToString(hasher.Sum(nil))

	var userID uint32
	var user string
	var databasePassword string
	var isLogedIn bool
	var isBanned int
	var isAdmin bool

	err := connection.Db.QueryRow("SELECT userID, username, password, isLogedIn, isBanned, isAdmin FROM users WHERE username=?", username).
		Scan(&userID, &user, &databasePassword, &isLogedIn, &isBanned, &isAdmin)

	result := byte(0x00)

	if err != nil {
		result = 0x05
	} else if hashedPassword != databasePassword {
		result = 0x04
	} else if isLogedIn {
		result = 0x07
	} else if isBanned > 0 {
		result = 0x02
	}

	// -Banned- = 2
	// Deleted or Blocked = 3
	// Invalid Password = 4
	// Not Registered = 5
	// Sys Error = 6
	// Already online = 7
	// System error = 9
	// Too many requests = 10
	// Older than 20 = 11
	// Master cannot login on this IP = 13

	pac := packet.NewPacket()
	pac.WriteByte(constants.LOGIN_RESPONCE)
	pac.WriteByte(result)
	pac.WriteByte(0x00)
	pac.WriteInt32(0)

	if result <= 0x01 {

		pac.WriteUint32(userID)
		pac.WriteByte(0x00)
		if isAdmin {
			pac.WriteByte(0x01)
		} else {
			pac.WriteByte(0x00)
		}
		pac.WriteByte(0x01)
		pac.WriteString(username)

		conn.SetUserID(userID)
		conn.SetIsLogedIn(true)
		_, err = connection.Db.Query("UPDATE users set isLogedIn=1 WHERE userID=?", userID)
	} else if result == 0x02 {
		pac.WriteByte(byte(isBanned))
		pac.WriteInt64(0) // Expire time, for now let set this to epoch
	}

	pac.WriteInt64(0)
	pac.WriteInt64(0)
	pac.WriteInt64(0)
	conn.Write(pac)

	if err != nil {
		fmt.Println(err)
	}
}

func handleCheckLogin(conn *loginConn.Connection) {
	// No idea what this packet is for
	pac := packet.NewPacket()
	pac.WriteByte(0x03)
	pac.WriteByte(0x04) // This value seems to denote server fullness?
	pac.WriteByte(0x00)
	conn.Write(pac)

	var username string

	userID := conn.GetUserID()

	err := connection.Db.QueryRow("SELECT username FROM users WHERE userID=?", userID).
		Scan(&username)

	if err != nil {
		fmt.Println("handleCheckLogin database retrieval issue for userID:", userID, err)
	}

	hasher := sha512.New()
	hasher.Write([]byte(username)) // Username should be unique so might as well use this
	hashedUsername := fmt.Sprintf("%x02", hasher.Sum(nil))

	conn.SetSessionHash(hashedUsername)

	pac = packet.NewPacket()
	pac.WriteByte(constants.LOGIN_SEND_SESSION_HASH)
	pac.WriteString(hashedUsername)
	conn.Write(pac)
}
<|endoftext|>


<commit_before>// Package multihash is the Go implementation of
// https://github.com/multiformats/multihash, or self-describing
// hashes.
package multihash

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"math"

	b58 "github.com/mr-tron/base58/base58"
)

// errors
var (
	ErrUnknownCode      = errors.New("unknown multihash code")
	ErrTooShort         = errors.New("multihash too short. must be >= 2 bytes")
	ErrTooLong          = errors.New("multihash too long. must be < 129 bytes")
	ErrLenNotSupported  = errors.New("multihash does not yet support digests longer than 127 bytes")
	ErrInvalidMultihash = errors.New("input isn't valid multihash")

	ErrVarintBufferShort = errors.New("uvarint: buffer too small")
	ErrVarintTooLong     = errors.New("uvarint: varint too big (max 64bit)")
)

// ErrInconsistentLen is returned when a decoded multihash has an inconsistent length
type ErrInconsistentLen struct {
	dm *DecodedMultihash
}

func (e ErrInconsistentLen) Error() string {
	return fmt.Sprintf("multihash length inconsistent: %v", e.dm)
}

// constants
const (
	ID         = 0x00
	SHA1       = 0x11
	SHA2_256   = 0x12
	SHA2_512   = 0x13
	SHA3_224   = 0x17
	SHA3_256   = 0x16
	SHA3_384   = 0x15
	SHA3_512   = 0x14
	SHA3       = SHA3_512
	KECCAK_224 = 0x1A
	KECCAK_256 = 0x1B
	KECCAK_384 = 0x1C
	KECCAK_512 = 0x1D

	SHAKE_128 = 0x18
	SHAKE_256 = 0x19

	BLAKE2B_MIN = 0xb201
	BLAKE2B_MAX = 0xb240
	BLAKE2S_MIN = 0xb241
	BLAKE2S_MAX = 0xb260

	MD5 = 0xd5

	DBL_SHA2_256 = 0x56

	MURMUR3 = 0x22

	X11 = 0x1100
)

func init() {
	// Add blake2b (64 codes)
	for c := uint64(BLAKE2B_MIN); c <= BLAKE2B_MAX; c++ {
		n := c - BLAKE2B_MIN + 1
		name := fmt.Sprintf("blake2b-%d", n*8)
		Names[name] = c
		Codes[c] = name
		DefaultLengths[c] = int(n)
	}

	// Add blake2s (32 codes)
	for c := uint64(BLAKE2S_MIN); c <= BLAKE2S_MAX; c++ {
		n := c - BLAKE2S_MIN + 1
		name := fmt.Sprintf("blake2s-%d", n*8)
		Names[name] = c
		Codes[c] = name
		DefaultLengths[c] = int(n)
	}
}

// Names maps the name of a hash to the code
var Names = map[string]uint64{
	"id":           ID,
	"sha1":         SHA1,
	"sha2-256":     SHA2_256,
	"sha2-512":     SHA2_512,
	"sha3":         SHA3_512,
	"sha3-224":     SHA3_224,
	"sha3-256":     SHA3_256,
	"sha3-384":     SHA3_384,
	"sha3-512":     SHA3_512,
	"dbl-sha2-256": DBL_SHA2_256,
	"murmur3":      MURMUR3,
	"keccak-224":   KECCAK_224,
	"keccak-256":   KECCAK_256,
	"keccak-384":   KECCAK_384,
	"keccak-512":   KECCAK_512,
	"shake-128":    SHAKE_128,
	"shake-256":    SHAKE_256,
	"x11":          X11,
	"md5":          MD5,
}

// Codes maps a hash code to it's name
var Codes = map[uint64]string{
	ID:           "id",
	SHA1:         "sha1",
	SHA2_256:     "sha2-256",
	SHA2_512:     "sha2-512",
	SHA3_224:     "sha3-224",
	SHA3_256:     "sha3-256",
	SHA3_384:     "sha3-384",
	SHA3_512:     "sha3-512",
	DBL_SHA2_256: "dbl-sha2-256",
	MURMUR3:      "murmur3",
	KECCAK_224:   "keccak-224",
	KECCAK_256:   "keccak-256",
	KECCAK_384:   "keccak-384",
	KECCAK_512:   "keccak-512",
	SHAKE_128:    "shake-128",
	SHAKE_256:    "shake-256",
	X11:          "x11",
	MD5:          "md5",
}

// DefaultLengths maps a hash code to it's default length
var DefaultLengths = map[uint64]int{
	ID:           -1,
	SHA1:         20,
	SHA2_256:     32,
	SHA2_512:     64,
	SHA3_224:     28,
	SHA3_256:     32,
	SHA3_384:     48,
	SHA3_512:     64,
	DBL_SHA2_256: 32,
	KECCAK_224:   28,
	KECCAK_256:   32,
	MURMUR3:      4,
	KECCAK_384:   48,
	KECCAK_512:   64,
	SHAKE_128:    32,
	SHAKE_256:    64,
	X11:          64,
	MD5:          16,
}

func uvarint(buf []byte) (uint64, []byte, error) {
	n, c := binary.Uvarint(buf)

	if c == 0 {
		return n, buf, ErrVarintBufferShort
	} else if c < 0 {
		return n, buf[-c:], ErrVarintTooLong
	} else {
		return n, buf[c:], nil
	}
}

// DecodedMultihash represents a parsed multihash and allows
// easy access to the different parts of a multihash.
type DecodedMultihash struct {
	Code   uint64
	Name   string
	Length int    // Length is just int as it is type of len() opearator
	Digest []byte // Digest holds the raw multihash bytes
}

// Multihash is byte slice with the following form:
// <hash function code><digest size><hash function output>.
// See the spec for more information.
type Multihash []byte

// HexString returns the hex-encoded representation of a multihash.
func (m *Multihash) HexString() string {
	return hex.EncodeToString([]byte(*m))
}

// String is an alias to HexString().
func (m *Multihash) String() string {
	return m.HexString()
}

// FromHexString parses a hex-encoded multihash.
func FromHexString(s string) (Multihash, error) {
	b, err := hex.DecodeString(s)
	if err != nil {
		return Multihash{}, err
	}

	return Cast(b)
}

// B58String returns the B58-encoded representation of a multihash.
func (m Multihash) B58String() string {
	return b58.Encode([]byte(m))
}

// FromB58String parses a B58-encoded multihash.
func FromB58String(s string) (m Multihash, err error) {
	b, err := b58.Decode(s)
	if err != nil {
		return Multihash{}, ErrInvalidMultihash
	}

	return Cast(b)
}

// Cast casts a buffer onto a multihash, and returns an error
// if it does not work.
func Cast(buf []byte) (Multihash, error) {
	dm, err := Decode(buf)
	if err != nil {
		return Multihash{}, err
	}

	if !ValidCode(dm.Code) {
		return Multihash{}, ErrUnknownCode
	}

	return Multihash(buf), nil
}

// Decode parses multihash bytes into a DecodedMultihash.
func Decode(buf []byte) (*DecodedMultihash, error) {

	if len(buf) < 2 {
		return nil, ErrTooShort
	}

	var err error
	var code, length uint64

	code, buf, err = uvarint(buf)
	if err != nil {
		return nil, err
	}

	length, buf, err = uvarint(buf)
	if err != nil {
		return nil, err
	}

	if length > math.MaxInt32 {
		return nil, errors.New("digest too long, supporting only <= 2^31-1")
	}

	dm := &DecodedMultihash{
		Code:   code,
		Name:   Codes[code],
		Length: int(length),
		Digest: buf,
	}

	if len(dm.Digest) != dm.Length {
		return nil, ErrInconsistentLen{dm}
	}

	return dm, nil
}

// Encode a hash digest along with the specified function code.
// Note: the length is derived from the length of the digest itself.
func Encode(buf []byte, code uint64) ([]byte, error) {

	if !ValidCode(code) {
		return nil, ErrUnknownCode
	}

	start := make([]byte, 2*binary.MaxVarintLen64, 2*binary.MaxVarintLen64+len(buf))
	spot := start
	n := binary.PutUvarint(spot, code)
	spot = start[n:]
	n += binary.PutUvarint(spot, uint64(len(buf)))

	return append(start[:n], buf...), nil
}

// EncodeName is like Encode() but providing a string name
// instead of a numeric code. See Names for allowed values.
func EncodeName(buf []byte, name string) ([]byte, error) {
	return Encode(buf, Names[name])
}

// ValidCode checks whether a multihash code is valid.
func ValidCode(code uint64) bool {
	_, ok := Codes[code]
	return ok
}
<commit_msg>improve inconsistent length error<commit_after>// Package multihash is the Go implementation of
// https://github.com/multiformats/multihash, or self-describing
// hashes.
package multihash

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"math"

	b58 "github.com/mr-tron/base58/base58"
)

// errors
var (
	ErrUnknownCode      = errors.New("unknown multihash code")
	ErrTooShort         = errors.New("multihash too short. must be >= 2 bytes")
	ErrTooLong          = errors.New("multihash too long. must be < 129 bytes")
	ErrLenNotSupported  = errors.New("multihash does not yet support digests longer than 127 bytes")
	ErrInvalidMultihash = errors.New("input isn't valid multihash")

	ErrVarintBufferShort = errors.New("uvarint: buffer too small")
	ErrVarintTooLong     = errors.New("uvarint: varint too big (max 64bit)")
)

// ErrInconsistentLen is returned when a decoded multihash has an inconsistent length
type ErrInconsistentLen struct {
	dm *DecodedMultihash
}

func (e ErrInconsistentLen) Error() string {
	return fmt.Sprintf("multihash length inconsistent: expected %d, got %d", e.dm.Length, len(e.dm.Digest))
}

// constants
const (
	ID         = 0x00
	SHA1       = 0x11
	SHA2_256   = 0x12
	SHA2_512   = 0x13
	SHA3_224   = 0x17
	SHA3_256   = 0x16
	SHA3_384   = 0x15
	SHA3_512   = 0x14
	SHA3       = SHA3_512
	KECCAK_224 = 0x1A
	KECCAK_256 = 0x1B
	KECCAK_384 = 0x1C
	KECCAK_512 = 0x1D

	SHAKE_128 = 0x18
	SHAKE_256 = 0x19

	BLAKE2B_MIN = 0xb201
	BLAKE2B_MAX = 0xb240
	BLAKE2S_MIN = 0xb241
	BLAKE2S_MAX = 0xb260

	MD5 = 0xd5

	DBL_SHA2_256 = 0x56

	MURMUR3 = 0x22

	X11 = 0x1100
)

func init() {
	// Add blake2b (64 codes)
	for c := uint64(BLAKE2B_MIN); c <= BLAKE2B_MAX; c++ {
		n := c - BLAKE2B_MIN + 1
		name := fmt.Sprintf("blake2b-%d", n*8)
		Names[name] = c
		Codes[c] = name
		DefaultLengths[c] = int(n)
	}

	// Add blake2s (32 codes)
	for c := uint64(BLAKE2S_MIN); c <= BLAKE2S_MAX; c++ {
		n := c - BLAKE2S_MIN + 1
		name := fmt.Sprintf("blake2s-%d", n*8)
		Names[name] = c
		Codes[c] = name
		DefaultLengths[c] = int(n)
	}
}

// Names maps the name of a hash to the code
var Names = map[string]uint64{
	"id":           ID,
	"sha1":         SHA1,
	"sha2-256":     SHA2_256,
	"sha2-512":     SHA2_512,
	"sha3":         SHA3_512,
	"sha3-224":     SHA3_224,
	"sha3-256":     SHA3_256,
	"sha3-384":     SHA3_384,
	"sha3-512":     SHA3_512,
	"dbl-sha2-256": DBL_SHA2_256,
	"murmur3":      MURMUR3,
	"keccak-224":   KECCAK_224,
	"keccak-256":   KECCAK_256,
	"keccak-384":   KECCAK_384,
	"keccak-512":   KECCAK_512,
	"shake-128":    SHAKE_128,
	"shake-256":    SHAKE_256,
	"x11":          X11,
	"md5":          MD5,
}

// Codes maps a hash code to it's name
var Codes = map[uint64]string{
	ID:           "id",
	SHA1:         "sha1",
	SHA2_256:     "sha2-256",
	SHA2_512:     "sha2-512",
	SHA3_224:     "sha3-224",
	SHA3_256:     "sha3-256",
	SHA3_384:     "sha3-384",
	SHA3_512:     "sha3-512",
	DBL_SHA2_256: "dbl-sha2-256",
	MURMUR3:      "murmur3",
	KECCAK_224:   "keccak-224",
	KECCAK_256:   "keccak-256",
	KECCAK_384:   "keccak-384",
	KECCAK_512:   "keccak-512",
	SHAKE_128:    "shake-128",
	SHAKE_256:    "shake-256",
	X11:          "x11",
	MD5:          "md5",
}

// DefaultLengths maps a hash code to it's default length
var DefaultLengths = map[uint64]int{
	ID:           -1,
	SHA1:         20,
	SHA2_256:     32,
	SHA2_512:     64,
	SHA3_224:     28,
	SHA3_256:     32,
	SHA3_384:     48,
	SHA3_512:     64,
	DBL_SHA2_256: 32,
	KECCAK_224:   28,
	KECCAK_256:   32,
	MURMUR3:      4,
	KECCAK_384:   48,
	KECCAK_512:   64,
	SHAKE_128:    32,
	SHAKE_256:    64,
	X11:          64,
	MD5:          16,
}

func uvarint(buf []byte) (uint64, []byte, error) {
	n, c := binary.Uvarint(buf)

	if c == 0 {
		return n, buf, ErrVarintBufferShort
	} else if c < 0 {
		return n, buf[-c:], ErrVarintTooLong
	} else {
		return n, buf[c:], nil
	}
}

// DecodedMultihash represents a parsed multihash and allows
// easy access to the different parts of a multihash.
type DecodedMultihash struct {
	Code   uint64
	Name   string
	Length int    // Length is just int as it is type of len() opearator
	Digest []byte // Digest holds the raw multihash bytes
}

// Multihash is byte slice with the following form:
// <hash function code><digest size><hash function output>.
// See the spec for more information.
type Multihash []byte

// HexString returns the hex-encoded representation of a multihash.
func (m *Multihash) HexString() string {
	return hex.EncodeToString([]byte(*m))
}

// String is an alias to HexString().
func (m *Multihash) String() string {
	return m.HexString()
}

// FromHexString parses a hex-encoded multihash.
func FromHexString(s string) (Multihash, error) {
	b, err := hex.DecodeString(s)
	if err != nil {
		return Multihash{}, err
	}

	return Cast(b)
}

// B58String returns the B58-encoded representation of a multihash.
func (m Multihash) B58String() string {
	return b58.Encode([]byte(m))
}

// FromB58String parses a B58-encoded multihash.
func FromB58String(s string) (m Multihash, err error) {
	b, err := b58.Decode(s)
	if err != nil {
		return Multihash{}, ErrInvalidMultihash
	}

	return Cast(b)
}

// Cast casts a buffer onto a multihash, and returns an error
// if it does not work.
func Cast(buf []byte) (Multihash, error) {
	dm, err := Decode(buf)
	if err != nil {
		return Multihash{}, err
	}

	if !ValidCode(dm.Code) {
		return Multihash{}, ErrUnknownCode
	}

	return Multihash(buf), nil
}

// Decode parses multihash bytes into a DecodedMultihash.
func Decode(buf []byte) (*DecodedMultihash, error) {

	if len(buf) < 2 {
		return nil, ErrTooShort
	}

	var err error
	var code, length uint64

	code, buf, err = uvarint(buf)
	if err != nil {
		return nil, err
	}

	length, buf, err = uvarint(buf)
	if err != nil {
		return nil, err
	}

	if length > math.MaxInt32 {
		return nil, errors.New("digest too long, supporting only <= 2^31-1")
	}

	dm := &DecodedMultihash{
		Code:   code,
		Name:   Codes[code],
		Length: int(length),
		Digest: buf,
	}

	if len(dm.Digest) != dm.Length {
		return nil, ErrInconsistentLen{dm}
	}

	return dm, nil
}

// Encode a hash digest along with the specified function code.
// Note: the length is derived from the length of the digest itself.
func Encode(buf []byte, code uint64) ([]byte, error) {

	if !ValidCode(code) {
		return nil, ErrUnknownCode
	}

	start := make([]byte, 2*binary.MaxVarintLen64, 2*binary.MaxVarintLen64+len(buf))
	spot := start
	n := binary.PutUvarint(spot, code)
	spot = start[n:]
	n += binary.PutUvarint(spot, uint64(len(buf)))

	return append(start[:n], buf...), nil
}

// EncodeName is like Encode() but providing a string name
// instead of a numeric code. See Names for allowed values.
func EncodeName(buf []byte, name string) ([]byte, error) {
	return Encode(buf, Names[name])
}

// ValidCode checks whether a multihash code is valid.
func ValidCode(code uint64) bool {
	_, ok := Codes[code]
	return ok
}
<|endoftext|>


<commit_before>package compress

import (
	"github.com/reducedb/encoding/cursor"
	"github.com/reducedb/encoding/delta/bp32"
)

// Compress compresses in[]int32 to out[]int32
func Compress32(in []int32) (out []int32, err error) {
	out = make([]int32, len(in)*2)
	inpos := cursor.New()
	outpos := cursor.New()

	if err = bp32.New().Compress(in, inpos, len(in), out, outpos); err != nil {
		return nil, err
	}

	return out[:outpos.Get()], nil
}

// Uncompress uncompresses in[]int32 to out[]int32
func Uncompress32(in []int32, buffer []int32) (out []int32, err error) {
	out = buffer
	inpos := cursor.New()
	outpos := cursor.New()

	if err = bp32.New().Uncompress(in, inpos, len(in), out, outpos); err != nil {
		return nil, err
	}

	return out[:outpos.Get()], nil
}
<commit_msg>update upstream<commit_after>package compress

import (
	"github.com/dataence/encoding/cursor"
	"github.com/dataence/encoding/delta/bp32"
)

// Compress compresses in[]int32 to out[]int32
func Compress32(in []int32) (out []int32, err error) {
	out = make([]int32, len(in)*2)
	inpos := cursor.New()
	outpos := cursor.New()

	if err = bp32.New().Compress(in, inpos, len(in), out, outpos); err != nil {
		return nil, err
	}

	return out[:outpos.Get()], nil
}

// Uncompress uncompresses in[]int32 to out[]int32
func Uncompress32(in []int32, buffer []int32) (out []int32, err error) {
	out = buffer
	inpos := cursor.New()
	outpos := cursor.New()

	if err = bp32.New().Uncompress(in, inpos, len(in), out, outpos); err != nil {
		return nil, err
	}

	return out[:outpos.Get()], nil
}
<|endoftext|>


<commit_before>package sequence

import (
	"fmt"
	"hash/fnv"

	"github.com/bwmarrin/snowflake"
	"github.com/chrislusf/seaweedfs/weed/glog"
)

// a simple snowflake Sequencer
type SnowflakeSequencer struct {
	node *snowflake.Node
}

func NewSnowflakeSequencer(nodeid string) (*SnowflakeSequencer, error) {
	nodeid_hash := hash(nodeid) & 0x3ff
	glog.V(0).Infof("use snowfalke seq id generator, nodeid:%s hex_of_nodeid: %x", nodeid, nodeid_hash)
	node, err := snowflake.NewNode(int64(nodeid_hash))
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	sequencer := &SnowflakeSequencer{node: node}
	return sequencer, nil
}

func hash(s string) uint32 {
	h := fnv.New32a()
	h.Write([]byte(s))
	return h.Sum32()
}

func (m *SnowflakeSequencer) NextFileId(count uint64) uint64 {
	return uint64(m.node.Generate().Int64())
}

// ignore setmax as we are snowflake
func (m *SnowflakeSequencer) SetMax(seenValue uint64) {
}

// return a new id as no Peek is stored
func (m *SnowflakeSequencer) Peek() uint64 {
	return uint64(m.node.Generate().Int64())
}
<commit_msg>fix typo<commit_after>package sequence

import (
	"fmt"
	"hash/fnv"

	"github.com/bwmarrin/snowflake"
	"github.com/chrislusf/seaweedfs/weed/glog"
)

// a simple snowflake Sequencer
type SnowflakeSequencer struct {
	node *snowflake.Node
}

func NewSnowflakeSequencer(nodeid string) (*SnowflakeSequencer, error) {
	nodeid_hash := hash(nodeid) & 0x3ff
	glog.V(0).Infof("use snowflake seq id generator, nodeid:%s hex_of_nodeid: %x", nodeid, nodeid_hash)
	node, err := snowflake.NewNode(int64(nodeid_hash))
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	sequencer := &SnowflakeSequencer{node: node}
	return sequencer, nil
}

func hash(s string) uint32 {
	h := fnv.New32a()
	h.Write([]byte(s))
	return h.Sum32()
}

func (m *SnowflakeSequencer) NextFileId(count uint64) uint64 {
	return uint64(m.node.Generate().Int64())
}

// ignore setmax as we are snowflake
func (m *SnowflakeSequencer) SetMax(seenValue uint64) {
}

// return a new id as no Peek is stored
func (m *SnowflakeSequencer) Peek() uint64 {
	return uint64(m.node.Generate().Int64())
}
<|endoftext|>


<commit_before><commit_msg>server: remove the old way to initialize cluster ID (#1019)<commit_after><|endoftext|>


<commit_before><commit_msg>Read file into []byte instead of string. We can then use c.HTMLBlog() to serve the content.<commit_after><|endoftext|>


<commit_before>// +build k8s

/*
 * Copyright (C) 2018 IBM, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package tests

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/skydive-project/skydive/common"
	g "github.com/skydive-project/skydive/gremlin"
	"github.com/skydive-project/skydive/tests/helper"
)

func k8sConfigFile(name string) string {
	return "./k8s/" + name + ".yaml"
}

const (
	manager         = "k8s"
	objName         = "skydive-test"
	k8sRetry        = 10
	k8sDelaySeconds = 1
)

var (
	nodeName, _       = os.Hostname()
	podName           = objName
	containerName     = objName
	daemonSetName     = objName
	deploymentName    = objName
	ingressName       = objName
	jobName           = objName
	networkPolicyName = objName
	namespaceName     = objName
	replicaSetName    = objName
	serviceName       = objName
	clusterName       = "cluster"
)

func makeCmdWaitUntilStatus(ty, name, status string) string {
	return fmt.Sprintf("echo 'for i in {1..%d}; do sleep %d; kubectl get --all-namespaces %s %s %s break; done' | bash", k8sRetry, k8sDelaySeconds, ty, name, status)
}

func makeCmdWaitUntilCreated(ty, name string) string {
	return makeCmdWaitUntilStatus(ty, name, "&&")
}

func makeCmdWaitUntilDeleted(ty, name string) string {
	return makeCmdWaitUntilStatus(ty, name, "||")
}

func setupFromConfigFile(ty, name string) []helper.Cmd {
	return []helper.Cmd{
		{"kubectl create -f " + k8sConfigFile(ty), true},
		{makeCmdWaitUntilCreated(ty, name), true},
	}
}

func tearDownFromConfigFile(ty, name string) []helper.Cmd {
	return []helper.Cmd{
		{"kubectl delete -f " + k8sConfigFile(ty), false},
		{makeCmdWaitUntilDeleted(ty, name), true},
	}
}

func testNodeCreation(t *testing.T, setupCmds, tearDownCmds []helper.Cmd, typ, name string) {
	test := &Test{
		mode:         OneShot,
		retries:      3,
		setupCmds:    append(tearDownCmds, setupCmds...),
		tearDownCmds: tearDownCmds,
		checks: []CheckFunction{func(c *CheckContext) error {
			return common.Retry(func() error {
				query := g.G.V().Has("Manager", "k8s", "Type", typ, "Name", name)
				t.Log("Gremlin Query: " + query)

				nodes, err := c.gh.GetNodes(query.String())
				if err != nil {
					return err
				}

				if len(nodes) != 1 {
					return fmt.Errorf("Ran '%s', expected 1 node, got %d nodes: %+v", query, len(nodes), nodes)
				}

				return nil
			}, k8sRetry, k8sDelaySeconds*time.Second)
		}},
	}
	RunTest(t, test)
}

func testNodeCreationFromConfig(t *testing.T, typ, name string) {
	setup := setupFromConfigFile(typ, name)
	tearDown := tearDownFromConfigFile(typ, name)
	testNodeCreation(t, setup, tearDown, typ, name)
}

func TestK8sClusterNode(t *testing.T) {
	testNodeCreation(t, nil, nil, "cluster", clusterName)
}

func TestK8sContainerNode(t *testing.T) {
	testNodeCreationFromConfig(t, "container", containerName)
}

func TestK8sDeploymentNode(t *testing.T) {
	testNodeCreationFromConfig(t, "deployment", deploymentName)
}

func TestK8sIngressNode(t *testing.T) {
	testNodeCreationFromConfig(t, "ingress", ingressName)
}

func TestK8sJobNode(t *testing.T) {
	testNodeCreationFromConfig(t, "job", jobName)
}

func TestK8sNamespaceNode(t *testing.T) {
	testNodeCreationFromConfig(t, "namespace", namespaceName)
}

func TestK8sDaemonSetNode(t *testing.T) {
	testNodeCreationFromConfig(t, "daemonset", daemonSetName)
}

func TestK8sNetworkPolicyNode(t *testing.T) {
	testNodeCreationFromConfig(t, "networkpolicy", networkPolicyName)
}

func TestK8sNodeNode(t *testing.T) {
	testNodeCreation(t, nil, nil, "node", nodeName)
}

func TestK8sPersistentVolumeNode(t *testing.T) {
	testNodeCreationFromConfig(t, "persistentvolume", objName+"-persistentvolume")
}

func TestK8sPersistentVolumeClaimNode(t *testing.T) {
	testNodeCreationFromConfig(t, "persistentvolumeclaim", objName+"-persistentvolumeclaim")
}

func TestK8sPodNode(t *testing.T) {
	testNodeCreationFromConfig(t, "pod", podName)
}

func TestK8sReplicaSetNode(t *testing.T) {
	testNodeCreationFromConfig(t, "replicaset", replicaSetName)
}

func TestK8sReplicationControllerNode(t *testing.T) {
	testNodeCreationFromConfig(t, "replicationcontroller", objName+"-replicationcontroller")
}

func TestK8sServiceNode(t *testing.T) {
	testNodeCreationFromConfig(t, "service", serviceName)
}

func TestK8sStatefulSetNode(t *testing.T) {
	testNodeCreationFromConfig(t, "statefulset", "web")
}
<commit_msg>k8s: added test scenario hello-node<commit_after>// +build k8s

/*
 * Copyright (C) 2018 IBM, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package tests

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/skydive-project/skydive/common"
	g "github.com/skydive-project/skydive/gremlin"
	"github.com/skydive-project/skydive/tests/helper"
	"github.com/skydive-project/skydive/topology/graph"
)

func k8sConfigFile(name string) string {
	return "./k8s/" + name + ".yaml"
}

const (
	manager  = "k8s"
	objName  = "skydive-test"
	k8sRetry = 10
	k8sDelay = time.Second
)

var (
	nodeName, _       = os.Hostname()
	podName           = objName
	containerName     = objName
	daemonSetName     = objName
	deploymentName    = objName
	ingressName       = objName
	jobName           = objName
	networkPolicyName = objName
	namespaceName     = objName
	replicaSetName    = objName
	serviceName       = objName
	clusterName       = "cluster"
)

func setupFromConfigFile(ty, name string) []helper.Cmd {
	return []helper.Cmd{
		{"kubectl create -f " + k8sConfigFile(ty), true},
	}
}

func tearDownFromConfigFile(ty, name string) []helper.Cmd {
	return []helper.Cmd{
		{"kubectl delete --grace-period=0 --force -f " + k8sConfigFile(ty), false},
	}
}

func makeHasArgsType(ty interface{}, args1 ...interface{}) []interface{} {
	args := []interface{}{"Manager", "k8s", "Type", ty}
	args = append(args, args1...)
	return args
}

func makeHasArgsNode(node *graph.Node, args1 ...interface{}) []interface{} {
	m := node.Metadata()
	args := []interface{}{"Namespace", m["Namespace"], "Name", m["Name"]}
	args = append(args, args1...)
	return makeHasArgsType(m["Type"], args...)
}

func queryNodeCreation(t *testing.T, c *CheckContext, query g.QueryString) (node *graph.Node, err error) {
	err = common.Retry(func() error {
		const expectedNumNodes = 1

		t.Logf("Gremlin Query: %s", query)
		nodes, e := c.gh.GetNodes(query.String())
		if e != nil {
			return e
		}

		if len(nodes) != expectedNumNodes {
			return fmt.Errorf("Ran '%s', expected %d node, got %d nodes: %+v", query, expectedNumNodes, len(nodes), nodes)
		}

		if expectedNumNodes > 0 {
			node = nodes[0]
		}
		return nil
	}, k8sRetry, k8sDelay)
	return
}

func checkNodeCreation(t *testing.T, c *CheckContext, ty string, values ...interface{}) (*graph.Node, error) {
	args := makeHasArgsType(ty, values...)
	query := g.G.V().Has(args...)
	return queryNodeCreation(t, c, query)
}

func checkEdgeCreation(t *testing.T, c *CheckContext, from, to *graph.Node, relType string) error {
	fromArgs := makeHasArgsNode(from)
	toArgs := makeHasArgsNode(to)
	query := g.G.V().Has(fromArgs...).OutE().Has("RelationType", relType).OutV().Has(toArgs...)
	_, err := queryNodeCreation(t, c, query)
	return err
}

/* -- test creation of single resource -- */
func testNodeCreation(t *testing.T, setupCmds, tearDownCmds []helper.Cmd, typ, name string) {
	test := &Test{
		mode:         OneShot,
		retries:      3,
		setupCmds:    append(tearDownCmds, setupCmds...),
		tearDownCmds: tearDownCmds,
		checks: []CheckFunction{func(c *CheckContext) error {
			_, err := checkNodeCreation(t, c, typ, "Name", name)
			return err
		}},
	}
	RunTest(t, test)
}

func testNodeCreationFromConfig(t *testing.T, typ, name string) {
	setup := setupFromConfigFile(typ, name)
	tearDown := tearDownFromConfigFile(typ, name)
	testNodeCreation(t, setup, tearDown, typ, name)
}

func TestK8sClusterNode(t *testing.T) {
	testNodeCreation(t, nil, nil, "cluster", clusterName)
}

func TestK8sContainerNode(t *testing.T) {
	testNodeCreationFromConfig(t, "container", containerName)
}

func TestK8sDeploymentNode(t *testing.T) {
	testNodeCreationFromConfig(t, "deployment", deploymentName)
}

func TestK8sIngressNode(t *testing.T) {
	testNodeCreationFromConfig(t, "ingress", ingressName)
}

func TestK8sJobNode(t *testing.T) {
	testNodeCreationFromConfig(t, "job", jobName)
}

func TestK8sNamespaceNode(t *testing.T) {
	testNodeCreationFromConfig(t, "namespace", namespaceName)
}

func TestK8sDaemonSetNode(t *testing.T) {
	testNodeCreationFromConfig(t, "daemonset", daemonSetName)
}

func TestK8sNetworkPolicyNode(t *testing.T) {
	testNodeCreationFromConfig(t, "networkpolicy", networkPolicyName)
}

func TestK8sNodeNode(t *testing.T) {
	testNodeCreation(t, nil, nil, "node", nodeName)
}

func TestK8sPersistentVolumeNode(t *testing.T) {
	testNodeCreationFromConfig(t, "persistentvolume", objName+"-persistentvolume")
}

func TestK8sPersistentVolumeClaimNode(t *testing.T) {
	testNodeCreationFromConfig(t, "persistentvolumeclaim", objName+"-persistentvolumeclaim")
}

func TestK8sPodNode(t *testing.T) {
	testNodeCreationFromConfig(t, "pod", podName)
}

func TestK8sReplicaSetNode(t *testing.T) {
	testNodeCreationFromConfig(t, "replicaset", replicaSetName)
}

func TestK8sReplicationControllerNode(t *testing.T) {
	testNodeCreationFromConfig(t, "replicationcontroller", objName+"-replicationcontroller")
}

func TestK8sServiceNode(t *testing.T) {
	testNodeCreationFromConfig(t, "service", serviceName)
}

func TestK8sStatefulSetNode(t *testing.T) {
	testNodeCreationFromConfig(t, "statefulset", "web")
}

/* -- test multi-node scenarios -- */
func testScenario(t *testing.T, setupCmds, tearDownCmds []helper.Cmd, checks []CheckFunction) {
	test := &Test{
		mode:         OneShot,
		retries:      3,
		setupCmds:    append(tearDownCmds, setupCmds...),
		tearDownCmds: tearDownCmds,
		checks:       checks,
	}
	RunTest(t, test)
}

func TestHelloNodeScenario(t *testing.T) {
	testScenario(
		t,
		[]helper.Cmd{
			{"kubectl run hello-node --image=hello-node:v1 --port=8080", true},
		},
		[]helper.Cmd{
			{"kubectl delete --grace-period=0 --force deploy hello-node", false},
		},
		[]CheckFunction{
			func(c *CheckContext) error {
				// check nodes exist
				cluster, err := checkNodeCreation(t, c, "cluster")
				if err != nil {
					return err
				}

				container, err := checkNodeCreation(t, c, "container", "Name", "hello-node")
				if err != nil {
					return err
				}

				deployment, err := checkNodeCreation(t, c, "deployment", "Name", "hello-node")
				if err != nil {
					return err
				}

				namespace, err := checkNodeCreation(t, c, "namespace", "Name", "default")
				if err != nil {
					return err
				}

				pod, err := checkNodeCreation(t, c, "pod", "Name", g.Regex("%s-.*", "hello-node"))
				if err != nil {
					return err
				}

				// check edges exist
				if err = checkEdgeCreation(t, c, cluster, namespace, "ownership"); err != nil {
					return err
				}

				if err = checkEdgeCreation(t, c, namespace, deployment, "ownership"); err != nil {
					return err
				}

				if err = checkEdgeCreation(t, c, namespace, pod, "ownership"); err != nil {
					return err
				}

				if err = checkEdgeCreation(t, c, pod, container, "ownership"); err != nil {
					return err
				}
				return nil
			},
		},
	)
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.10.4"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
var VersionPrerelease = "dev"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<commit_msg>v0.10.4<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.10.4"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
var VersionPrerelease = ""

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<|endoftext|>


<commit_before>package stats_test

import (
	"math"
	"testing"
	"github.com/mbu/stats"
	"github.com/shirou/gopsutil/mem"
	"github.com/shirou/gopsutil/disk"
	"github.com/stretchr/testify/assert"
)

func TestGetAverage_OK(t *testing.T) {

	float64Array := []float64{ 11.904762, 27.5, 5, 2.5, 7.692308, 31.707317, 2.4390242, 7.317073 }

	expected := float64(12.007560525)
	actual := stats.GetAverage(float64Array)
	difference := math.Abs(float64(expected - actual))

	if (difference >= 1) {
		t.Error("Expected a difference less than 1, got: ", difference)
	}
}

func TestGetAverage_NOK_emptyArray(t *testing.T) {

	float64Array := []float64{}

	expected := float64(0)
	actual := stats.GetAverage(float64Array)
	assert.Equal(t, expected, actual)
}

func TestBuildCPUMultipleStat_OK(t *testing.T) {

	float64Array := []float64{ 11.904762, 27.5, 5, 2.5, 7.692308, 31.707317, 2.4390242, 7.317073 }
	cpu := stats.BuildCPUMultipleStat(float64Array)

	assert := assert.New(t)
	assert.NotNil(cpu)
	assert.Equal(stats.GetAverage(float64Array), cpu.AverageUsagePercentage)
	assert.Equal(len(float64Array), len(cpu.UsagePercentagePerCore))
}

func TestBuildCPUMultipleStat_NOK_emptyArray(t *testing.T) {

	float64Array := []float64{}
	cpu := stats.BuildCPUMultipleStat(float64Array)

	assert := assert.New(t)
	assert.NotNil(cpu)
	assert.Equal(stats.GetAverage(float64Array), cpu.AverageUsagePercentage)
	assert.Equal(float64(0), cpu.AverageUsagePercentage)
	assert.Equal(len(float64Array), len(cpu.UsagePercentagePerCore))
	assert.Equal(0, len(cpu.UsagePercentagePerCore))
}

func TestBuildRamStat_OK(t *testing.T) {

	zero := uint64(0)
	total := uint64(100000000000)
	used := uint64(77000000000)
	available := uint64(23000000000)
	usagePercentage := float64(77.00)

	vm := mem.VirtualMemoryStat{ total, available, used, usagePercentage, zero, zero, zero, zero, zero, zero, zero }
	ram := stats.BuildRamStat(vm)

	assert := assert.New(t)
	assert.NotNil(ram)
	assert.Equal(float64(total)/stats.GB, ram.TotalGB)
	assert.Equal(float64(used)/stats.GB, ram.UsedGB)
	assert.Equal(float64(available)/stats.GB, ram.AvailableGB)
	assert.Equal(float64(usagePercentage), ram.UsagePercentage)
}

func TestBuildRamStat_NOK_zeroedRAM(t *testing.T) {

	zero := uint64(0)
	total := uint64(0)
	used := uint64(0)
	available := uint64(0)
	usagePercentage := float64(0.00)

	vm := mem.VirtualMemoryStat{ total, available, used, usagePercentage, zero, zero, zero, zero, zero, zero, zero }
	ram := stats.BuildRamStat(vm)

	assert := assert.New(t)
	assert.NotNil(ram)
	assert.Equal(float64(total)/stats.GB, ram.TotalGB)
	assert.Equal(float64(used)/stats.GB, ram.UsedGB)
	assert.Equal(float64(available)/stats.GB, ram.AvailableGB)
	assert.Equal(float64(usagePercentage), ram.UsagePercentage)
}

func TestBuildDiskStat_OK(t *testing.T) {

	stringValue := ""
	total := uint64(975979800000)
	free := uint64(931929260000)
	used := uint64(44050570000)
	usagePercentage := float64(4.5134716)
	zeroInt := uint64(0)
	zeroFloat := float64(0)

	diskUsage := disk.DiskUsageStat{ stringValue, stringValue, total, free, used, usagePercentage, zeroInt, zeroInt, zeroInt, zeroFloat }
	disk := stats.BuildDiskStat(diskUsage)

	assert := assert.New(t)
	assert.NotNil(disk)
	assert.Equal(float64(total)/stats.GB, disk.TotalGB)
	assert.Equal(float64(used)/stats.GB, disk.UsedGB)
	assert.Equal(float64(free)/stats.GB, disk.AvailableGB)
	assert.Equal(float64(usagePercentage), disk.UsagePercentage)
}

func TestBuildDiskStat_NOK_zeroedDisk(t *testing.T) {

	stringValue := ""
	total := uint64(0)
	free := uint64(0)
	used := uint64(0)
	usagePercentage := float64(0)
	zeroInt := uint64(0)
	zeroFloat := float64(0)

	diskUsage := disk.DiskUsageStat{ stringValue, stringValue, total, free, used, usagePercentage, zeroInt, zeroInt, zeroInt, zeroFloat }
	disk := stats.BuildDiskStat(diskUsage)

	assert := assert.New(t)
	assert.NotNil(disk)
	assert.Equal(float64(total)/stats.GB, disk.TotalGB)
	assert.Equal(float64(used)/stats.GB, disk.UsedGB)
	assert.Equal(float64(free)/stats.GB, disk.AvailableGB)
	assert.Equal(float64(usagePercentage), disk.UsagePercentage)
}
<commit_msg>Add number of cores to factory tests.<commit_after>package stats_test

import (
	"math"
	"testing"
	"github.com/mbu/stats"
	"github.com/shirou/gopsutil/mem"
	"github.com/shirou/gopsutil/disk"
	"github.com/stretchr/testify/assert"
)

func TestGetAverage_OK(t *testing.T) {

	float64Array := []float64{ 11.904762, 27.5, 5, 2.5, 7.692308, 31.707317, 2.4390242, 7.317073 }

	expected := float64(12.007560525)
	actual := stats.GetAverage(float64Array)
	difference := math.Abs(float64(expected - actual))

	if (difference >= 1) {
		t.Error("Expected a difference less than 1, got: ", difference)
	}
}

func TestGetAverage_NOK_emptyArray(t *testing.T) {

	float64Array := []float64{}

	expected := float64(0)
	actual := stats.GetAverage(float64Array)
	assert.Equal(t, expected, actual)
}

func TestBuildCPUMultipleStat_OK(t *testing.T) {

	float64Array := []float64{ 11.904762, 27.5, 5, 2.5, 7.692308, 31.707317, 2.4390242, 7.317073 }
	cpu := stats.BuildCPUMultipleStat(float64Array)

	assert := assert.New(t)
	assert.NotNil(cpu)
	assert.Equal(len(float64Array), cpu.NumberOfCores)
	assert.Equal(stats.GetAverage(float64Array), cpu.AverageUsagePercentage)
	assert.Equal(len(float64Array), len(cpu.UsagePercentagePerCore))
}

func TestBuildCPUMultipleStat_NOK_emptyArray(t *testing.T) {

	float64Array := []float64{}
	cpu := stats.BuildCPUMultipleStat(float64Array)

	assert := assert.New(t)
	assert.NotNil(cpu)
	assert.Equal(len(float64Array), cpu.NumberOfCores)
	assert.Equal(stats.GetAverage(float64Array), cpu.AverageUsagePercentage)
	assert.Equal(float64(0), cpu.AverageUsagePercentage)
	assert.Equal(len(float64Array), len(cpu.UsagePercentagePerCore))
	assert.Equal(0, len(cpu.UsagePercentagePerCore))
}

func TestBuildRamStat_OK(t *testing.T) {

	zero := uint64(0)
	total := uint64(100000000000)
	used := uint64(77000000000)
	available := uint64(23000000000)
	usagePercentage := float64(77.00)

	vm := mem.VirtualMemoryStat{ total, available, used, usagePercentage, zero, zero, zero, zero, zero, zero, zero }
	ram := stats.BuildRamStat(vm)

	assert := assert.New(t)
	assert.NotNil(ram)
	assert.Equal(float64(total)/stats.GB, ram.TotalGB)
	assert.Equal(float64(used)/stats.GB, ram.UsedGB)
	assert.Equal(float64(available)/stats.GB, ram.AvailableGB)
	assert.Equal(float64(usagePercentage), ram.UsagePercentage)
}

func TestBuildRamStat_NOK_zeroedRAM(t *testing.T) {

	zero := uint64(0)
	total := uint64(0)
	used := uint64(0)
	available := uint64(0)
	usagePercentage := float64(0.00)

	vm := mem.VirtualMemoryStat{ total, available, used, usagePercentage, zero, zero, zero, zero, zero, zero, zero }
	ram := stats.BuildRamStat(vm)

	assert := assert.New(t)
	assert.NotNil(ram)
	assert.Equal(float64(total)/stats.GB, ram.TotalGB)
	assert.Equal(float64(used)/stats.GB, ram.UsedGB)
	assert.Equal(float64(available)/stats.GB, ram.AvailableGB)
	assert.Equal(float64(usagePercentage), ram.UsagePercentage)
}

func TestBuildDiskStat_OK(t *testing.T) {

	stringValue := ""
	total := uint64(975979800000)
	free := uint64(931929260000)
	used := uint64(44050570000)
	usagePercentage := float64(4.5134716)
	zeroInt := uint64(0)
	zeroFloat := float64(0)

	diskUsage := disk.DiskUsageStat{ stringValue, stringValue, total, free, used, usagePercentage, zeroInt, zeroInt, zeroInt, zeroFloat }
	disk := stats.BuildDiskStat(diskUsage)

	assert := assert.New(t)
	assert.NotNil(disk)
	assert.Equal(float64(total)/stats.GB, disk.TotalGB)
	assert.Equal(float64(used)/stats.GB, disk.UsedGB)
	assert.Equal(float64(free)/stats.GB, disk.AvailableGB)
	assert.Equal(float64(usagePercentage), disk.UsagePercentage)
}

func TestBuildDiskStat_NOK_zeroedDisk(t *testing.T) {

	stringValue := ""
	total := uint64(0)
	free := uint64(0)
	used := uint64(0)
	usagePercentage := float64(0)
	zeroInt := uint64(0)
	zeroFloat := float64(0)

	diskUsage := disk.DiskUsageStat{ stringValue, stringValue, total, free, used, usagePercentage, zeroInt, zeroInt, zeroInt, zeroFloat }
	disk := stats.BuildDiskStat(diskUsage)

	assert := assert.New(t)
	assert.NotNil(disk)
	assert.Equal(float64(total)/stats.GB, disk.TotalGB)
	assert.Equal(float64(used)/stats.GB, disk.UsedGB)
	assert.Equal(float64(free)/stats.GB, disk.AvailableGB)
	assert.Equal(float64(usagePercentage), disk.UsagePercentage)
}
<|endoftext|>


<commit_before>package auth

import (
	"crypto/sha1"
	"encoding/base64"
	"encoding/csv"
	"net/http"
	"os"
	"regexp"
	"strings"

	"golang.org/x/crypto/bcrypt"
)

var (
	shaRe = regexp.MustCompile(`^{SHA}`)
	bcrRe = regexp.MustCompile(`^\$2b\$|^\$2a\$|^\$2y\$`)
)

// HtpasswdFile is a map for usernames to passwords.
type HtpasswdFile struct {
	path  string
	users map[string]string
}

// NewHtpasswdFromFile reads the users and passwords from a htpasswd file and returns them.
func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
	r, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	cr := csv.NewReader(r)
	cr.Comma = ':'
	cr.Comment = '#'
	cr.TrimLeadingSpace = true

	records, err := cr.ReadAll()
	if err != nil {
		return nil, err
	}

	users := make(map[string]string)
	for _, record := range records {
		users[record[0]] = record[1]
	}

	return &HtpasswdFile{
		path:  path,
		users: users,
	}, nil
}

// Validate HTTP request credentials
func (h *HtpasswdFile) Validate(r *http.Request) bool {
	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 {
		return false
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false
	}

	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false
	}

	return h.validateCredentials(pair[0], pair[1])
}

func (h *HtpasswdFile) validateCredentials(user string, password string) bool {
	pwd, exists := h.users[user]
	if !exists {
		return false
	}

	switch {
	case shaRe.MatchString(pwd):
		d := sha1.New()
		_, _ = d.Write([]byte(password))
		if pwd[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
			return true
		}
	case bcrRe.MatchString(pwd):
		err := bcrypt.CompareHashAndPassword([]byte(pwd), []byte(password))
		if err == nil {
			return true
		}
	}
	return false
}
<commit_msg>feat(auth): simplify validate method<commit_after>package auth

import (
	"crypto/sha1"
	"encoding/base64"
	"encoding/csv"
	"net/http"
	"os"
	"regexp"

	"golang.org/x/crypto/bcrypt"
)

var (
	shaRe = regexp.MustCompile(`^{SHA}`)
	bcrRe = regexp.MustCompile(`^\$2b\$|^\$2a\$|^\$2y\$`)
)

// HtpasswdFile is a map for usernames to passwords.
type HtpasswdFile struct {
	path  string
	users map[string]string
}

// NewHtpasswdFromFile reads the users and passwords from a htpasswd file and returns them.
func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
	r, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	cr := csv.NewReader(r)
	cr.Comma = ':'
	cr.Comment = '#'
	cr.TrimLeadingSpace = true

	records, err := cr.ReadAll()
	if err != nil {
		return nil, err
	}

	users := make(map[string]string)
	for _, record := range records {
		users[record[0]] = record[1]
	}

	return &HtpasswdFile{
		path:  path,
		users: users,
	}, nil
}

// Validate HTTP request credentials
func (h *HtpasswdFile) Validate(r *http.Request) bool {
	user, passwd, ok := r.BasicAuth()
	if !ok {
		return false
	}
	return h.validateCredentials(user, passwd)
}

func (h *HtpasswdFile) validateCredentials(user string, password string) bool {
	pwd, exists := h.users[user]
	if !exists {
		return false
	}

	switch {
	case shaRe.MatchString(pwd):
		d := sha1.New()
		_, _ = d.Write([]byte(password))
		if pwd[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
			return true
		}
	case bcrRe.MatchString(pwd):
		err := bcrypt.CompareHashAndPassword([]byte(pwd), []byte(password))
		if err == nil {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Paolo Galeone. All right reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tfgo

import (
	"fmt"
	tf "github.com/tensorflow/tensorflow/tensorflow/go"
	"io/ioutil"
)

// Model represents a trained model
type Model struct {
	saved *tf.SavedModel
}

// LoadModel creates a new *Model, loading it from the exportDir.
// The graph loaded is identified by the set of tags specified when exporting it.
// This operation creates a session with specified `options`
// Panics if the model can't be loaded
func LoadModel(exportDir string, tags []string, options *tf.SessionOptions) (model *Model) {
	var err error
	model = new(Model)
	model.saved, err = tf.LoadSavedModel(exportDir, tags, options)
	if err != nil {
		panic(err.Error())
	}
	return
}

// ImportModel creates a new *Model, loading the graph from the serialized representation.
// This operation creates a session with specified `options`
// Panics if the model can't be loaded
func ImportModel(serializedModel, prefix string, options *tf.SessionOptions) (model *Model) {
	model = new(Model)
	contents, err := ioutil.ReadFile(serializedModel)
	if err != nil {
		panic(err.Error())
	}

	graph := tf.NewGraph()
	if err := graph.Import(contents, prefix); err != nil {
		panic(err.Error())
	}

	session, err := tf.NewSession(graph, options)
	if err != nil {
		panic(err.Error())
	}

	model.saved = &tf.SavedModel{Session: session, Graph: graph}
	return
}

// Exec executes the nodes/tensors that must be present in the loaded model
// feedDict values to feed to placeholders (that must have been saved in the model definition)
// panics on error
func (model *Model) Exec(tensors []tf.Output, feedDict map[tf.Output]*tf.Tensor) (results []*tf.Tensor) {
	var err error
	if results, err = model.saved.Session.Run(feedDict, tensors, nil); err == nil {
		return results
	}
	panic(err)
}

// Op extracts the output in position idx of the tensor with the specified name from the model graph
func (model *Model) Op(name string, idx int) tf.Output {
	op := model.saved.Graph.Operation(name)
	if op == nil {
		panic(fmt.Errorf("op %s not found", name))
	}
	nout := op.NumOutputs()
	if nout <= idx {
		panic(fmt.Errorf("op %s has %d outputs. Requested output number %d", name, nout, idx))
	}
	return op.Output(idx)
}

// EstimatorServe runs the inference on the model, exported as an estimator for serving.
// The data can be in numpy or pandas format, e.g.
// Pandas: { "a": 6.4, "b": 3.2, "c": 4.5,  "d": 1.5 }
// Numpy: { "inputs": [6.4, 3.2, 4.5, 1.5] }
// For pandas you have to wrap your values into an array, e.g: { "a": [6.4], "b": [3.2], ...}.
// Internally it uses preprocessor.PythonDictToByteArray.
func (model *Model) EstimatorServe(tensors []tf.Output, input *tf.Tensor) (results []*tf.Tensor) {
	return model.Exec(tensors, map[tf.Output]*tf.Tensor{
		model.Op("input_example_tensor", 0): input})
}
<commit_msg>Update documentation<commit_after>/*
Copyright 2017 Paolo Galeone. All right reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tfgo

import (
	"fmt"
	tf "github.com/tensorflow/tensorflow/tensorflow/go"
	"io/ioutil"
)

// Model represents a trained model
type Model struct {
	saved *tf.SavedModel
}

// LoadModel creates a new *Model, loading it from the exportDir.
// The graph loaded is identified by the set of tags specified when exporting it.
// This operation creates a session with specified `options`
// Panics if the model can't be loaded
func LoadModel(exportDir string, tags []string, options *tf.SessionOptions) (model *Model) {
	var err error
	model = new(Model)
	model.saved, err = tf.LoadSavedModel(exportDir, tags, options)
	if err != nil {
		panic(err.Error())
	}
	return
}

// ImportModel creates a new *Model, loading the graph from the serialized representation.
// This operation creates a session with specified `options`
// Panics if the model can't be loaded
func ImportModel(serializedModel, prefix string, options *tf.SessionOptions) (model *Model) {
	model = new(Model)
	contents, err := ioutil.ReadFile(serializedModel)
	if err != nil {
		panic(err.Error())
	}

	graph := tf.NewGraph()
	if err := graph.Import(contents, prefix); err != nil {
		panic(err.Error())
	}

	session, err := tf.NewSession(graph, options)
	if err != nil {
		panic(err.Error())
	}

	model.saved = &tf.SavedModel{Session: session, Graph: graph}
	return
}

// Exec executes the nodes/tensors that must be present in the loaded model
// feedDict values to feed to placeholders (that must have been saved in the model definition)
// panics on error
func (model *Model) Exec(tensors []tf.Output, feedDict map[tf.Output]*tf.Tensor) (results []*tf.Tensor) {
	var err error
	if results, err = model.saved.Session.Run(feedDict, tensors, nil); err == nil {
		return results
	}
	panic(err)
}

// Op extracts the output in position idx of the tensor with the specified name from the model graph
func (model *Model) Op(name string, idx int) tf.Output {
	op := model.saved.Graph.Operation(name)
	if op == nil {
		panic(fmt.Errorf("op %s not found", name))
	}
	nout := op.NumOutputs()
	if nout <= idx {
		panic(fmt.Errorf("op %s has %d outputs. Requested output number %d", name, nout, idx))
	}
	return op.Output(idx)
}

// EstimatorServe runs the inference on the model, exported as an estimator for serving.
// The data can be in numpy or pandas format, e.g.
// Pandas: { "a": 6.4, "b": 3.2, "c": 4.5,  "d": 1.5 }
// Numpy: { "inputs": [6.4, 3.2, 4.5, 1.5] }
// For pandas you have to wrap your values into an array, e.g: { "a": [6.4], "b": [3.2], ...}.
// After that, use preprocessor.PythonDictToByteArray to create the correct input for this method.
func (model *Model) EstimatorServe(tensors []tf.Output, input *tf.Tensor) (results []*tf.Tensor) {
	return model.Exec(tensors, map[tf.Output]*tf.Tensor{
		model.Op("input_example_tensor", 0): input})
}
<|endoftext|>


<commit_before>package qbs

import (
	"bytes"
	"database/sql"
	"reflect"
	"strconv"
	"strings"
	"time"
)

type TableNamer interface {
	TableName() string
}

const QBS_COLTYPE_INT = "int"
const QBS_COLTYPE_BOOL = "boolean"
const QBS_COLTYPE_BIGINT = "bigint"
const QBS_COLTYPE_DOUBLE = "double"
const QBS_COLTYPE_TIME = "timestamp"
const QBS_COLTYPE_TEXT = "text"

//convert struct field name to column name.
var FieldNameToColumnName func(string) string = toSnake

//convert struct name to table name.
var StructNameToTableName func(string) string = toSnake

//onvert column name to struct field name.
var ColumnNameToFieldName func(string) string = snakeToUpperCamel

//convert table name to struct name.
var TableNameToStructName func(string) string = snakeToUpperCamel

// Index represents a table index and is returned via the Indexed interface.
type index struct {
	name    string
	columns []string
	unique  bool
}

// Indexes represents an array of indexes.
type Indexes []*index

type Indexed interface {
	Indexes(indexes *Indexes)
}

// Add adds an index
func (ix *Indexes) Add(columns ...string) {
	name := strings.Join(columns, "_")
	*ix = append(*ix, &index{name: name, columns: columns, unique: false})
}

// AddUnique adds an unique index
func (ix *Indexes) AddUnique(columns ...string) {
	name := strings.Join(columns, "_")
	*ix = append(*ix, &index{name: name, columns: columns, unique: true})
}

// ModelField represents a schema field of a parsed model.
type modelField struct {
	name      string // Column name
	camelName string
	value     interface{} // Value
	pk        bool
	notnull   bool
	index     bool
	unique    bool
	updated   bool
	created   bool
	size      int
	dfault    string
	fk        string
	join      string
	colType   string
	nullable  reflect.Kind
}

// Model represents a parsed schema interface{}.
type model struct {
	pk      *modelField
	table   string
	fields  []*modelField
	refs    map[string]*reference
	indexes Indexes
}

type reference struct {
	refKey     string
	model      *model
	foreignKey bool
}

func (model *model) columnsAndValues(forUpdate bool) ([]string, []interface{}) {
	columns := make([]string, 0, len(model.fields))
	values := make([]interface{}, 0, len(columns))
	for _, column := range model.fields {
		var include bool
		if forUpdate {
			include = column.value != nil && !column.pk
		} else {
			include = true
			if column.value == nil && column.nullable == reflect.Invalid {
				include = false
			} else if column.pk {
				if intValue, ok := column.value.(int64); ok {
					include = intValue != 0
				} else if strValue, ok := column.value.(string); ok {
					include = strValue != ""
				}
			}
		}
		if include {
			columns = append(columns, column.name)
			values = append(values, column.value)
		}
	}
	return columns, values
}

func (model *model) timeField(name string) *modelField {
	for _, v := range model.fields {
		if _, ok := v.value.(time.Time); ok {
			if name == "created" {
				if v.created {
					return v
				}
			} else if name == "updated" {
				if v.updated {
					return v
				}
			}
			if v.name == name {
				return v
			}
		}
	}
	return nil
}

func (model *model) pkZero() bool {
	if model.pk == nil {
		return true
	}
	switch model.pk.value.(type) {
	case string:
		return model.pk.value.(string) == ""
	case int8:
		return model.pk.value.(int8) == 0
	case int16:
		return model.pk.value.(int16) == 0
	case int32:
		return model.pk.value.(int32) == 0
	case int64:
		return model.pk.value.(int64) == 0
	case uint8:
		return model.pk.value.(uint8) == 0
	case uint16:
		return model.pk.value.(uint16) == 0
	case uint32:
		return model.pk.value.(uint32) == 0
	case uint64:
		return model.pk.value.(uint64) == 0
	}
	return true
}

func structPtrToModel(f interface{}, root bool, omitFields []string) *model {
	model := &model{
		pk:      nil,
		table:   tableName(f),
		fields:  []*modelField{},
		indexes: Indexes{},
	}
	structType := reflect.TypeOf(f).Elem()
	structValue := reflect.ValueOf(f).Elem()
	for i := 0; i < structType.NumField(); i++ {
		structField := structType.Field(i)
		omit := false
		for _, v := range omitFields {
			if v == structField.Name {
				omit = true
			}
		}
		if omit {
			continue
		}
		fieldValue := structValue.FieldByName(structField.Name)
		if !fieldValue.CanInterface() {
			continue
		}
		sqlTag := structField.Tag.Get("qbs")
		if sqlTag == "-" {
			continue
		}
		fieldIsNullable := false
		kind := structField.Type.Kind()
		switch kind {
		case reflect.Ptr:
			switch structField.Type.Elem().Kind() {
			case reflect.Bool, reflect.String, reflect.Int64, reflect.Float64:
				kind = structField.Type.Elem().Kind()
				fieldIsNullable = true
			default:
				continue
			}
		case reflect.Map:
			continue
		case reflect.Slice:
			elemKind := structField.Type.Elem().Kind()
			if elemKind != reflect.Uint8 {
				continue
			}
		}

		fd := new(modelField)
		parseTags(fd, sqlTag)
		fd.camelName = structField.Name
		fd.name = FieldNameToColumnName(structField.Name)
		if fieldIsNullable {
			fd.nullable = kind
			if fieldValue.IsNil() {
				fd.value = nil
			} else {
				fd.value = fieldValue.Elem().Interface()
			}
		} else {
			//not nullable case
			fd.value = fieldValue.Interface()
		}
		if _, ok := fd.value.(int64); ok && fd.camelName == "Id" {
			fd.pk = true
		}
		if fd.pk {
			model.pk = fd
		}

		model.fields = append(model.fields, fd)
		// fill in references map only in root model.
		if root {
			var fk, explicitJoin, implicitJoin bool
			var refName string
			if fd.fk != "" {
				refName = fd.fk
				fk = true
			} else if fd.join != "" {
				refName = fd.join
				explicitJoin = true
			}

			if len(fd.camelName) > 3 && strings.HasSuffix(fd.camelName, "Id") {
				fdValue := reflect.ValueOf(fd.value)
				if _, ok := fd.value.(sql.NullInt64); ok || fdValue.Kind() == reflect.Int64 {
					i := strings.LastIndex(fd.camelName, "Id")
					refName = fd.camelName[:i]
					implicitJoin = true
				}
			}

			if fk || explicitJoin || implicitJoin {
				omit := false
				for _, v := range omitFields {
					if v == refName {
						omit = true
					}
				}
				if field, ok := structType.FieldByName(refName); ok && !omit {
					fieldValue := structValue.FieldByName(refName)
					if fieldValue.Kind() == reflect.Ptr {
						model.indexes.Add(fd.name)
						if fieldValue.IsNil() {
							fieldValue.Set(reflect.New(field.Type.Elem()))
						}
						refModel := structPtrToModel(fieldValue.Interface(), false, nil)
						ref := new(reference)
						ref.foreignKey = fk
						ref.model = refModel
						ref.refKey = fd.name
						if model.refs == nil {
							model.refs = make(map[string]*reference)
						}
						model.refs[refName] = ref
					} else if !implicitJoin {
						panic("Referenced field is not pointer")
					}
				} else if !implicitJoin {
					panic("Can not find referenced field")
				}
			}
			if fd.unique {
				model.indexes.AddUnique(fd.name)
			} else if fd.index {
				model.indexes.Add(fd.name)
			}
		}
	}
	if root {
		if indexed, ok := f.(Indexed); ok {
			indexed.Indexes(&model.indexes)
		}
	}
	return model
}

func tableName(talbe interface{}) string {
	if t, ok := talbe.(string); ok {
		return t
	}
	t := reflect.TypeOf(talbe).Elem()
	for {
		c := false
		switch t.Kind() {
		case reflect.Array, reflect.Chan, reflect.Map, reflect.Ptr, reflect.Slice:
			t = t.Elem()
			c = true
		}
		if !c {
			break
		}
	}
	if tn, ok := talbe.(TableNamer); ok {
		return tn.TableName()
	}
	return StructNameToTableName(t.Name())
}

func parseTags(fd *modelField, s string) {
	if s == "" {
		return
	}
	c := strings.Split(s, ",")
	for _, v := range c {
		c2 := strings.Split(v, ":")
		if len(c2) == 2 {
			switch c2[0] {
			case "fk":
				fd.fk = c2[1]
			case "size":
				fd.size, _ = strconv.Atoi(c2[1])
			case "default":
				fd.dfault = c2[1]
			case "join":
				fd.join = c2[1]
			case "coltype":
				fd.colType = c2[1]
			default:
				panic(c2[0] + " tag syntax error")
			}
		} else {
			switch c2[0] {
			case "created":
				fd.created = true
			case "pk":
				fd.pk = true
			case "updated":
				fd.updated = true
			case "index":
				fd.index = true
			case "unique":
				fd.unique = true
			case "notnull":
				fd.notnull = true
			default:
				panic(c2[0] + " tag syntax error")
			}
		}
	}
	return
}

func toSnake(s string) string {
	buf := new(bytes.Buffer)
	for i := 0; i < len(s); i++ {
		c := s[i]
		if c >= 'A' && c <= 'Z' {
			if i > 0 {
				buf.WriteByte('_')
			}
			buf.WriteByte(c + 32)
		} else {
			buf.WriteByte(c)
		}
	}
	return buf.String()
}

func snakeToUpperCamel(s string) string {
	buf := new(bytes.Buffer)
	first := true
	for i := 0; i < len(s); i++ {
		c := s[i]
		if c >= 'a' && c <= 'z' && first {
			buf.WriteByte(c - 32)
			first = false
		} else if c == '_' {
			first = true
			continue
		} else {
			buf.WriteByte(c)
		}
	}
	return buf.String()
}

var ValidTags = map[string]bool{
	"pk":      true, //primary key
	"fk":      true, //foreign key
	"size":    true,
	"default": true,
	"join":    true,
	"-":       true, //ignore
	"index":   true,
	"unique":  true,
	"notnull": true,
	"updated": true,
	"created": true,
	"coltype": true,
}
<commit_msg>improve error message in a common case<commit_after>package qbs

import (
	"bytes"
	"database/sql"
	"reflect"
	"strconv"
	"strings"
	"time"
)

type TableNamer interface {
	TableName() string
}

const QBS_COLTYPE_INT = "int"
const QBS_COLTYPE_BOOL = "boolean"
const QBS_COLTYPE_BIGINT = "bigint"
const QBS_COLTYPE_DOUBLE = "double"
const QBS_COLTYPE_TIME = "timestamp"
const QBS_COLTYPE_TEXT = "text"

//convert struct field name to column name.
var FieldNameToColumnName func(string) string = toSnake

//convert struct name to table name.
var StructNameToTableName func(string) string = toSnake

//onvert column name to struct field name.
var ColumnNameToFieldName func(string) string = snakeToUpperCamel

//convert table name to struct name.
var TableNameToStructName func(string) string = snakeToUpperCamel

// Index represents a table index and is returned via the Indexed interface.
type index struct {
	name    string
	columns []string
	unique  bool
}

// Indexes represents an array of indexes.
type Indexes []*index

type Indexed interface {
	Indexes(indexes *Indexes)
}

// Add adds an index
func (ix *Indexes) Add(columns ...string) {
	name := strings.Join(columns, "_")
	*ix = append(*ix, &index{name: name, columns: columns, unique: false})
}

// AddUnique adds an unique index
func (ix *Indexes) AddUnique(columns ...string) {
	name := strings.Join(columns, "_")
	*ix = append(*ix, &index{name: name, columns: columns, unique: true})
}

// ModelField represents a schema field of a parsed model.
type modelField struct {
	name      string // Column name
	camelName string
	value     interface{} // Value
	pk        bool
	notnull   bool
	index     bool
	unique    bool
	updated   bool
	created   bool
	size      int
	dfault    string
	fk        string
	join      string
	colType   string
	nullable  reflect.Kind
}

// Model represents a parsed schema interface{}.
type model struct {
	pk      *modelField
	table   string
	fields  []*modelField
	refs    map[string]*reference
	indexes Indexes
}

type reference struct {
	refKey     string
	model      *model
	foreignKey bool
}

func (model *model) columnsAndValues(forUpdate bool) ([]string, []interface{}) {
	columns := make([]string, 0, len(model.fields))
	values := make([]interface{}, 0, len(columns))
	for _, column := range model.fields {
		var include bool
		if forUpdate {
			include = column.value != nil && !column.pk
		} else {
			include = true
			if column.value == nil && column.nullable == reflect.Invalid {
				include = false
			} else if column.pk {
				if intValue, ok := column.value.(int64); ok {
					include = intValue != 0
				} else if strValue, ok := column.value.(string); ok {
					include = strValue != ""
				}
			}
		}
		if include {
			columns = append(columns, column.name)
			values = append(values, column.value)
		}
	}
	return columns, values
}

func (model *model) timeField(name string) *modelField {
	for _, v := range model.fields {
		if _, ok := v.value.(time.Time); ok {
			if name == "created" {
				if v.created {
					return v
				}
			} else if name == "updated" {
				if v.updated {
					return v
				}
			}
			if v.name == name {
				return v
			}
		}
	}
	return nil
}

func (model *model) pkZero() bool {
	if model.pk == nil {
		return true
	}
	switch model.pk.value.(type) {
	case string:
		return model.pk.value.(string) == ""
	case int8:
		return model.pk.value.(int8) == 0
	case int16:
		return model.pk.value.(int16) == 0
	case int32:
		return model.pk.value.(int32) == 0
	case int64:
		return model.pk.value.(int64) == 0
	case uint8:
		return model.pk.value.(uint8) == 0
	case uint16:
		return model.pk.value.(uint16) == 0
	case uint32:
		return model.pk.value.(uint32) == 0
	case uint64:
		return model.pk.value.(uint64) == 0
	}
	return true
}

func structPtrToModel(f interface{}, root bool, omitFields []string) *model {
	model := &model{
		pk:      nil,
		table:   tableName(f),
		fields:  []*modelField{},
		indexes: Indexes{},
	}
	structType := reflect.TypeOf(f).Elem()
	structValue := reflect.ValueOf(f).Elem()
	if structType.Kind() == reflect.Ptr {
		if structType.Elem().Kind() == reflect.Struct {
			panic("did you pass a pointer to a pointer to a struct?")
		}
	}
	for i := 0; i < structType.NumField(); i++ {
		structField := structType.Field(i)
		omit := false
		for _, v := range omitFields {
			if v == structField.Name {
				omit = true
			}
		}
		if omit {
			continue
		}
		fieldValue := structValue.FieldByName(structField.Name)
		if !fieldValue.CanInterface() {
			continue
		}
		sqlTag := structField.Tag.Get("qbs")
		if sqlTag == "-" {
			continue
		}
		fieldIsNullable := false
		kind := structField.Type.Kind()
		switch kind {
		case reflect.Ptr:
			switch structField.Type.Elem().Kind() {
			case reflect.Bool, reflect.String, reflect.Int64, reflect.Float64:
				kind = structField.Type.Elem().Kind()
				fieldIsNullable = true
			default:
				continue
			}
		case reflect.Map:
			continue
		case reflect.Slice:
			elemKind := structField.Type.Elem().Kind()
			if elemKind != reflect.Uint8 {
				continue
			}
		}

		fd := new(modelField)
		parseTags(fd, sqlTag)
		fd.camelName = structField.Name
		fd.name = FieldNameToColumnName(structField.Name)
		if fieldIsNullable {
			fd.nullable = kind
			if fieldValue.IsNil() {
				fd.value = nil
			} else {
				fd.value = fieldValue.Elem().Interface()
			}
		} else {
			//not nullable case
			fd.value = fieldValue.Interface()
		}
		if _, ok := fd.value.(int64); ok && fd.camelName == "Id" {
			fd.pk = true
		}
		if fd.pk {
			model.pk = fd
		}

		model.fields = append(model.fields, fd)
		// fill in references map only in root model.
		if root {
			var fk, explicitJoin, implicitJoin bool
			var refName string
			if fd.fk != "" {
				refName = fd.fk
				fk = true
			} else if fd.join != "" {
				refName = fd.join
				explicitJoin = true
			}

			if len(fd.camelName) > 3 && strings.HasSuffix(fd.camelName, "Id") {
				fdValue := reflect.ValueOf(fd.value)
				if _, ok := fd.value.(sql.NullInt64); ok || fdValue.Kind() == reflect.Int64 {
					i := strings.LastIndex(fd.camelName, "Id")
					refName = fd.camelName[:i]
					implicitJoin = true
				}
			}

			if fk || explicitJoin || implicitJoin {
				omit := false
				for _, v := range omitFields {
					if v == refName {
						omit = true
					}
				}
				if field, ok := structType.FieldByName(refName); ok && !omit {
					fieldValue := structValue.FieldByName(refName)
					if fieldValue.Kind() == reflect.Ptr {
						model.indexes.Add(fd.name)
						if fieldValue.IsNil() {
							fieldValue.Set(reflect.New(field.Type.Elem()))
						}
						refModel := structPtrToModel(fieldValue.Interface(), false, nil)
						ref := new(reference)
						ref.foreignKey = fk
						ref.model = refModel
						ref.refKey = fd.name
						if model.refs == nil {
							model.refs = make(map[string]*reference)
						}
						model.refs[refName] = ref
					} else if !implicitJoin {
						panic("Referenced field is not pointer")
					}
				} else if !implicitJoin {
					panic("Can not find referenced field")
				}
			}
			if fd.unique {
				model.indexes.AddUnique(fd.name)
			} else if fd.index {
				model.indexes.Add(fd.name)
			}
		}
	}
	if root {
		if indexed, ok := f.(Indexed); ok {
			indexed.Indexes(&model.indexes)
		}
	}
	return model
}

func tableName(talbe interface{}) string {
	if t, ok := talbe.(string); ok {
		return t
	}
	t := reflect.TypeOf(talbe).Elem()
	for {
		c := false
		switch t.Kind() {
		case reflect.Array, reflect.Chan, reflect.Map, reflect.Ptr, reflect.Slice:
			t = t.Elem()
			c = true
		}
		if !c {
			break
		}
	}
	if tn, ok := talbe.(TableNamer); ok {
		return tn.TableName()
	}
	return StructNameToTableName(t.Name())
}

func parseTags(fd *modelField, s string) {
	if s == "" {
		return
	}
	c := strings.Split(s, ",")
	for _, v := range c {
		c2 := strings.Split(v, ":")
		if len(c2) == 2 {
			switch c2[0] {
			case "fk":
				fd.fk = c2[1]
			case "size":
				fd.size, _ = strconv.Atoi(c2[1])
			case "default":
				fd.dfault = c2[1]
			case "join":
				fd.join = c2[1]
			case "coltype":
				fd.colType = c2[1]
			default:
				panic(c2[0] + " tag syntax error")
			}
		} else {
			switch c2[0] {
			case "created":
				fd.created = true
			case "pk":
				fd.pk = true
			case "updated":
				fd.updated = true
			case "index":
				fd.index = true
			case "unique":
				fd.unique = true
			case "notnull":
				fd.notnull = true
			default:
				panic(c2[0] + " tag syntax error")
			}
		}
	}
	return
}

func toSnake(s string) string {
	buf := new(bytes.Buffer)
	for i := 0; i < len(s); i++ {
		c := s[i]
		if c >= 'A' && c <= 'Z' {
			if i > 0 {
				buf.WriteByte('_')
			}
			buf.WriteByte(c + 32)
		} else {
			buf.WriteByte(c)
		}
	}
	return buf.String()
}

func snakeToUpperCamel(s string) string {
	buf := new(bytes.Buffer)
	first := true
	for i := 0; i < len(s); i++ {
		c := s[i]
		if c >= 'a' && c <= 'z' && first {
			buf.WriteByte(c - 32)
			first = false
		} else if c == '_' {
			first = true
			continue
		} else {
			buf.WriteByte(c)
		}
	}
	return buf.String()
}

var ValidTags = map[string]bool{
	"pk":      true, //primary key
	"fk":      true, //foreign key
	"size":    true,
	"default": true,
	"join":    true,
	"-":       true, //ignore
	"index":   true,
	"unique":  true,
	"notnull": true,
	"updated": true,
	"created": true,
	"coltype": true,
}
<|endoftext|>