<commit_before>package controller

import (
	"crypto/md5"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/tidwall/tile38/controller/log"
	"github.com/tidwall/tile38/core"
)

const backwardsBufferSize = 50000

// checksum performs a simple md5 checksum on the aof file
func (c *Controller) checksum(pos, size int64) (sum string, err error) {
	if pos+size > int64(c.aofsz) {
		return "", io.EOF
	}
	var f *os.File
	f, err = os.Open(c.f.Name())
	if err != nil {
		return
	}
	defer f.Close()
	sumr := md5.New()
	err = func() error {
		if size == 0 {
			n, err := f.Seek(int64(c.aofsz), 0)
			if err != nil {
				return err
			}
			if pos >= n {
				return io.EOF
			}
			return nil
		}
		_, err = f.Seek(pos, 0)
		if err != nil {
			return err
		}
		_, err = io.CopyN(sumr, f, size)
		if err != nil {
			return err
		}
		return nil
	}()
	if err != nil {
		if err == io.ErrUnexpectedEOF {
			err = io.EOF
		}
		return "", err
	}
	return fmt.Sprintf("%x", sumr.Sum(nil)), nil
}

func connAOFMD5(conn *Conn, pos, size int64) (sum string, err error) {
	v, err := conn.Do("aofmd5", pos, size)
	if err != nil {
		return "", err
	}
	if v.Error() != nil {
		return "", v.Error()
	}
	sum = v.String()
	if len(sum) != 32 {
		return "", errors.New("checksum not ok")
	}
	return sum, nil
}

func (c *Controller) matchChecksums(conn *Conn, pos, size int64) (match bool, err error) {
	sum, err := c.checksum(pos, size)
	if err != nil {
		if err == io.EOF {
			return false, nil
		}
		return false, err
	}
	csum, err := connAOFMD5(conn, pos, size)
	if err != nil {
		if err == io.EOF {
			return false, nil
		}
		return false, err
	}
	return csum == sum, nil
}

// followCheckSome is not a full checksum. It just "checks some" data.
// We will do some various checksums on the leader until we find the correct position to start at.
func (c *Controller) followCheckSome(addr string, followc uint64) (pos int64, err error) {
	if core.ShowDebugMessages {
		log.Debug("follow:", addr, ":check some")
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.followc != followc {
		return 0, errNoLongerFollowing
	}
	if c.aofsz < checksumsz {
		return 0, nil
	}

	conn, err := DialTimeout(addr, time.Second*2)
	if err != nil {
		return 0, err
	}
	defer conn.Close()

	min := int64(0)
	max := int64(c.aofsz) - checksumsz
	limit := int64(c.aofsz)
	match, err := c.matchChecksums(conn, min, checksumsz)
	if err != nil {
		return 0, err
	}
	if match {
		min += checksumsz // bump up the min
		for {
			if max < min || max+checksumsz > limit {
				pos = min
				break
			} else {
				match, err = c.matchChecksums(conn, max, checksumsz)
				if err != nil {
					return 0, err
				}
				if match {
					min = max + checksumsz
				} else {
					limit = max
				}
				max = (limit-min)/2 - checksumsz/2 + min // multiply
			}
		}
	}
	fullpos := pos
	fname := c.f.Name()
	if pos == 0 {
		c.f.Close()
		c.f, err = os.Create(fname)
		if err != nil {
			log.Fatalf("could not recreate aof, possible data loss. %s", err.Error())
			return 0, err
		}
		return 0, nil
	}

	// we want to truncate at a command location
	// search for nearest command
	f, err := os.Open(c.f.Name())
	if err != nil {
		return 0, err
	}
	defer f.Close()
	if _, err := f.Seek(pos, 0); err != nil {
		return 0, err
	}
	// need to read backwards looking for null byte
	const bufsz = backwardsBufferSize
	buf := make([]byte, bufsz)
outer:
	for {
		if pos < int64(len(buf)) {
			pos = 0
			break
		}
		if _, err := f.Seek(pos-bufsz, 0); err != nil {
			return 0, err
		}
		if _, err := io.ReadFull(f, buf); err != nil {
			return 0, err
		}
		for i := len(buf) - 1; i >= 0; i-- {
			if buf[i] == 0 {
				tpos := pos - bufsz + int64(i) - 4
				if tpos < 0 {
					pos = 0
					break outer // at beginning of file
				}
				if _, err := f.Seek(tpos, 0); err != nil {
					return 0, err
				}
				szb := make([]byte, 4)
				if _, err := io.ReadFull(f, szb); err != nil {
					return 0, err
				}
				sz2 := int64(binary.LittleEndian.Uint32(szb))
				tpos = tpos - sz2 - 4
				if tpos < 0 {
					continue // keep scanning
				}
				if _, err := f.Seek(tpos, 0); err != nil {
					return 0, err
				}
				if _, err := io.ReadFull(f, szb); err != nil {
					return 0, err
				}
				sz1 := int64(binary.LittleEndian.Uint32(szb))
				if sz1 == sz2 {
					pos = pos - bufsz + int64(i) + 1
					break outer // we found our match
				}
			}
		}
		pos -= bufsz
	}
	if pos == fullpos {
		if core.ShowDebugMessages {
			log.Debug("follow: aof fully intact")
		}
		return pos, nil
	}
	log.Warnf("truncating aof to %d", pos)
	// any errror below are fatal.
	f.Close()
	c.f.Close()
	if err := os.Truncate(fname, pos); err != nil {
		log.Fatalf("could not truncate aof, possible data loss. %s", err.Error())
		return 0, err
	}
	c.f, err = os.OpenFile(fname, os.O_CREATE|os.O_RDWR, 0600)
	if err != nil {
		log.Fatalf("could not create aof, possible data loss. %s", err.Error())
		return 0, err
	}
	// reset the entire system.
	log.Infof("reloading aof commands")
	c.reset()
	if err := c.loadAOF(); err != nil {
		log.Fatalf("could not reload aof, possible data loss. %s", err.Error())
		return 0, err
	}
	if int64(c.aofsz) != pos {
		log.Fatalf("aof size mismatch during reload, possible data loss.")
		return 0, errors.New("?")
	}
	return pos, nil
}
<commit_msg>read backward multibulk<commit_after>package controller

import (
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/tidwall/resp"
	"github.com/tidwall/tile38/controller/log"
	"github.com/tidwall/tile38/core"
)

// checksum performs a simple md5 checksum on the aof file
func (c *Controller) checksum(pos, size int64) (sum string, err error) {
	if pos+size > int64(c.aofsz) {
		return "", io.EOF
	}
	var f *os.File
	f, err = os.Open(c.f.Name())
	if err != nil {
		return
	}
	defer f.Close()
	sumr := md5.New()
	err = func() error {
		if size == 0 {
			n, err := f.Seek(int64(c.aofsz), 0)
			if err != nil {
				return err
			}
			if pos >= n {
				return io.EOF
			}
			return nil
		}
		_, err = f.Seek(pos, 0)
		if err != nil {
			return err
		}
		_, err = io.CopyN(sumr, f, size)
		if err != nil {
			return err
		}
		return nil
	}()
	if err != nil {
		if err == io.ErrUnexpectedEOF {
			err = io.EOF
		}
		return "", err
	}
	return fmt.Sprintf("%x", sumr.Sum(nil)), nil
}

func connAOFMD5(conn *Conn, pos, size int64) (sum string, err error) {
	v, err := conn.Do("aofmd5", pos, size)
	if err != nil {
		return "", err
	}
	if v.Error() != nil {
		errmsg := v.Error().Error()
		if errmsg == "ERR EOF" || errmsg == "EOF" {
			return "", io.EOF
		}
		return "", v.Error()
	}
	sum = v.String()
	if len(sum) != 32 {
		return "", errors.New("checksum not ok")
	}
	return sum, nil
}

func (c *Controller) matchChecksums(conn *Conn, pos, size int64) (match bool, err error) {
	sum, err := c.checksum(pos, size)
	if err != nil {
		if err == io.EOF {
			return false, nil
		}
		return false, err
	}
	csum, err := connAOFMD5(conn, pos, size)
	if err != nil {
		if err == io.EOF {
			return false, nil
		}
		return false, err
	}
	return csum == sum, nil
}

// getEndOfLastValuePositionInFile is a very slow operation because it reads the file
// backwards on byte at a time. Eek. It seek+read, seek+read, etc.
func getEndOfLastValuePositionInFile(fname string, startPos int64) (int64, error) {
	pos := startPos
	f, err := os.Open(fname)
	if err != nil {
		return 0, err
	}
	defer f.Close()
	readByte := func() (byte, error) {
		if pos <= 0 {
			return 0, io.EOF
		}
		pos--
		if _, err := f.Seek(pos, 0); err != nil {
			return 0, err
		}
		b := make([]byte, 1)
		if n, err := f.Read(b); err != nil {
			return 0, err
		} else if n != 1 {
			return 0, errors.New("invalid read")
		}
		return b[0], nil
	}
	for {
		c, err := readByte()
		if err != nil {
			return 0, err
		}
		if c == '*' {
			if _, err := f.Seek(pos, 0); err != nil {
				return 0, err
			}
			rd := resp.NewReader(f)
			_, telnet, n, err := rd.ReadMultiBulk()
			if err != nil || telnet {
				continue // keep reading backwards
			}
			return pos + int64(n), nil
		}
	}
}

// followCheckSome is not a full checksum. It just "checks some" data.
// We will do some various checksums on the leader until we find the correct position to start at.
func (c *Controller) followCheckSome(addr string, followc uint64) (pos int64, err error) {
	if core.ShowDebugMessages {
		log.Debug("follow:", addr, ":check some")
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.followc != followc {
		return 0, errNoLongerFollowing
	}
	if c.aofsz < checksumsz {
		return 0, nil
	}

	conn, err := DialTimeout(addr, time.Second*2)
	if err != nil {
		return 0, err
	}
	defer conn.Close()

	min := int64(0)
	max := int64(c.aofsz) - checksumsz
	limit := int64(c.aofsz)
	match, err := c.matchChecksums(conn, min, checksumsz)
	if err != nil {
		return 0, err
	}

	if match {
		min += checksumsz // bump up the min
		for {
			if max < min || max+checksumsz > limit {
				pos = min
				break
			} else {
				match, err = c.matchChecksums(conn, max, checksumsz)
				if err != nil {
					return 0, err
				}
				if match {
					min = max + checksumsz
				} else {
					limit = max
				}
				max = (limit-min)/2 - checksumsz/2 + min // multiply
			}
		}
	}
	fullpos := pos
	fname := c.f.Name()
	if pos == 0 {
		c.f.Close()
		c.f, err = os.Create(fname)
		if err != nil {
			log.Fatalf("could not recreate aof, possible data loss. %s", err.Error())
			return 0, err
		}
		return 0, nil
	}

	// we want to truncate at a command location
	// search for nearest command
	pos, err = getEndOfLastValuePositionInFile(c.f.Name(), fullpos)
	if err != nil {
		return 0, err
	}
	if pos == fullpos {
		if core.ShowDebugMessages {
			log.Debug("follow: aof fully intact")
		}
		return pos, nil
	}
	log.Warnf("truncating aof to %d", pos)
	// any errror below are fatal.
	c.f.Close()
	if err := os.Truncate(fname, pos); err != nil {
		log.Fatalf("could not truncate aof, possible data loss. %s", err.Error())
		return 0, err
	}
	c.f, err = os.OpenFile(fname, os.O_CREATE|os.O_RDWR, 0600)
	if err != nil {
		log.Fatalf("could not create aof, possible data loss. %s", err.Error())
		return 0, err
	}
	// reset the entire system.
	log.Infof("reloading aof commands")
	c.reset()
	if err := c.loadAOF(); err != nil {
		log.Fatalf("could not reload aof, possible data loss. %s", err.Error())
		return 0, err
	}
	if int64(c.aofsz) != pos {
		log.Fatalf("aof size mismatch during reload, possible data loss.")
		return 0, errors.New("?")
	}
	return pos, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package yubikey

const (
	numberOfRounds = 10
)

var (
	rc           = []byte{0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36}
	rijndaelSbox = []byte{
		0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5,
		0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76,
		0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0,
		0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0,
		0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC,
		0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15,
		0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A,
		0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75,
		0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0,
		0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84,
		0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B,
		0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF,
		0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85,
		0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8,
		0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5,
		0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2,
		0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17,
		0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73,
		0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88,
		0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB,
		0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C,
		0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79,
		0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9,
		0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08,
		0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6,
		0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A,
		0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E,
		0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E,
		0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94,
		0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF,
		0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68,
		0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16,
	}
	rijndaelInvSbox = []byte{
		0x52, 0x09, 0x6A, 0xD5, 0x30, 0x36, 0xA5, 0x38,
		0xBF, 0x40, 0xA3, 0x9E, 0x81, 0xF3, 0xD7, 0xFB,
		0x7C, 0xE3, 0x39, 0x82, 0x9B, 0x2F, 0xFF, 0x87,
		0x34, 0x8E, 0x43, 0x44, 0xC4, 0xDE, 0xE9, 0xCB,
		0x54, 0x7B, 0x94, 0x32, 0xA6, 0xC2, 0x23, 0x3D,
		0xEE, 0x4C, 0x95, 0x0B, 0x42, 0xFA, 0xC3, 0x4E,
		0x08, 0x2E, 0xA1, 0x66, 0x28, 0xD9, 0x24, 0xB2,
		0x76, 0x5B, 0xA2, 0x49, 0x6D, 0x8B, 0xD1, 0x25,
		0x72, 0xF8, 0xF6, 0x64, 0x86, 0x68, 0x98, 0x16,
		0xD4, 0xA4, 0x5C, 0xCC, 0x5D, 0x65, 0xB6, 0x92,
		0x6C, 0x70, 0x48, 0x50, 0xFD, 0xED, 0xB9, 0xDA,
		0x5E, 0x15, 0x46, 0x57, 0xA7, 0x8D, 0x9D, 0x84,
		0x90, 0xD8, 0xAB, 0x00, 0x8C, 0xBC, 0xD3, 0x0A,
		0xF7, 0xE4, 0x58, 0x05, 0xB8, 0xB3, 0x45, 0x06,
		0xD0, 0x2C, 0x1E, 0x8F, 0xCA, 0x3F, 0x0F, 0x02,
		0xC1, 0xAF, 0xBD, 0x03, 0x01, 0x13, 0x8A, 0x6B,
		0x3A, 0x91, 0x11, 0x41, 0x4F, 0x67, 0xDC, 0xEA,
		0x97, 0xF2, 0xCF, 0xCE, 0xF0, 0xB4, 0xE6, 0x73,
		0x96, 0xAC, 0x74, 0x22, 0xE7, 0xAD, 0x35, 0x85,
		0xE2, 0xF9, 0x37, 0xE8, 0x1C, 0x75, 0xDF, 0x6E,
		0x47, 0xF1, 0x1A, 0x71, 0x1D, 0x29, 0xC5, 0x89,
		0x6F, 0xB7, 0x62, 0x0E, 0xAA, 0x18, 0xBE, 0x1B,
		0xFC, 0x56, 0x3E, 0x4B, 0xC6, 0xD2, 0x79, 0x20,
		0x9A, 0xDB, 0xC0, 0xFE, 0x78, 0xCD, 0x5A, 0xF4,
		0x1F, 0xDD, 0xA8, 0x33, 0x88, 0x07, 0xC7, 0x31,
		0xB1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xEC, 0x5F,
		0x60, 0x51, 0x7F, 0xA9, 0x19, 0xB5, 0x4A, 0x0D,
		0x2D, 0xE5, 0x7A, 0x9F, 0x93, 0xC9, 0x9C, 0xEF,
		0xA0, 0xE0, 0x3B, 0x4D, 0xAE, 0x2A, 0xF5, 0xB0,
		0xC8, 0xEB, 0xBB, 0x3C, 0x83, 0x53, 0x99, 0x61,
		0x17, 0x2B, 0x04, 0x7E, 0xBA, 0x77, 0xD6, 0x26,
		0xE1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0C, 0x7D,
	}
)

func AesDecrypt(src []byte, key Key) []byte {
	state := make([]byte, len(src))
	copy(state, src)

	round_key := key

	for i := 0; i < numberOfRounds; i++ {
		round_key[0] ^= rc[i]

		round_key[0] ^= rijndaelSbox[round_key[13]]
		round_key[1] ^= rijndaelSbox[round_key[14]]
		round_key[2] ^= rijndaelSbox[round_key[15]]
		round_key[3] ^= rijndaelSbox[round_key[12]]

		for j := 4; j < 16; j++ {
			round_key[j] ^= round_key[j-4]
		}
	}
	for i := 0; i < 0x10; i++ {
		state[i] ^= round_key[i]
	}
	for i := 1; i <= numberOfRounds; i++ {
		/* First row: 0 shift, 0 4 8 12 */
		state[0] = rijndaelInvSbox[state[0]]
		state[4] = rijndaelInvSbox[state[4]]
		state[8] = rijndaelInvSbox[state[8]]
		state[12] = rijndaelInvSbox[state[12]]

		/* Second row: -1 shift, 1 5 9 13 */
		j := state[13]
		state[13] = rijndaelInvSbox[state[9]]
		state[9] = rijndaelInvSbox[state[5]]
		state[5] = rijndaelInvSbox[state[1]]
		state[1] = rijndaelInvSbox[j]

		/* Third row: -2 shift, 2 6 10 14 */
		j = state[2]
		state[2] = rijndaelInvSbox[state[10]]
		state[10] = rijndaelInvSbox[j]
		j = state[6]
		state[6] = rijndaelInvSbox[state[14]]
		state[14] = rijndaelInvSbox[j]

		/* Fourth row: -3 shift, 3 7 11 15 */
		j = state[3]
		state[3] = rijndaelInvSbox[state[7]]
		state[7] = rijndaelInvSbox[state[11]]
		state[11] = rijndaelInvSbox[state[15]]
		state[15] = rijndaelInvSbox[j]

		for j := 15; j > 3; j-- {
			round_key[j] ^= round_key[j-4]
		}
		round_key[0] ^=
			(rc[numberOfRounds-i] ^ rijndaelSbox[round_key[13]])

		round_key[1] ^= rijndaelSbox[round_key[14]]
		round_key[2] ^= rijndaelSbox[round_key[15]]
		round_key[3] ^= rijndaelSbox[round_key[12]]

		for j := 0; j < 16; j++ {
			state[j] ^= round_key[j]
		}
		if i != numberOfRounds {
			for j = 0; j < 16; j += 4 {
				k1 := state[j] ^ state[j+2]
				a02x := xtime(k1)
				k2 := state[j+1] ^ state[j+3]
				a13x := xtime(k2)

				k1 ^= (k2 ^ xtime(state[j+1]^state[j+2]))
				k2 = k1

				a02xx := xtime(a02x)
				a13xx := xtime(a13x)

				k1 ^= (xtime(a02xx^a13xx) ^ a02xx)
				k2 ^= (xtime(a02xx^a13xx) ^ a13xx)

				state[j] ^= (k1 ^ a02x)
				state[j+1] ^= k2
				state[j+2] ^= (k1 ^ a13x)
				state[j+3] ^= (k2 ^ a02x ^ a13x)
			}
		}
	}

	return state
}

func AesEncrypt(src []byte, key Key) []byte {
	state := make([]byte, len(src))
	copy(state, src)

	round_key := key

	for i := 0; i < 16; i++ {
		state[i] ^= key[i]
	}

	for i := 0; i < numberOfRounds; i++ {
		/* First row: 0 shift, 0 4 8 12 */
		state[0] = rijndaelSbox[state[0]]
		state[4] = rijndaelSbox[state[4]]
		state[8] = rijndaelSbox[state[8]]
		state[12] = rijndaelSbox[state[12]]

		/* Second row: 1 shift, 1 5 9 13 */
		tmp := state[1]
		state[1] = rijndaelSbox[state[5]]
		state[5] = rijndaelSbox[state[9]]
		state[9] = rijndaelSbox[state[13]]
		state[13] = rijndaelSbox[tmp]

		/* Third row: 2 shift, 2 6 10 14 */
		tmp = state[2]
		state[2] = rijndaelSbox[state[10]]
		state[10] = rijndaelSbox[tmp]
		tmp = state[6]
		state[6] = rijndaelSbox[state[14]]
		state[14] = rijndaelSbox[tmp]

		/* Fourth row: 3 shift, 3 7 11 15 */
		tmp = state[15]
		state[15] = rijndaelSbox[state[11]]
		state[11] = rijndaelSbox[state[7]]
		state[7] = rijndaelSbox[state[3]]
		state[3] = rijndaelSbox[tmp]

		if i != numberOfRounds-1 {
			for k := 0; k < 16; k += 4 {
				j := state[k] ^ state[k+1]
				tmp = j ^ state[k+2] ^ state[k+3]

				j = xtime(j)

				state[k] ^= (j ^ tmp)

				j = state[k+1] ^ state[k+2]
				j = xtime(j)

				state[k+1] ^= (j ^ tmp)

				j = state[k+2] ^ state[k+3]
				j = xtime(j)

				state[k+2] ^= (j ^ tmp)
				state[k+3] = state[k] ^ state[k+1] ^ state[k+2] ^ tmp
			}
		}

		round_key[0] ^= rc[i]

		round_key[0] ^= rijndaelSbox[round_key[13]]
		round_key[1] ^= rijndaelSbox[round_key[14]]
		round_key[2] ^= rijndaelSbox[round_key[15]]
		round_key[3] ^= rijndaelSbox[round_key[12]]

		for k := 4; k < 16; k++ {
			round_key[k] ^= round_key[k-4]
		}
		for j := 0; j < 16; j++ {
			state[j] ^= round_key[j]
		}
	}
	return state
}

func xtime(b byte) byte {
	if b&0x80 > 0 {
		return (b << 1) ^ 0x1b
	} else {
		return b << 1
	}
}
<commit_msg>use stdlib aes implementation<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package yubikey

import (
	"crypto/aes"
)

func AesDecrypt(src []byte, key Key) []byte {
	dst := make([]byte, len(src))
	cipher, _ := aes.NewCipher(key[:])
	cipher.Decrypt(dst, src)
	return dst

}

func AesEncrypt(src []byte, key Key) []byte {
	dst := make([]byte, len(src))
	cipher, _ := aes.NewCipher(key[:])
	cipher.Encrypt(dst, src)
	return dst
}
<|endoftext|>


<commit_before>// Package mq provides an ability to integrate with message broker via AMQP in a declarative way.
package mq

import (
	"fmt"
	"net"
	"sync/atomic"
	"time"

	"github.com/NeowayLabs/wabbit"
	"github.com/NeowayLabs/wabbit/amqp"
	"github.com/NeowayLabs/wabbit/amqptest"
	"github.com/NeowayLabs/wabbit/utils"
	amqpDriver "github.com/streadway/amqp"
)

const (
	// Describes states during reconnect.
	statusReadyForReconnect int32 = 0
	statusReconnecting            = 1
)

// Used for creating connection to the fake AMQP server for tests.
var brokerIsMocked bool

type conn interface {
	Channel() (wabbit.Channel, error)
	Close() error
	NotifyClose(chan wabbit.Error) chan wabbit.Error
}

// MQ describes methods provided by message broker adapter.
type MQ interface {
	// Consumer returns consumer object by its name.
	Consumer(name string) (Consumer, error)
	// SetConsumerHandler allows you to set handler callback without getting consumer.
	SetConsumerHandler(name string, handler ConsumerHandler) error
	// AsyncProducer returns async producer. Should be used in most cases.
	AsyncProducer(name string) (AsyncProducer, error)
	// SyncProducer returns sync producer.
	SyncProducer(name string) (SyncProducer, error)
	// Error returns channel with all occurred errors.
	// Errors from sync producer won't be accessible.
	Error() <-chan error
	// Close stop all consumers and producers and close connection to broker.
	Close()
}

type mq struct {
	channel              wabbit.Channel
	config               Config
	connection           conn
	errorChannel         chan error
	internalErrorChannel chan error
	consumers            *consumersRegistry
	producers            *producersRegistry
	reconnectStatus      int32 // Defines whether client is trying to reconnect or not.
}

// New initializes AMQP connection to the message broker
// and returns adapter that provides an ability
// to get configured consumers and producers, read occurred errors and shutdown all workers.
func New(config Config) (MQ, error) {
	config.normalize()

	mq := &mq{
		config:               config,
		errorChannel:         make(chan error),
		internalErrorChannel: make(chan error),
		consumers:            newConsumersRegistry(len(config.Consumers)),
		producers:            newProducersRegistry(len(config.Producers)),
	}

	if err := mq.connect(); err != nil {
		return nil, err
	}

	go mq.errorHandler()

	return mq, mq.initialSetup()
}

// Set handler for consumer by its name. Returns false if consumer wasn't found.
// Can be called once for each consumer.
func (mq *mq) SetConsumerHandler(name string, handler ConsumerHandler) error {
	consumer, err := mq.Consumer(name)
	if err != nil {
		return err
	}

	consumer.Consume(handler)

	return nil
}

// Consumer returns a consumer by its name or error if consumer wasn't found.
func (mq *mq) Consumer(name string) (consumer Consumer, err error) {
	consumer, ok := mq.consumers.Get(name)
	if !ok {
		err = fmt.Errorf("consumer '%s' is not registered. Check your configuration", name)
	}

	return
}

// AsyncProducer returns an async producer by its name or error if producer wasn't found.
func (mq *mq) AsyncProducer(name string) (AsyncProducer, error) {
	item, exists := mq.producers.Get(name)
	producer, asserted := item.(*asyncProducer)

	if !exists || !asserted {
		return nil, fmt.Errorf("producer '%s' is not registered. Check your configuration", name)
	}

	return producer, nil
}

// SyncProducer returns a sync producer by its name or error if producer wasn't found.
func (mq *mq) SyncProducer(name string) (SyncProducer, error) {
	item, exists := mq.producers.Get(name)
	producer, asserted := item.(*syncProducer)

	if !exists || !asserted {
		return nil, fmt.Errorf("producer '%s' is not registered. Check your configuration", name)
	}

	return producer, nil
}

// Error provides an ability to access occurring errors.
func (mq *mq) Error() <-chan error {
	return mq.errorChannel
}

// Shutdown all workers and close connection to the message broker.
func (mq *mq) Close() {
	mq.stopProducersAndConsumers()

	if mq.channel != nil {
		mq.channel.Close()
	}

	if mq.connection != nil {
		mq.connection.Close()
	}
}

func (mq *mq) connect() error {
	connection, err := mq.createConnection()
	if err != nil {
		return err
	}

	channel, err := connection.Channel()
	if err != nil {
		connection.Close()

		return err
	}

	mq.connection = connection
	mq.channel = channel

	go mq.handleCloseEvent()

	return nil
}

func (mq *mq) createConnection() (conn conn, err error) {
	if brokerIsMocked || mq.config.TestMode {
		return amqptest.Dial(mq.config.DSN)
	}

	return amqp.Dial(mq.config.DSN)
}

// Register close handler.
// To get more details visit https://godoc.org/github.com/streadway/amqp#Connection.NotifyClose.
func (mq *mq) handleCloseEvent() {
	err := <-mq.connection.NotifyClose(make(chan wabbit.Error))
	if err != nil {
		mq.internalErrorChannel <- err
	}
}

func (mq *mq) errorHandler() {
	for err := range mq.internalErrorChannel {
		select {
		case mq.errorChannel <- err: // Proxies errors to the user.
		default: // For those clients who don't read errors.
		}

		mq.processError(err)
	}
}

func (mq *mq) processError(err error) {
	switch err.(type) {
	case *net.OpError:
		go mq.reconnect()
	case *utils.Error: // Broken connection. Used in tests.
		go mq.reconnect()
	case *amqpDriver.Error:
		rmqErr, _ := err.(*amqpDriver.Error)
		if rmqErr.Server == false { // For example channel was closed.
			go mq.reconnect()
		}
	default:
		// Wabbit error. Means that server is down.
		// Used in tests.
		if err.Error() == "Network unreachable" {
			go mq.reconnect()
		}
	}
}

func (mq *mq) initialSetup() error {
	if err := mq.setupExchanges(); err != nil {
		return err
	}

	if err := mq.setupQueues(); err != nil {
		return err
	}

	if err := mq.setupProducers(); err != nil {
		return err
	}

	return mq.setupConsumers()
}

// Called after each reconnect to recreate non-durable queues and exchanges.
func (mq *mq) setupAfterReconnect() error {
	if err := mq.setupExchanges(); err != nil {
		return err
	}

	if err := mq.setupQueues(); err != nil {
		return err
	}

	mq.producers.GoEach(func(producer internalProducer) {
		if err := mq.reconnectProducer(producer); err != nil {
			mq.internalErrorChannel <- err
		}
	})

	mq.consumers.GoEach(func(consumer *consumer) {
		if err := mq.reconnectConsumer(consumer); err != nil {
			mq.internalErrorChannel <- err
		}
	})

	return nil
}

func (mq *mq) setupExchanges() error {
	for _, config := range mq.config.Exchanges {
		if err := mq.declareExchange(config); err != nil {
			return err
		}
	}

	return nil
}

func (mq *mq) declareExchange(config ExchangeConfig) error {
	return mq.channel.ExchangeDeclare(config.Name, config.Type, wabbit.Option(config.Options))
}

func (mq *mq) setupQueues() error {
	for _, config := range mq.config.Queues {
		if err := mq.declareQueue(config); err != nil {
			return err
		}
	}

	return nil
}

func (mq *mq) declareQueue(config QueueConfig) error {
	if _, err := mq.channel.QueueDeclare(config.Name, wabbit.Option(config.Options)); err != nil {
		return err
	}

	return mq.channel.QueueBind(config.Name, config.RoutingKey, config.Exchange, wabbit.Option(config.BindingOptions))
}

func (mq *mq) setupProducers() error {
	for _, config := range mq.config.Producers {
		if err := mq.registerProducer(config); err != nil {
			return err
		}
	}

	return nil
}

func (mq *mq) registerProducer(config ProducerConfig) error {
	if _, ok := mq.producers.Get(config.Name); ok {
		return fmt.Errorf(`producer with name "%s" is already registered`, config.Name)
	}

	channel, err := mq.connection.Channel()
	if err != nil {
		return err
	}

	producer := newInternalProducer(channel, mq.internalErrorChannel, config)
	producer.init()
	mq.producers.Set(config.Name, producer)

	return nil
}

func (mq *mq) reconnectProducer(producer internalProducer) error {
	channel, err := mq.connection.Channel()
	if err != nil {
		return err
	}

	producer.setChannel(channel)
	producer.init()

	return nil
}

func (mq *mq) setupConsumers() error {
	for _, config := range mq.config.Consumers {
		if err := mq.registerConsumer(config); err != nil {
			return err
		}
	}

	return nil
}

func (mq *mq) registerConsumer(config ConsumerConfig) error {
	if _, ok := mq.consumers.Get(config.Name); ok {
		return fmt.Errorf(`consumer with name "%s" is already registered`, config.Name)
	}

	// Consumer must have at least one worker.
	if config.Workers < 1 {
		config.Workers = 1
	}

	consumer := newConsumer(config) // We need to save a whole config for reconnect.
	consumer.prefetchCount = config.PrefetchCount
	consumer.prefetchSize = config.PrefetchSize

	for i := 0; i < config.Workers; i++ {
		worker := newWorker(mq.internalErrorChannel)

		if err := mq.initializeConsumersWorker(consumer, worker); err != nil {
			return err
		}

		consumer.workers[i] = worker
	}

	mq.consumers.Set(config.Name, consumer) // Workers will start after consumer.Consume method call.

	return nil
}

func (mq *mq) reconnectConsumer(consumer *consumer) error {
	for _, worker := range consumer.workers {
		if err := mq.initializeConsumersWorker(consumer, worker); err != nil {
			return err
		}
	}

	consumer.consume(consumer.handler)

	return nil
}

func (mq *mq) initializeConsumersWorker(consumer *consumer, worker *worker) error {
	channel, err := mq.connection.Channel()
	if err != nil {
		return err
	}

	if err := channel.Qos(consumer.prefetchCount, consumer.prefetchSize, false); err != nil {
		return err
	}

	deliveries, err := channel.Consume(consumer.queue, "", consumer.options)
	if err != nil {
		return err
	}

	worker.setChannel(channel)
	worker.deliveries = deliveries

	return nil
}

// Reconnect stops current producers and consumers,
// recreates connection to the rabbit and than runs producers and consumers.
func (mq *mq) reconnect() {
	startedReconnect := atomic.CompareAndSwapInt32(&mq.reconnectStatus, statusReadyForReconnect, statusReconnecting)
	// There is no need to start a new reconnect if the previous one is not finished yet.
	if !startedReconnect {
		return
	}

	defer func() {
		atomic.StoreInt32(&mq.reconnectStatus, statusReadyForReconnect)
	}()

	time.Sleep(mq.config.ReconnectDelay) // TODO Add incremental sleep.

	mq.stopProducersAndConsumers()

	if err := mq.connect(); err != nil {
		mq.internalErrorChannel <- err

		return
	}

	if err := mq.setupAfterReconnect(); err != nil {
		mq.internalErrorChannel <- err
	}
}

func (mq *mq) stopProducersAndConsumers() {
	mq.producers.GoEach(func(producer internalProducer) {
		producer.Stop()
	})

	mq.consumers.GoEach(func(consumer *consumer) {
		consumer.Stop()
	})
}
<commit_msg>Applied code style rules. (#31)<commit_after>// Package mq provides an ability to integrate with message broker via AMQP in a declarative way.
package mq

import (
	"fmt"
	"net"
	"sync/atomic"
	"time"

	"github.com/NeowayLabs/wabbit"
	"github.com/NeowayLabs/wabbit/amqp"
	"github.com/NeowayLabs/wabbit/amqptest"
	"github.com/NeowayLabs/wabbit/utils"
	amqpDriver "github.com/streadway/amqp"
)

const (
	// Describes states during reconnect.
	statusReadyForReconnect int32 = 0
	statusReconnecting      int32 = 1
)

// Used for creating connection to the fake AMQP server for tests.
var brokerIsMocked bool

type conn interface {
	Channel() (wabbit.Channel, error)
	Close() error
	NotifyClose(chan wabbit.Error) chan wabbit.Error
}

// MQ describes methods provided by message broker adapter.
type MQ interface {
	// Consumer returns consumer object by its name.
	Consumer(name string) (Consumer, error)
	// SetConsumerHandler allows you to set handler callback without getting consumer.
	SetConsumerHandler(name string, handler ConsumerHandler) error
	// AsyncProducer returns async producer. Should be used in most cases.
	AsyncProducer(name string) (AsyncProducer, error)
	// SyncProducer returns sync producer.
	SyncProducer(name string) (SyncProducer, error)
	// Error returns channel with all occurred errors.
	// Errors from sync producer won't be accessible.
	Error() <-chan error
	// Close stop all consumers and producers and close connection to broker.
	Close()
}

type mq struct {
	channel              wabbit.Channel
	config               Config
	connection           conn
	errorChannel         chan error
	internalErrorChannel chan error
	consumers            *consumersRegistry
	producers            *producersRegistry
	reconnectStatus      int32 // Defines whether client is trying to reconnect or not.
}

// New initializes AMQP connection to the message broker
// and returns adapter that provides an ability
// to get configured consumers and producers, read occurred errors and shutdown all workers.
func New(config Config) (MQ, error) {
	config.normalize()

	mq := &mq{
		config:               config,
		errorChannel:         make(chan error),
		internalErrorChannel: make(chan error),
		consumers:            newConsumersRegistry(len(config.Consumers)),
		producers:            newProducersRegistry(len(config.Producers)),
	}

	if err := mq.connect(); err != nil {
		return nil, err
	}

	go mq.errorHandler()

	return mq, mq.initialSetup()
}

// Set handler for consumer by its name. Returns false if consumer wasn't found.
// Can be called once for each consumer.
func (mq *mq) SetConsumerHandler(name string, handler ConsumerHandler) error {
	consumer, err := mq.Consumer(name)
	if err != nil {
		return err
	}

	consumer.Consume(handler)

	return nil
}

// Consumer returns a consumer by its name or error if consumer wasn't found.
func (mq *mq) Consumer(name string) (Consumer, error) {
	consumer, ok := mq.consumers.Get(name)
	if !ok {
		err := fmt.Errorf("consumer '%s' is not registered. Check your configuration", name)

		return nil, err
	}

	return consumer, nil
}

// AsyncProducer returns an async producer by its name or error if producer wasn't found.
func (mq *mq) AsyncProducer(name string) (AsyncProducer, error) {
	item, exists := mq.producers.Get(name)
	producer, asserted := item.(*asyncProducer)

	if !exists || !asserted {
		return nil, fmt.Errorf("producer '%s' is not registered. Check your configuration", name)
	}

	return producer, nil
}

// SyncProducer returns a sync producer by its name or error if producer wasn't found.
func (mq *mq) SyncProducer(name string) (SyncProducer, error) {
	item, exists := mq.producers.Get(name)
	producer, asserted := item.(*syncProducer)

	if !exists || !asserted {
		return nil, fmt.Errorf("producer '%s' is not registered. Check your configuration", name)
	}

	return producer, nil
}

// Error provides an ability to access occurring errors.
func (mq *mq) Error() <-chan error {
	return mq.errorChannel
}

// Shutdown all workers and close connection to the message broker.
func (mq *mq) Close() {
	mq.stopProducersAndConsumers()

	if mq.channel != nil {
		_ = mq.channel.Close()
	}

	if mq.connection != nil {
		_ = mq.connection.Close()
	}
}

func (mq *mq) connect() error {
	connection, err := mq.createConnection()
	if err != nil {
		return err
	}

	channel, err := connection.Channel()
	if err != nil {
		_ = connection.Close()

		return err
	}

	mq.connection = connection
	mq.channel = channel

	go mq.handleCloseEvent()

	return nil
}

func (mq *mq) createConnection() (conn, error) {
	if brokerIsMocked || mq.config.TestMode {
		return amqptest.Dial(mq.config.DSN)
	}

	return amqp.Dial(mq.config.DSN)
}

// Register close handler.
// To get more details visit https://godoc.org/github.com/streadway/amqp#Connection.NotifyClose.
func (mq *mq) handleCloseEvent() {
	err := <-mq.connection.NotifyClose(make(chan wabbit.Error))
	if err != nil {
		mq.internalErrorChannel <- err
	}
}

func (mq *mq) errorHandler() {
	for err := range mq.internalErrorChannel {
		select {
		case mq.errorChannel <- err: // Proxies errors to the user.
		default: // For those clients who don't read errors.
		}

		mq.processError(err)
	}
}

func (mq *mq) processError(err error) {
	switch err.(type) {
	case *net.OpError:
		go mq.reconnect()
	case *utils.Error: // Broken connection. Used in tests.
		go mq.reconnect()
	case *amqpDriver.Error:
		rmqErr, _ := err.(*amqpDriver.Error)
		if rmqErr.Server == false { // For example channel was closed.
			go mq.reconnect()
		}
	default:
		// Wabbit error. Means that server is down.
		// Used in tests.
		if err.Error() == "Network unreachable" {
			go mq.reconnect()
		}
	}
}

func (mq *mq) initialSetup() error {
	if err := mq.setupExchanges(); err != nil {
		return err
	}

	if err := mq.setupQueues(); err != nil {
		return err
	}

	if err := mq.setupProducers(); err != nil {
		return err
	}

	return mq.setupConsumers()
}

// Called after each reconnect to recreate non-durable queues and exchanges.
func (mq *mq) setupAfterReconnect() error {
	if err := mq.setupExchanges(); err != nil {
		return err
	}

	if err := mq.setupQueues(); err != nil {
		return err
	}

	mq.producers.GoEach(func(producer internalProducer) {
		if err := mq.reconnectProducer(producer); err != nil {
			mq.internalErrorChannel <- err
		}
	})

	mq.consumers.GoEach(func(consumer *consumer) {
		if err := mq.reconnectConsumer(consumer); err != nil {
			mq.internalErrorChannel <- err
		}
	})

	return nil
}

func (mq *mq) setupExchanges() error {
	for _, config := range mq.config.Exchanges {
		if err := mq.declareExchange(config); err != nil {
			return err
		}
	}

	return nil
}

func (mq *mq) declareExchange(config ExchangeConfig) error {
	return mq.channel.ExchangeDeclare(config.Name, config.Type, wabbit.Option(config.Options))
}

func (mq *mq) setupQueues() error {
	for _, config := range mq.config.Queues {
		if err := mq.declareQueue(config); err != nil {
			return err
		}
	}

	return nil
}

func (mq *mq) declareQueue(config QueueConfig) error {
	if _, err := mq.channel.QueueDeclare(config.Name, wabbit.Option(config.Options)); err != nil {
		return err
	}

	return mq.channel.QueueBind(config.Name, config.RoutingKey, config.Exchange, wabbit.Option(config.BindingOptions))
}

func (mq *mq) setupProducers() error {
	for _, config := range mq.config.Producers {
		if err := mq.registerProducer(config); err != nil {
			return err
		}
	}

	return nil
}

func (mq *mq) registerProducer(config ProducerConfig) error {
	if _, ok := mq.producers.Get(config.Name); ok {
		return fmt.Errorf(`producer with name "%s" is already registered`, config.Name)
	}

	channel, err := mq.connection.Channel()
	if err != nil {
		return err
	}

	producer := newInternalProducer(channel, mq.internalErrorChannel, config)
	producer.init()
	mq.producers.Set(config.Name, producer)

	return nil
}

func (mq *mq) reconnectProducer(producer internalProducer) error {
	channel, err := mq.connection.Channel()
	if err != nil {
		return err
	}

	producer.setChannel(channel)
	producer.init()

	return nil
}

func (mq *mq) setupConsumers() error {
	for _, config := range mq.config.Consumers {
		if err := mq.registerConsumer(config); err != nil {
			return err
		}
	}

	return nil
}

func (mq *mq) registerConsumer(config ConsumerConfig) error {
	if _, ok := mq.consumers.Get(config.Name); ok {
		return fmt.Errorf(`consumer with name "%s" is already registered`, config.Name)
	}

	// Consumer must have at least one worker.
	if config.Workers < 1 {
		config.Workers = 1
	}

	consumer := newConsumer(config) // We need to save a whole config for reconnect.
	consumer.prefetchCount = config.PrefetchCount
	consumer.prefetchSize = config.PrefetchSize

	for i := 0; i < config.Workers; i++ {
		worker := newWorker(mq.internalErrorChannel)

		if err := mq.initializeConsumersWorker(consumer, worker); err != nil {
			return err
		}

		consumer.workers[i] = worker
	}

	mq.consumers.Set(config.Name, consumer) // Workers will start after consumer.Consume method call.

	return nil
}

func (mq *mq) reconnectConsumer(consumer *consumer) error {
	for _, worker := range consumer.workers {
		if err := mq.initializeConsumersWorker(consumer, worker); err != nil {
			return err
		}
	}

	consumer.consume(consumer.handler)

	return nil
}

func (mq *mq) initializeConsumersWorker(consumer *consumer, worker *worker) error {
	channel, err := mq.connection.Channel()
	if err != nil {
		return err
	}

	if err := channel.Qos(consumer.prefetchCount, consumer.prefetchSize, false); err != nil {
		return err
	}

	deliveries, err := channel.Consume(consumer.queue, "", consumer.options)
	if err != nil {
		return err
	}

	worker.setChannel(channel)
	worker.deliveries = deliveries

	return nil
}

// Reconnect stops current producers and consumers,
// recreates connection to the rabbit and than runs producers and consumers.
func (mq *mq) reconnect() {
	startedReconnect := atomic.CompareAndSwapInt32(&mq.reconnectStatus, statusReadyForReconnect, statusReconnecting)
	// There is no need to start a new reconnect if the previous one is not finished yet.
	if !startedReconnect {
		return
	}

	defer func() {
		atomic.StoreInt32(&mq.reconnectStatus, statusReadyForReconnect)
	}()

	time.Sleep(mq.config.ReconnectDelay) // TODO Add incremental sleep.

	mq.stopProducersAndConsumers()

	if err := mq.connect(); err != nil {
		mq.internalErrorChannel <- err

		return
	}

	if err := mq.setupAfterReconnect(); err != nil {
		mq.internalErrorChannel <- err
	}
}

func (mq *mq) stopProducersAndConsumers() {
	mq.producers.GoEach(func(producer internalProducer) {
		producer.Stop()
	})

	mq.consumers.GoEach(func(consumer *consumer) {
		consumer.Stop()
	})
}
<|endoftext|>


<commit_before>package hoverfly

import (
	"encoding/json"
	"fmt"
	"github.com/SpectoLabs/hoverfly/core/handlers/v2"
	. "github.com/onsi/gomega"
	"net/http"
	"testing"
	"time"
)

func BenchmarkProcessRequest(b *testing.B) {

	RegisterTestingT(b)
	hoverfly := NewHoverflyWithConfiguration(&Configuration{
		Webserver: true,
		ProxyPort: "8500",
		AdminPort: "8888",
		Mode:      "simulate",
	})

//	simulation := v2.SimulationViewV5{}
//	_ = json.Unmarshal([]byte(`{
//	"data": {
//		"pairs": [{
//			"response": {
//				"status": 200,
//				"body": "5iNe8dxWH5Ca8pZqAfEHv3rgC0SsvKNLu6o3K",
//				"encodedBody": false,
//				"headers": {
//					"Accept-Ranges": ["bytes"],
//					"Cache-Control": ["max-age=3600"],
//					"Connection": ["keep-alive"],
//					"Content-Type": ["text/html; charset=utf-8"],
//					"Date": ["Tue, 30 May 2017 13:23:09 GMT"],
//					"Etag": ["\"82b6bafbc0c4af5e3886d07802f4b62d\""],
//					"Hoverfly": ["Was-Here"],
//					"Last-Modified": ["Fri, 19 May 2017 10:59:12 GMT"],
//					"Server": ["nginx"],
//					"Strict-Transport-Security": ["max-age=31556926"],
//					"Transfer-Encoding": ["chunked"],
//					"Vary": ["Accept-Encoding"]
//				}
//			},
//			"request": {
//				"path": [{
//					"matcher": "exact",
//					"value": "/bar"
//				}],
//				"method": [{
//					"matcher": "exact",
//					"value": "GET"
//				}],
//				"query": {},
//				"body": [{
//					"matcher": "exact",
//					"value": ""
//				}]
//			}
//		}],
//		"globalActions": {
//			"delays": []
//		}
//	},
//	"meta": {
//		"schemaVersion": "v5",
//		"hoverflyVersion": "v0.17.0",
//		"timeExported": "2017-05-30T14:23:44+01:00"
//	}
//}`), &simulation)

	templated := v2.SimulationViewV5{}
	_ = json.Unmarshal([]byte(`{
	"data": {
		"pairs": [{
			"response": {
				"status": 200,
				"body": "{\"st\": 1,\"sid\": 418,\"tt\": \"{{ Request.Path.[0] }}\",\"gr\": 0,\"uuid\": \"{{ randomUuid }}\",\"ip\": \"127.0.0.1\",\"ua\": \"user_agent\",\"tz\": -6,\"v\": 1}",
				"encodedBody": false,
				"templated": true,
				"headers": {
					"Accept-Ranges": ["bytes"],
					"Cache-Control": ["max-age=3600"],
					"Connection": ["keep-alive"],
					"Content-Type": ["text/html; charset=utf-8"],
					"Date": ["Tue, 30 May 2017 13:23:09 GMT"],
					"Etag": ["\"82b6bafbc0c4af5e3886d07802f4b62d\""],
					"Hoverfly": ["Was-Here"],
					"Last-Modified": ["Fri, 19 May 2017 10:59:12 GMT"],
					"Server": ["nginx"],
					"Strict-Transport-Security": ["max-age=31556926"],
					"Transfer-Encoding": ["chunked"],
					"Vary": ["Accept-Encoding"]
				}
			},
			"request": {
				"path": [{
					"matcher": "exact",
					"value": "/bar"
				}],
				"method": [{
					"matcher": "exact",
					"value": "GET"
				}],
				"query": {},
				"body": [{
					"matcher": "exact",
					"value": ""
				}]
			}
		}],
		"globalActions": {
			"delays": []
		}
	},
	"meta": {
		"schemaVersion": "v5",
		"hoverflyVersion": "v0.17.0",
		"timeExported": "2017-05-30T14:23:44+01:00"
	}
}`), &templated)

	fmt.Println(hoverfly.StartProxy())
	time.Sleep(time.Second)
	request, _ := http.NewRequest(http.MethodGet, "http://localhost:8500/bar", nil)
	var resp *http.Response
	//hoverfly.PutSimulation(simulation)
	//
	//b.Run("Simple simulation", func(b *testing.B) {
	//
	//	for n := 0; n < b.N; n++ {
	//		resp = hoverfly.processRequest(request)
	//	}
	//})

	hoverfly.PutSimulation(templated)

	b.Run("Templated simulation", func(b *testing.B) {

		for n := 0; n < b.N; n++ {
			resp = hoverfly.processRequest(request)
		}
	})

	Expect(resp.StatusCode).To(Equal(200))

	hoverfly.StopProxy()
}
<commit_msg>Create sub benchmarks<commit_after>package hoverfly

import (
	"encoding/json"
	"fmt"
	"github.com/SpectoLabs/hoverfly/core/handlers/v2"
	. "github.com/onsi/gomega"
	"net/http"
	"testing"
	"time"
)

func BenchmarkProcessRequest(b *testing.B) {

	RegisterTestingT(b)

	hoverfly := NewHoverflyWithConfiguration(&Configuration{
		Webserver: true,
		ProxyPort: "8500",
		AdminPort: "8888",
		Mode:      "simulate",
	})

	simulation := v2.SimulationViewV5{}
	_ = json.Unmarshal([]byte(`{
	"data": {
		"pairs": [{
			"response": {
				"status": 200,
				"body": "5iNe8dxWH5Ca8pZqAfEHv3rgC0SsvKNLu6o3K",
				"encodedBody": false,
				"headers": {
					"Accept-Ranges": ["bytes"],
					"Cache-Control": ["max-age=3600"],
					"Connection": ["keep-alive"],
					"Content-Type": ["text/html; charset=utf-8"],
					"Date": ["Tue, 30 May 2017 13:23:09 GMT"],
					"Etag": ["\"82b6bafbc0c4af5e3886d07802f4b62d\""],
					"Hoverfly": ["Was-Here"],
					"Last-Modified": ["Fri, 19 May 2017 10:59:12 GMT"],
					"Server": ["nginx"],
					"Strict-Transport-Security": ["max-age=31556926"],
					"Transfer-Encoding": ["chunked"],
					"Vary": ["Accept-Encoding"]
				}
			},
			"request": {
				"path": [{
					"matcher": "exact",
					"value": "/bar"
				}],
				"method": [{
					"matcher": "exact",
					"value": "GET"
				}],
				"query": {},
				"body": [{
					"matcher": "exact",
					"value": ""
				}]
			}
		}],
		"globalActions": {
			"delays": []
		}
	},
	"meta": {
		"schemaVersion": "v5",
		"hoverflyVersion": "v0.17.0",
		"timeExported": "2017-05-30T14:23:44+01:00"
	}
}`), &simulation)

	templated := v2.SimulationViewV5{}
	_ = json.Unmarshal([]byte(`{
	"data": {
		"pairs": [{
			"response": {
				"status": 200,
				"body": "{\"st\": 1,\"sid\": 418,\"tt\": \"{{ Request.Path.[0] }}\",\"gr\": 0,\"uuid\": \"{{ randomUuid }}\",\"ip\": \"127.0.0.1\",\"ua\": \"user_agent\",\"tz\": -6,\"v\": 1}",
				"encodedBody": false,
				"templated": true,
				"headers": {
					"Accept-Ranges": ["bytes"],
					"Cache-Control": ["max-age=3600"],
					"Connection": ["keep-alive"],
					"Content-Type": ["text/html; charset=utf-8"],
					"Date": ["Tue, 30 May 2017 13:23:09 GMT"],
					"Etag": ["\"82b6bafbc0c4af5e3886d07802f4b62d\""],
					"Hoverfly": ["Was-Here"],
					"Last-Modified": ["Fri, 19 May 2017 10:59:12 GMT"],
					"Server": ["nginx"],
					"Strict-Transport-Security": ["max-age=31556926"],
					"Transfer-Encoding": ["chunked"],
					"Vary": ["Accept-Encoding"]
				}
			},
			"request": {
				"path": [{
					"matcher": "exact",
					"value": "/bar"
				}],
				"method": [{
					"matcher": "exact",
					"value": "GET"
				}],
				"query": {},
				"body": [{
					"matcher": "exact",
					"value": ""
				}]
			}
		}],
		"globalActions": {
			"delays": []
		}
	},
	"meta": {
		"schemaVersion": "v5",
		"hoverflyVersion": "v0.17.0",
		"timeExported": "2017-05-30T14:23:44+01:00"
	}
}`), &templated)

	benchmarks := []struct{
		name string
		simulation v2.SimulationViewV5
	} {
		{"Simple simulation", simulation},
		{"Templated simulation", templated},
	}

	fmt.Println(hoverfly.StartProxy())
	time.Sleep(time.Second)
	request, _ := http.NewRequest(http.MethodGet, "http://localhost:8500/bar", nil)
	var resp *http.Response

	for _, bm := range benchmarks {
		hoverfly.DeleteSimulation()
		hoverfly.PutSimulation(bm.simulation)

		b.Run(bm.name, func(b *testing.B) {

			for n := 0; n < b.N; n++ {
				resp = hoverfly.processRequest(request)
			}
		})
	}

	Expect(resp.StatusCode).To(Equal(200))

	hoverfly.StopProxy()
}
<|endoftext|>


<commit_before>package component

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/url"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/TheThingsNetwork/go-account-lib/cache"
	"github.com/TheThingsNetwork/go-account-lib/claims"
	"github.com/TheThingsNetwork/go-account-lib/keys"
	"github.com/TheThingsNetwork/go-account-lib/oauth"
	"github.com/TheThingsNetwork/go-account-lib/tokenkey"
	"github.com/TheThingsNetwork/ttn/api"
	pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/security"
	jwt "github.com/dgrijalva/jwt-go"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

// InitAuth initializes Auth functionality
func (c *Component) InitAuth() error {
	inits := []func() error{
		c.initAuthServers,
		c.initKeyPair,
		c.initRoots,
	}
	if c.Config.UseTLS {
		inits = append(inits, c.initTLS)
	}

	for _, init := range inits {
		if err := init(); err != nil {
			return err
		}
	}

	return nil
}

type authServer struct {
	url      string
	username string
	password string
}

func parseAuthServer(str string) (srv authServer, err error) {
	url, err := url.Parse(str)
	if err != nil {
		return srv, err
	}
	srv.url = fmt.Sprintf("%s://%s", url.Scheme, url.Host)
	if url.User != nil {
		srv.username = url.User.Username()
		srv.password, _ = url.User.Password()
	}
	return srv, nil
}

func (c *Component) initAuthServers() error {
	urlMap := make(map[string]string)
	funcMap := make(map[string]tokenkey.TokenFunc)
	var httpProvider tokenkey.Provider
	for id, url := range c.Config.AuthServers {
		if strings.HasPrefix(url, "file://") {
			file := strings.TrimPrefix(url, "file://")
			contents, err := ioutil.ReadFile(path.Clean(file))
			if err != nil {
				return err
			}
			funcMap[id] = func(renew bool) (*tokenkey.TokenKey, error) {
				return &tokenkey.TokenKey{Algorithm: "ES256", Key: string(contents)}, nil
			}
			continue
		}
		srv, err := parseAuthServer(url)
		if err != nil {
			return err
		}
		urlMap[id] = srv.url
		funcMap[id] = func(renew bool) (*tokenkey.TokenKey, error) {
			return httpProvider.Get(id, renew)
		}
	}
	httpProvider = tokenkey.HTTPProvider(
		urlMap,
		cache.WriteTroughCacheWithFormat(c.Config.KeyDir, "auth-%s.pub"),
	)
	c.TokenKeyProvider = tokenkey.FuncProvider(funcMap)
	return nil
}

// UpdateTokenKey updates the OAuth Bearer token key
func (c *Component) UpdateTokenKey() error {
	if c.TokenKeyProvider == nil {
		return errors.NewErrInternal("No public key provider configured for token validation")
	}

	// Set up Auth Server Token Validation
	err := c.TokenKeyProvider.Update()
	if err != nil {
		c.Ctx.Warnf("ttn: Failed to refresh public keys for token validation: %s", err.Error())
	} else {
		c.Ctx.Info("ttn: Got public keys for token validation")
	}

	return nil
}

func (c *Component) initKeyPair() error {
	priv, err := security.LoadKeypair(c.Config.KeyDir)
	if err != nil {
		return err
	}
	c.privateKey = priv

	pubPEM, _ := security.PublicPEM(priv)
	c.Identity.PublicKey = string(pubPEM)

	return nil
}

func (c *Component) initTLS() error {
	cert, err := security.LoadCert(c.Config.KeyDir)
	if err != nil {
		return err
	}
	c.Identity.Certificate = string(cert)

	privPEM, _ := security.PrivatePEM(c.privateKey)
	cer, err := tls.X509KeyPair(cert, privPEM)
	if err != nil {
		return err
	}

	c.tlsConfig = &tls.Config{Certificates: []tls.Certificate{cer}}
	return nil
}

func (c *Component) initRoots() error {
	path := filepath.Clean(c.Config.KeyDir + "/ca.cert")
	cert, err := ioutil.ReadFile(path)
	if err != nil {
		return nil
	}
	if !api.RootCAs.AppendCertsFromPEM(cert) {
		return fmt.Errorf("Could not add root certificates from %s", path)
	}
	return nil
}

// BuildJWT builds a short-lived JSON Web Token for this component
func (c *Component) BuildJWT() (string, error) {
	if c.privateKey != nil {
		privPEM, err := security.PrivatePEM(c.privateKey)
		if err != nil {
			return "", err
		}
		return security.BuildJWT(c.Identity.Id, 20*time.Second, privPEM)
	}
	return "", nil
}

// GetContext returns a context for outgoing RPC request. If token is "", this function will generate a short lived token from the component
func (c *Component) GetContext(token string) context.Context {
	var serviceName, serviceVersion, id, netAddress string
	if c.Identity != nil {
		serviceName = c.Identity.ServiceName
		id = c.Identity.Id
		if token == "" {
			token, _ = c.BuildJWT()
		}
		serviceVersion = c.Identity.ServiceVersion
		netAddress = c.Identity.NetAddress
	}
	md := metadata.Pairs(
		"service-name", serviceName,
		"service-version", serviceVersion,
		"id", id,
		"token", token,
		"net-address", netAddress,
	)
	ctx := metadata.NewContext(context.Background(), md)
	return ctx
}

// ExchangeAppKeyForToken enables authentication with the App Access Key
func (c *Component) ExchangeAppKeyForToken(appID, key string) (string, error) {
	issuerID := keys.KeyIssuer(key)
	if issuerID == "" {
		// Take the first configured auth server
		for k := range c.Config.AuthServers {
			issuerID = k
			break
		}
		key = fmt.Sprintf("%s.%s", issuerID, key)
	}
	issuer, ok := c.Config.AuthServers[issuerID]
	if !ok {
		return "", fmt.Errorf("Auth server %s not registered", issuer)
	}

	srv, _ := parseAuthServer(issuer)

	oauth := oauth.OAuth(srv.url, &oauth.Client{
		ID:     srv.username,
		Secret: srv.password,
	})

	token, err := oauth.ExchangeAppKeyForToken(appID, key)
	if err != nil {
		return "", err
	}

	return token.AccessToken, nil
}

// ValidateNetworkContext validates the context of a network request (router-broker, broker-handler, etc)
func (c *Component) ValidateNetworkContext(ctx context.Context) (component *pb_discovery.Announcement, err error) {
	defer func() {
		if err != nil {
			time.Sleep(time.Second)
		}
	}()

	md, ok := metadata.FromContext(ctx)
	if !ok {
		err = errors.NewErrInternal("Could not get metadata from context")
		return
	}
	var id, serviceName, token string
	if ids, ok := md["id"]; ok && len(ids) == 1 {
		id = ids[0]
	}
	if id == "" {
		err = errors.NewErrInvalidArgument("Metadata", "id missing")
		return
	}
	if serviceNames, ok := md["service-name"]; ok && len(serviceNames) == 1 {
		serviceName = serviceNames[0]
	}
	if serviceName == "" {
		err = errors.NewErrInvalidArgument("Metadata", "service-name missing")
		return
	}
	if tokens, ok := md["token"]; ok && len(tokens) == 1 {
		token = tokens[0]
	}

	var announcement *pb_discovery.Announcement
	announcement, err = c.Discover(serviceName, id)
	if err != nil {
		return
	}

	if announcement.PublicKey == "" {
		return announcement, nil
	}

	if token == "" {
		err = errors.NewErrInvalidArgument("Metadata", "token missing")
		return
	}

	var claims *jwt.StandardClaims
	claims, err = security.ValidateJWT(token, []byte(announcement.PublicKey))
	if err != nil {
		return
	}
	if claims.Issuer != id {
		err = errors.NewErrInvalidArgument("Metadata", "token was issued by different component id")
		return
	}

	return announcement, nil
}

// ValidateTTNAuthContext gets a token from the context and validates it
func (c *Component) ValidateTTNAuthContext(ctx context.Context) (*claims.Claims, error) {
	token, err := api.TokenFromContext(ctx)
	if err != nil {
		return nil, err
	}

	if c.TokenKeyProvider == nil {
		return nil, errors.NewErrInternal("No token provider configured")
	}

	claims, err := claims.FromToken(c.TokenKeyProvider, token)
	if err != nil {
		return nil, errors.NewErrPermissionDenied(err.Error())
	}

	return claims, nil
}
<commit_msg>Fix shadowing of Auth Server ID and URL<commit_after>package component

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/url"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/TheThingsNetwork/go-account-lib/cache"
	"github.com/TheThingsNetwork/go-account-lib/claims"
	"github.com/TheThingsNetwork/go-account-lib/keys"
	"github.com/TheThingsNetwork/go-account-lib/oauth"
	"github.com/TheThingsNetwork/go-account-lib/tokenkey"
	"github.com/TheThingsNetwork/ttn/api"
	pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/security"
	jwt "github.com/dgrijalva/jwt-go"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

// InitAuth initializes Auth functionality
func (c *Component) InitAuth() error {
	inits := []func() error{
		c.initAuthServers,
		c.initKeyPair,
		c.initRoots,
	}
	if c.Config.UseTLS {
		inits = append(inits, c.initTLS)
	}

	for _, init := range inits {
		if err := init(); err != nil {
			return err
		}
	}

	return nil
}

type authServer struct {
	url      string
	username string
	password string
}

func parseAuthServer(str string) (srv authServer, err error) {
	url, err := url.Parse(str)
	if err != nil {
		return srv, err
	}
	srv.url = fmt.Sprintf("%s://%s", url.Scheme, url.Host)
	if url.User != nil {
		srv.username = url.User.Username()
		srv.password, _ = url.User.Password()
	}
	return srv, nil
}

func (c *Component) initAuthServers() error {
	urlMap := make(map[string]string)
	funcMap := make(map[string]tokenkey.TokenFunc)
	var httpProvider tokenkey.Provider
	for id, url := range c.Config.AuthServers {
		id, url := id, url // deliberately shadow these
		if strings.HasPrefix(url, "file://") {
			file := strings.TrimPrefix(url, "file://")
			contents, err := ioutil.ReadFile(path.Clean(file))
			if err != nil {
				return err
			}
			funcMap[id] = func(renew bool) (*tokenkey.TokenKey, error) {
				return &tokenkey.TokenKey{Algorithm: "ES256", Key: string(contents)}, nil
			}
			continue
		}
		srv, err := parseAuthServer(url)
		if err != nil {
			return err
		}
		urlMap[id] = srv.url
		funcMap[id] = func(renew bool) (*tokenkey.TokenKey, error) {
			return httpProvider.Get(id, renew)
		}
	}
	httpProvider = tokenkey.HTTPProvider(
		urlMap,
		cache.WriteTroughCacheWithFormat(c.Config.KeyDir, "auth-%s.pub"),
	)
	c.TokenKeyProvider = tokenkey.FuncProvider(funcMap)
	return nil
}

// UpdateTokenKey updates the OAuth Bearer token key
func (c *Component) UpdateTokenKey() error {
	if c.TokenKeyProvider == nil {
		return errors.NewErrInternal("No public key provider configured for token validation")
	}

	// Set up Auth Server Token Validation
	err := c.TokenKeyProvider.Update()
	if err != nil {
		c.Ctx.Warnf("ttn: Failed to refresh public keys for token validation: %s", err.Error())
	} else {
		c.Ctx.Info("ttn: Got public keys for token validation")
	}

	return nil
}

func (c *Component) initKeyPair() error {
	priv, err := security.LoadKeypair(c.Config.KeyDir)
	if err != nil {
		return err
	}
	c.privateKey = priv

	pubPEM, _ := security.PublicPEM(priv)
	c.Identity.PublicKey = string(pubPEM)

	return nil
}

func (c *Component) initTLS() error {
	cert, err := security.LoadCert(c.Config.KeyDir)
	if err != nil {
		return err
	}
	c.Identity.Certificate = string(cert)

	privPEM, _ := security.PrivatePEM(c.privateKey)
	cer, err := tls.X509KeyPair(cert, privPEM)
	if err != nil {
		return err
	}

	c.tlsConfig = &tls.Config{Certificates: []tls.Certificate{cer}}
	return nil
}

func (c *Component) initRoots() error {
	path := filepath.Clean(c.Config.KeyDir + "/ca.cert")
	cert, err := ioutil.ReadFile(path)
	if err != nil {
		return nil
	}
	if !api.RootCAs.AppendCertsFromPEM(cert) {
		return fmt.Errorf("Could not add root certificates from %s", path)
	}
	return nil
}

// BuildJWT builds a short-lived JSON Web Token for this component
func (c *Component) BuildJWT() (string, error) {
	if c.privateKey != nil {
		privPEM, err := security.PrivatePEM(c.privateKey)
		if err != nil {
			return "", err
		}
		return security.BuildJWT(c.Identity.Id, 20*time.Second, privPEM)
	}
	return "", nil
}

// GetContext returns a context for outgoing RPC request. If token is "", this function will generate a short lived token from the component
func (c *Component) GetContext(token string) context.Context {
	var serviceName, serviceVersion, id, netAddress string
	if c.Identity != nil {
		serviceName = c.Identity.ServiceName
		id = c.Identity.Id
		if token == "" {
			token, _ = c.BuildJWT()
		}
		serviceVersion = c.Identity.ServiceVersion
		netAddress = c.Identity.NetAddress
	}
	md := metadata.Pairs(
		"service-name", serviceName,
		"service-version", serviceVersion,
		"id", id,
		"token", token,
		"net-address", netAddress,
	)
	ctx := metadata.NewContext(context.Background(), md)
	return ctx
}

// ExchangeAppKeyForToken enables authentication with the App Access Key
func (c *Component) ExchangeAppKeyForToken(appID, key string) (string, error) {
	issuerID := keys.KeyIssuer(key)
	if issuerID == "" {
		// Take the first configured auth server
		for k := range c.Config.AuthServers {
			issuerID = k
			break
		}
		key = fmt.Sprintf("%s.%s", issuerID, key)
	}
	issuer, ok := c.Config.AuthServers[issuerID]
	if !ok {
		return "", fmt.Errorf("Auth server %s not registered", issuer)
	}

	srv, _ := parseAuthServer(issuer)

	oauth := oauth.OAuth(srv.url, &oauth.Client{
		ID:     srv.username,
		Secret: srv.password,
	})

	token, err := oauth.ExchangeAppKeyForToken(appID, key)
	if err != nil {
		return "", err
	}

	return token.AccessToken, nil
}

// ValidateNetworkContext validates the context of a network request (router-broker, broker-handler, etc)
func (c *Component) ValidateNetworkContext(ctx context.Context) (component *pb_discovery.Announcement, err error) {
	defer func() {
		if err != nil {
			time.Sleep(time.Second)
		}
	}()

	md, ok := metadata.FromContext(ctx)
	if !ok {
		err = errors.NewErrInternal("Could not get metadata from context")
		return
	}
	var id, serviceName, token string
	if ids, ok := md["id"]; ok && len(ids) == 1 {
		id = ids[0]
	}
	if id == "" {
		err = errors.NewErrInvalidArgument("Metadata", "id missing")
		return
	}
	if serviceNames, ok := md["service-name"]; ok && len(serviceNames) == 1 {
		serviceName = serviceNames[0]
	}
	if serviceName == "" {
		err = errors.NewErrInvalidArgument("Metadata", "service-name missing")
		return
	}
	if tokens, ok := md["token"]; ok && len(tokens) == 1 {
		token = tokens[0]
	}

	var announcement *pb_discovery.Announcement
	announcement, err = c.Discover(serviceName, id)
	if err != nil {
		return
	}

	if announcement.PublicKey == "" {
		return announcement, nil
	}

	if token == "" {
		err = errors.NewErrInvalidArgument("Metadata", "token missing")
		return
	}

	var claims *jwt.StandardClaims
	claims, err = security.ValidateJWT(token, []byte(announcement.PublicKey))
	if err != nil {
		return
	}
	if claims.Issuer != id {
		err = errors.NewErrInvalidArgument("Metadata", "token was issued by different component id")
		return
	}

	return announcement, nil
}

// ValidateTTNAuthContext gets a token from the context and validates it
func (c *Component) ValidateTTNAuthContext(ctx context.Context) (*claims.Claims, error) {
	token, err := api.TokenFromContext(ctx)
	if err != nil {
		return nil, err
	}

	if c.TokenKeyProvider == nil {
		return nil, errors.NewErrInternal("No token provider configured")
	}

	claims, err := claims.FromToken(c.TokenKeyProvider, token)
	if err != nil {
		return nil, errors.NewErrPermissionDenied(err.Error())
	}

	return claims, nil
}
<|endoftext|>


<commit_before>package qf

import (
	context "context"
	fmt "fmt"
	net "net"
	sync "sync"
	"testing"
	time "time"

	grpc "google.golang.org/grpc"
)

const requestValue = 0x1001_1001

type testQSpec struct {
	quorum int
}

func (q testQSpec) UseReqQF(in *Request, replies []*Response) (*Response, bool) {
	if len(replies) < q.quorum {
		return nil, false
	}
	expected := in.GetValue()
	for _, reply := range replies {
		if expected != reply.GetResult() {
			return reply, true
		}
	}
	return replies[0], true
}

func (q testQSpec) IgnoreReqQF(_ *Request, replies []*Response) (*Response, bool) {
	if len(replies) < q.quorum {
		return nil, false
	}
	expected := int64(requestValue)
	for _, reply := range replies {
		if expected != reply.GetResult() {
			return reply, true
		}
	}
	return replies[0], true
}

func (q testQSpec) WithoutReqQF(replies []*Response) (*Response, bool) {
	if len(replies) < q.quorum {
		return nil, false
	}
	expected := int64(requestValue)
	for _, reply := range replies {
		if expected != reply.GetResult() {
			return reply, true
		}
	}
	return replies[0], true
}

func BenchmarkQF(b *testing.B) {
	for n := 3; n < 20; n += 2 {
		quorum := n / 2
		qspec := &testQSpec{quorum: quorum}
		request := &Request{Value: 1}

		b.Run(fmt.Sprintf("UseReq_%d", n), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				replies := make([]*Response, 0, n)
				for j := 0; j < n; j++ {
					replies = append(replies, &Response{Result: request.Value})
					resp, q := qspec.UseReqQF(request, replies)
					if q {
						_ = resp.GetResult()
					}
				}
			}
		})
		b.Run(fmt.Sprintf("IgnoreReq_%d", n), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				replies := make([]*Response, 0, n)
				for j := 0; j < n; j++ {
					replies = append(replies, &Response{Result: request.Value})
					resp, q := qspec.IgnoreReqQF(request, replies)
					if q {
						_ = resp.GetResult()
					}
				}
			}
		})
		b.Run(fmt.Sprintf("WithoutReq_%d", n), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				replies := make([]*Response, 0, n)
				for j := 0; j < n; j++ {
					replies = append(replies, &Response{Result: request.Value})
					resp, q := qspec.WithoutReqQF(replies)
					if q {
						_ = resp.GetResult()
					}
				}
			}
		})
	}
}

// TODO(meling) consider making these things generally available, at least for testing, perhaps putting some of these things in an internal package.

type portSupplier struct {
	p int
	sync.Mutex
}

func (p *portSupplier) get() int {
	p.Lock()
	_p := p.p
	p.p++
	p.Unlock()
	return _p
}

var supplier = portSupplier{p: 22332}

func getListener() (net.Listener, error) {
	return net.Listen("tcp", fmt.Sprintf(":%d", supplier.get()))
}

type testSrv struct {
	dummy int64
}

func (s testSrv) UseReq(_ context.Context, req *Request) (*Response, error) {
	return &Response{Result: req.GetValue()}, nil
}

func (s testSrv) IgnoreReq(_ context.Context, req *Request) (*Response, error) {
	return &Response{Result: req.GetValue()}, nil
}

func (s testSrv) WithoutReq(_ context.Context, req *Request) (*Response, error) {
	return &Response{Result: req.GetValue()}, nil
}

func setup(b *testing.B, numServers int) ([]*grpc.Server, *Manager, *Configuration) {
	quorum := numServers / 2
	servers := make([]*grpc.Server, numServers)
	addrs := make([]string, numServers)
	for i := 0; i < numServers; i++ {
		srv := grpc.NewServer()
		RegisterQuorumFunctionServer(srv, &testSrv{})
		lis, err := getListener()
		if err != nil {
			b.Fatalf("Failed to listen on port: %v", err)
		}
		addrs[i] = lis.Addr().String()
		servers[i] = srv
		go srv.Serve(lis)
	}

	// client setup
	man, err := NewManager(addrs,
		WithGrpcDialOptions(grpc.WithInsecure()),
		WithDialTimeout(10*time.Second),
	)
	if err != nil {
		b.Fatalf("Failed to create manager: %v", err)
	}
	c, err := man.NewConfiguration(man.NodeIDs(), &testQSpec{quorum: quorum})
	if err != nil {
		b.Fatalf("Failed to create configuration: %v", err)
	}
	return servers, man, c
}

func BenchmarkUseReq(b *testing.B) {
	servers, man, c := setup(b, 5)

	b.ReportAllocs()
	b.ResetTimer()

	// begin benchmarking
	for i := 0; i < b.N; i++ {
		resp, err := c.UseReq(context.Background(), &Request{Value: int64(requestValue)})
		if err != nil {
			b.Fatalf("UseReq error: %v", err)
		}
		_ = resp.GetResult()
	}

	man.Close()
	for _, srv := range servers {
		srv.Stop()
	}
}

func BenchmarkIgnoreReq(b *testing.B) {
	servers, man, c := setup(b, 5)

	b.ReportAllocs()
	b.ResetTimer()

	// begin benchmarking
	for i := 0; i < b.N; i++ {
		resp, err := c.IgnoreReq(context.Background(), &Request{Value: int64(requestValue)})
		if err != nil {
			b.Fatalf("IgnoreReq error: %v", err)
		}
		_ = resp.GetResult()
	}

	man.Close()
	for _, srv := range servers {
		srv.Stop()
	}
}

func BenchmarkWithoutReq(b *testing.B) {
	servers, man, c := setup(b, 5)

	b.ReportAllocs()
	b.ResetTimer()

	// begin benchmarking
	for i := 0; i < b.N; i++ {
		resp, err := c.WithoutReq(context.Background(), &Request{Value: int64(requestValue)})
		if err != nil {
			b.Fatalf("WithoutReq error: %v", err)
		}
		_ = resp.GetResult()
	}

	man.Close()
	for _, srv := range servers {
		srv.Stop()
	}
}
<commit_msg>removed useless import names added by code editor, or something<commit_after>package qf

import (
	"context"
	"fmt"
	"net"
	"sync"
	"testing"
	"time"

	"google.golang.org/grpc"
)

const requestValue = 0x1001_1001

type testQSpec struct {
	quorum int
}

func (q testQSpec) UseReqQF(in *Request, replies []*Response) (*Response, bool) {
	if len(replies) < q.quorum {
		return nil, false
	}
	expected := in.GetValue()
	for _, reply := range replies {
		if expected != reply.GetResult() {
			return reply, true
		}
	}
	return replies[0], true
}

func (q testQSpec) IgnoreReqQF(_ *Request, replies []*Response) (*Response, bool) {
	if len(replies) < q.quorum {
		return nil, false
	}
	expected := int64(requestValue)
	for _, reply := range replies {
		if expected != reply.GetResult() {
			return reply, true
		}
	}
	return replies[0], true
}

func (q testQSpec) WithoutReqQF(replies []*Response) (*Response, bool) {
	if len(replies) < q.quorum {
		return nil, false
	}
	expected := int64(requestValue)
	for _, reply := range replies {
		if expected != reply.GetResult() {
			return reply, true
		}
	}
	return replies[0], true
}

func BenchmarkQF(b *testing.B) {
	for n := 3; n < 20; n += 2 {
		quorum := n / 2
		qspec := &testQSpec{quorum: quorum}
		request := &Request{Value: 1}

		b.Run(fmt.Sprintf("UseReq_%d", n), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				replies := make([]*Response, 0, n)
				for j := 0; j < n; j++ {
					replies = append(replies, &Response{Result: request.Value})
					resp, q := qspec.UseReqQF(request, replies)
					if q {
						_ = resp.GetResult()
					}
				}
			}
		})
		b.Run(fmt.Sprintf("IgnoreReq_%d", n), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				replies := make([]*Response, 0, n)
				for j := 0; j < n; j++ {
					replies = append(replies, &Response{Result: request.Value})
					resp, q := qspec.IgnoreReqQF(request, replies)
					if q {
						_ = resp.GetResult()
					}
				}
			}
		})
		b.Run(fmt.Sprintf("WithoutReq_%d", n), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				replies := make([]*Response, 0, n)
				for j := 0; j < n; j++ {
					replies = append(replies, &Response{Result: request.Value})
					resp, q := qspec.WithoutReqQF(replies)
					if q {
						_ = resp.GetResult()
					}
				}
			}
		})
	}
}

// TODO(meling) consider making these things generally available, at least for testing, perhaps putting some of these things in an internal package.

type portSupplier struct {
	p int
	sync.Mutex
}

func (p *portSupplier) get() int {
	p.Lock()
	_p := p.p
	p.p++
	p.Unlock()
	return _p
}

var supplier = portSupplier{p: 22332}

func getListener() (net.Listener, error) {
	return net.Listen("tcp", fmt.Sprintf(":%d", supplier.get()))
}

type testSrv struct {
	dummy int64
}

func (s testSrv) UseReq(_ context.Context, req *Request) (*Response, error) {
	return &Response{Result: req.GetValue()}, nil
}

func (s testSrv) IgnoreReq(_ context.Context, req *Request) (*Response, error) {
	return &Response{Result: req.GetValue()}, nil
}

func (s testSrv) WithoutReq(_ context.Context, req *Request) (*Response, error) {
	return &Response{Result: req.GetValue()}, nil
}

func setup(b *testing.B, numServers int) ([]*grpc.Server, *Manager, *Configuration) {
	quorum := numServers / 2
	servers := make([]*grpc.Server, numServers)
	addrs := make([]string, numServers)
	for i := 0; i < numServers; i++ {
		srv := grpc.NewServer()
		RegisterQuorumFunctionServer(srv, &testSrv{})
		lis, err := getListener()
		if err != nil {
			b.Fatalf("Failed to listen on port: %v", err)
		}
		addrs[i] = lis.Addr().String()
		servers[i] = srv
		go srv.Serve(lis)
	}

	// client setup
	man, err := NewManager(addrs,
		WithGrpcDialOptions(grpc.WithInsecure()),
		WithDialTimeout(10*time.Second),
	)
	if err != nil {
		b.Fatalf("Failed to create manager: %v", err)
	}
	c, err := man.NewConfiguration(man.NodeIDs(), &testQSpec{quorum: quorum})
	if err != nil {
		b.Fatalf("Failed to create configuration: %v", err)
	}
	return servers, man, c
}

func BenchmarkUseReq(b *testing.B) {
	servers, man, c := setup(b, 5)

	b.ReportAllocs()
	b.ResetTimer()

	// begin benchmarking
	for i := 0; i < b.N; i++ {
		resp, err := c.UseReq(context.Background(), &Request{Value: int64(requestValue)})
		if err != nil {
			b.Fatalf("UseReq error: %v", err)
		}
		_ = resp.GetResult()
	}

	man.Close()
	for _, srv := range servers {
		srv.Stop()
	}
}

func BenchmarkIgnoreReq(b *testing.B) {
	servers, man, c := setup(b, 5)

	b.ReportAllocs()
	b.ResetTimer()

	// begin benchmarking
	for i := 0; i < b.N; i++ {
		resp, err := c.IgnoreReq(context.Background(), &Request{Value: int64(requestValue)})
		if err != nil {
			b.Fatalf("IgnoreReq error: %v", err)
		}
		_ = resp.GetResult()
	}

	man.Close()
	for _, srv := range servers {
		srv.Stop()
	}
}

func BenchmarkWithoutReq(b *testing.B) {
	servers, man, c := setup(b, 5)

	b.ReportAllocs()
	b.ResetTimer()

	// begin benchmarking
	for i := 0; i < b.N; i++ {
		resp, err := c.WithoutReq(context.Background(), &Request{Value: int64(requestValue)})
		if err != nil {
			b.Fatalf("WithoutReq error: %v", err)
		}
		_ = resp.GetResult()
	}

	man.Close()
	for _, srv := range servers {
		srv.Stop()
	}
}
<|endoftext|>


<commit_before>package collectors

import (
	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/wmi"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_physical_disk_windows})
	collectors = append(collectors, &IntervalCollector{F: c_diskspace_windows})
}

func c_diskspace_windows() (opentsdb.MultiDataPoint, error) {
	const megabyte = 1048576
	var dst []Win32_PerfFormattedData_PerfDisk_LogicalDisk
	var q = wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
	err := queryWmi(q, &dst)
	if err != nil {
		return nil, err
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		Add(&md, "win.disk.fs.space_free", v.FreeMegabytes*megabyte, opentsdb.TagSet{"partition": v.Name}, metadata.Gauge, metadata.Bytes, "")
		Add(&md, osDiskFree, v.FreeMegabytes*megabyte, opentsdb.TagSet{"disk": v.Name}, metadata.Gauge, metadata.Bytes, "")
		if v.PercentFreeSpace != 0 {
			space_total := v.FreeMegabytes * megabyte * 100 / v.PercentFreeSpace
			space_used := space_total - v.FreeMegabytes*megabyte
			Add(&md, "win.disk.fs.space_total", space_total, opentsdb.TagSet{"partition": v.Name}, metadata.Gauge, metadata.Bytes, "")
			Add(&md, "win.disk.fs.space_used", space_used, opentsdb.TagSet{"partition": v.Name}, metadata.Gauge, metadata.Bytes, "")
			Add(&md, osDiskTotal, space_total, opentsdb.TagSet{"disk": v.Name}, metadata.Gauge, metadata.Bytes, "")
			Add(&md, osDiskUsed, space_used, opentsdb.TagSet{"disk": v.Name}, metadata.Gauge, metadata.Bytes, "")
		}

		Add(&md, "win.disk.fs.percent_free", v.PercentFreeSpace, opentsdb.TagSet{"partition": v.Name}, metadata.Gauge, metadata.Pct, "")
		Add(&md, osDiskPctFree, v.PercentFreeSpace, opentsdb.TagSet{"disk": v.Name}, metadata.Gauge, metadata.Pct, "")
	}
	return md, nil
}

type Win32_PerfFormattedData_PerfDisk_LogicalDisk struct {
	FreeMegabytes    uint64
	Name             string
	PercentFreeSpace uint64
}

func c_physical_disk_windows() (opentsdb.MultiDataPoint, error) {
	var dst []Win32_PerfRawData_PerfDisk_PhysicalDisk
	var q = wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
	err := queryWmi(q, &dst)
	if err != nil {
		return nil, err
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		Add(&md, "win.disk.duration", v.AvgDiskSecPerRead/1000000, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.MilliSecond, "Time, in milliseconds, of a read from the disk.")
		Add(&md, "win.disk.duration", v.AvgDiskSecPerWrite/1000000, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.MilliSecond, "Time, in milliseconds, of a write to the disk.")
		Add(&md, "win.disk.queue", v.AvgDiskReadQueueLength/10000000, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.Operation, "Number of read requests that were queued for the disk.")
		Add(&md, "win.disk.queue", v.AvgDiskWriteQueueLength/10000000, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.Operation, "Number of write requests that were queued for the disk.")
		Add(&md, "win.disk.ops", v.DiskReadsPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.PerSecond, "Number of read operations on the disk.")
		Add(&md, "win.disk.ops", v.DiskWritesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.PerSecond, "Number of write operations on the disk.")
		Add(&md, "win.disk.bytes", v.DiskReadBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.BytesPerSecond, "Number of bytes read from the disk.")
		Add(&md, "win.disk.bytes", v.DiskWriteBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.BytesPerSecond, "Number of bytes written to the disk.")
		Add(&md, "win.disk.percent_time", v.PercentDiskReadTime/100000, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.Pct, "Percentage of time that the disk was busy servicing read requests.")
		Add(&md, "win.disk.percent_time", v.PercentDiskWriteTime/100000, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.Pct, "Percentage of time that the disk was busy servicing write requests.")
		Add(&md, "win.disk.spltio", v.SplitIOPerSec, opentsdb.TagSet{"disk": v.Name}, metadata.Counter, metadata.PerSecond, "Number of requests to the disk that were split into multiple requests due to size or fragmentation.")
	}
	return md, nil
}

//See msdn for counter types http://msdn.microsoft.com/en-us/library/ms804035.aspx
type Win32_PerfRawData_PerfDisk_PhysicalDisk struct {
	AvgDiskReadQueueLength  uint64
	AvgDiskSecPerRead       uint32
	AvgDiskSecPerWrite      uint32
	AvgDiskWriteQueueLength uint64
	DiskReadBytesPerSec     uint64
	DiskReadsPerSec         uint32
	DiskWriteBytesPerSec    uint64
	DiskWritesPerSec        uint32
	Name                    string
	PercentDiskReadTime     uint64
	PercentDiskWriteTime    uint64
	SplitIOPerSec           uint32
}
<commit_msg>cmd/scollector: Move 100nS Sample conversion factors to a constant with documentation<commit_after>package collectors

import (
	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/wmi"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_physical_disk_windows})
	collectors = append(collectors, &IntervalCollector{F: c_diskspace_windows})
}

const (
	//Converts 100nS samples to 1S samples
	winDisk100nS_1S = 10000000

	//Converts 100nS samples to 1mS samples
	winDisk100nS_1mS = 1000000

	//Converts 100nS samples to 0-100 Percent samples
	winDisk100nS_Pct = 100000
)

func c_diskspace_windows() (opentsdb.MultiDataPoint, error) {
	const megabyte = 1048576
	var dst []Win32_PerfFormattedData_PerfDisk_LogicalDisk
	var q = wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
	err := queryWmi(q, &dst)
	if err != nil {
		return nil, err
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		Add(&md, "win.disk.fs.space_free", v.FreeMegabytes*megabyte, opentsdb.TagSet{"partition": v.Name}, metadata.Gauge, metadata.Bytes, "")
		Add(&md, osDiskFree, v.FreeMegabytes*megabyte, opentsdb.TagSet{"disk": v.Name}, metadata.Gauge, metadata.Bytes, "")
		if v.PercentFreeSpace != 0 {
			space_total := v.FreeMegabytes * megabyte * 100 / v.PercentFreeSpace
			space_used := space_total - v.FreeMegabytes*megabyte
			Add(&md, "win.disk.fs.space_total", space_total, opentsdb.TagSet{"partition": v.Name}, metadata.Gauge, metadata.Bytes, "")
			Add(&md, "win.disk.fs.space_used", space_used, opentsdb.TagSet{"partition": v.Name}, metadata.Gauge, metadata.Bytes, "")
			Add(&md, osDiskTotal, space_total, opentsdb.TagSet{"disk": v.Name}, metadata.Gauge, metadata.Bytes, "")
			Add(&md, osDiskUsed, space_used, opentsdb.TagSet{"disk": v.Name}, metadata.Gauge, metadata.Bytes, "")
		}

		Add(&md, "win.disk.fs.percent_free", v.PercentFreeSpace, opentsdb.TagSet{"partition": v.Name}, metadata.Gauge, metadata.Pct, "")
		Add(&md, osDiskPctFree, v.PercentFreeSpace, opentsdb.TagSet{"disk": v.Name}, metadata.Gauge, metadata.Pct, "")
	}
	return md, nil
}

type Win32_PerfFormattedData_PerfDisk_LogicalDisk struct {
	FreeMegabytes    uint64
	Name             string
	PercentFreeSpace uint64
}

func c_physical_disk_windows() (opentsdb.MultiDataPoint, error) {
	var dst []Win32_PerfRawData_PerfDisk_PhysicalDisk
	var q = wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
	err := queryWmi(q, &dst)
	if err != nil {
		return nil, err
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		Add(&md, "win.disk.duration", v.AvgDiskSecPerRead/winDisk100nS_1mS, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.MilliSecond, "Time, in milliseconds, of a read from the disk.")
		Add(&md, "win.disk.duration", v.AvgDiskSecPerWrite/winDisk100nS_1mS, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.MilliSecond, "Time, in milliseconds, of a write to the disk.")
		Add(&md, "win.disk.queue", v.AvgDiskReadQueueLength/winDisk100nS_1S, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.Operation, "Number of read requests that were queued for the disk.")
		Add(&md, "win.disk.queue", v.AvgDiskWriteQueueLength/winDisk100nS_1S, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.Operation, "Number of write requests that were queued for the disk.")
		Add(&md, "win.disk.ops", v.DiskReadsPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.PerSecond, "Number of read operations on the disk.")
		Add(&md, "win.disk.ops", v.DiskWritesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.PerSecond, "Number of write operations on the disk.")
		Add(&md, "win.disk.bytes", v.DiskReadBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.BytesPerSecond, "Number of bytes read from the disk.")
		Add(&md, "win.disk.bytes", v.DiskWriteBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.BytesPerSecond, "Number of bytes written to the disk.")
		Add(&md, "win.disk.percent_time", v.PercentDiskReadTime/winDisk100nS_Pct, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.Pct, "Percentage of time that the disk was busy servicing read requests.")
		Add(&md, "win.disk.percent_time", v.PercentDiskWriteTime/winDisk100nS_Pct, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.Pct, "Percentage of time that the disk was busy servicing write requests.")
		Add(&md, "win.disk.spltio", v.SplitIOPerSec, opentsdb.TagSet{"disk": v.Name}, metadata.Counter, metadata.PerSecond, "Number of requests to the disk that were split into multiple requests due to size or fragmentation.")
	}
	return md, nil
}

//See msdn for counter types http://msdn.microsoft.com/en-us/library/ms804035.aspx
type Win32_PerfRawData_PerfDisk_PhysicalDisk struct {
	AvgDiskReadQueueLength  uint64
	AvgDiskSecPerRead       uint32
	AvgDiskSecPerWrite      uint32
	AvgDiskWriteQueueLength uint64
	DiskReadBytesPerSec     uint64
	DiskReadsPerSec         uint32
	DiskWriteBytesPerSec    uint64
	DiskWritesPerSec        uint32
	Name                    string
	PercentDiskReadTime     uint64
	PercentDiskWriteTime    uint64
	SplitIOPerSec           uint32
}
<|endoftext|>


<commit_before>package compressor

import (
	"archive/tar"
	"compress/gzip"
	"io"
	"io/ioutil"
)

type TarCompressor struct{}

func NewTarCompressor() *TarCompressor {
	return &TarCompressor{}
}

func (c *TarCompressor) Compress(compressedFile io.Writer, targetDir string, files []string) error {

	gw := gzip.NewWriter(compressedFile)
	defer gw.Close()
	tw := tar.NewWriter(gw)

	for _, file := range files {
		body, _ := ioutil.ReadFile(targetDir + "/" + file)
		hdr := &tar.Header{
			Name: file,
			Mode: 0600,
			Size: int64(len(body)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}

		if _, err := tw.Write(body); err != nil {
			return err
		}
	}

	if err := tw.Close(); err != nil {
		return err
	}

	return nil
}
<commit_msg>tarの場合でもディレクトリが格納されるように対応<commit_after>package compressor

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"log"
	"os"
)

type TarCompressor struct{}

func NewTarCompressor() *TarCompressor {
	return &TarCompressor{}
}

func (c *TarCompressor) Compress(compressedFile io.Writer, targetDir string, files []string) error {

	gw := gzip.NewWriter(compressedFile)
	defer gw.Close()

	tw := tar.NewWriter(gw)

	for _, filename := range files {
		filepath := fmt.Sprintf("%s/%s", targetDir, filename)

		info, err := os.Stat(filepath)
		if err != nil {
			return err
		}

		if info.IsDir() {
			continue
		}

		file, err := os.Open(filepath)
		if err != nil {
			return err
		}
		defer file.Close()

		hdr, err := tar.FileInfoHeader(info, "")
		if err != nil {
			return err
		}

		hdr.Name = filename
		if err := tw.WriteHeader(hdr); err != nil {
			log.Print(err)
			return err
		}

		if _, err = io.Copy(tw, file); err != nil {
			return err
		}
	}

	if err := tw.Close(); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package crawler

import (
	"bytes"
	"compress/zlib"
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"math"
)

// CheckDb - check db
func CheckDb() error {
	db := new(DB)
	err := db.Open()
	if err != nil {
		return err
	}

	defer db.Close()

	isSuccess := true
	maxLen := 0
	var maxLenURL string
	minLen := math.MaxUint32
	var minLenURL string
	hashMap := make(map[[16]byte]string)
	err = db.View(func(tx *Tx) error {
		c := tx.Bucket(DbBucketContents).Cursor()

		var content DbContent
		for url, contentBytes := c.First(); url != nil; url, contentBytes = c.Next() {
			_, err := content.UnmarshalMsg(contentBytes)
			if err != nil {
				fmt.Printf("Error unmarshal value in db for url %s, message: %s\n", url, err)
				isSuccess = false
				continue
			}

			r, err := zlib.NewReader(bytes.NewReader(content.Content))
			if err != nil {
				fmt.Printf("Error unzip content for url %s, message: %s\n", url, err)
				isSuccess = false
				continue
			}
			contentOrig, err := ioutil.ReadAll(r)
			r.Close()
			if err != nil {
				fmt.Printf("Error read unzip content for url %s, message: %s\n", url, err)
				isSuccess = false
				continue
			}

			lenContent := len(contentOrig)
			if lenContent < minLen {
				minLen = lenContent
				minLenURL = string(url)
			}
			if lenContent > maxLen {
				maxLen = lenContent
				maxLenURL = string(url)
			}

			hash := md5.Sum(contentOrig)
			if hash != content.Hash {
				fmt.Printf("Error content hash does not match for url %s\n", url)
				isSuccess = false
				continue
			}

			if val, ok := hashMap[hash]; ok {
				fmt.Printf("Duplicated pages content:\n%s\n%s\n\n", url, val)
			} else {
				hashMap[hash] = string(url[:])
			}
		}

		return nil
	})

	if isSuccess {
		fmt.Printf("Min len = %d (%s)\n", minLen, minLenURL)
		fmt.Printf("Max len = %d (%s)\n", maxLen, maxLenURL)
		fmt.Println("Checking ended successfully")
	}
	return err
}
<commit_msg>рефакторинг db_check<commit_after>package crawler

import (
	"bytes"
	"compress/zlib"
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"math"
)

// CheckDb - check db
func CheckDb() error {
	db := new(DB)
	err := db.Open()
	if err != nil {
		return err
	}

	defer db.Close()

	isSuccess := true
	maxLen := 0
	var maxLenURL string
	minLen := math.MaxUint32
	var minLenURL string
	hashMap := make(map[[16]byte]string)
	err = db.View(func(tx *Tx) error {
		c := tx.Bucket(DbBucketContents).Cursor()

		var content DbContent
		for url, contentBytes := c.First(); url != nil; url, contentBytes = c.Next() {
			_, err := content.UnmarshalMsg(contentBytes)
			if err != nil {
				fmt.Printf("Error unmarshal value in db for url %s, message: %s\n", urlRaw, err)
				isSuccess = false
				continue
			}

			r, err := zlib.NewReader(bytes.NewReader(content.Content))
			if err != nil {
				fmt.Printf("Error unzip content for url %s, message: %s\n", urlRaw, err)
				isSuccess = false
				continue
			}
			contentOrig, err := ioutil.ReadAll(r)
			r.Close()
			if err != nil {
				fmt.Printf("Error read unzip content for url %s, message: %s\n", urlRaw, err)
				isSuccess = false
				continue
			}

			lenContent := len(contentOrig)
			if lenContent < minLen {
				minLen = lenContent
				minLenURL = string(urlRaw)
			}
			if lenContent > maxLen {
				maxLen = lenContent
				maxLenURL = string(urlRaw)
			}

			hash := md5.Sum(contentOrig)
			if hash != content.Hash {
				fmt.Printf("Error content hash does not match for url %s\n", urlRaw)
				isSuccess = false
				continue
			}

			if val, ok := hashMap[hash]; ok {
				fmt.Printf("Duplicated pages content:\n%s\n%s\n\n", urlRaw, val)
			} else {
				hashMap[hash] = string(urlRaw)
			}
		})

		return nil
	})

	if isSuccess {
		fmt.Printf("Min len = %d (%s)\n", minLen, minLenURL)
		fmt.Printf("Max len = %d (%s)\n", maxLen, maxLenURL)
		fmt.Println("Checking ended successfully")
	}
	return err
}
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.67"
<commit_msg>functions: 0.3.68 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.68"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.59"
<commit_msg>functions: 0.3.60 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.60"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.236"
<commit_msg>fnserver: 0.3.237 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.237"
<|endoftext|>


<commit_before>package gateway

import (
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"testing"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
)

// newTestingGateway returns a gateway read to use in a testing environment.
func newTestingGateway(name string, t *testing.T) *Gateway {
	g, err := New("localhost:0", build.TempDir("gateway", name))
	if err != nil {
		t.Fatal(err)
	}
	// Manually add myAddr as a node. This is necessary because g.addNode
	// rejects loopback addresses.
	id := g.mu.Lock()
	g.nodes[g.myAddr] = struct{}{}
	g.mu.Unlock(id)
	return g
}

func TestAddress(t *testing.T) {
	g := newTestingGateway("TestAddress", t)
	defer g.Close()
	if g.Address() != g.myAddr {
		t.Fatal("Address does not return g.myAddr")
	}
	port := modules.NetAddress(g.listener.Addr().String()).Port()
	expAddr := modules.NetAddress(net.JoinHostPort("::", port))
	if g.Address() != expAddr {
		t.Fatalf("Wrong address: expected %v, got %v", expAddr, g.Address())
	}
}

func TestPeers(t *testing.T) {
	g1 := newTestingGateway("TestRPC1", t)
	defer g1.Close()
	g2 := newTestingGateway("TestRPC2", t)
	defer g2.Close()
	err := g1.Connect(g2.Address())
	if err != nil {
		t.Fatal("failed to connect:", err)
	}
	peers := g1.Peers()
	if len(peers) != 1 || peers[0].NetAddress != g2.Address() {
		t.Fatal("g1 has bad peer list:", peers)
	}
	err = g1.Disconnect(g2.Address())
	if err != nil {
		t.Fatal("failed to disconnect:", err)
	}
	peers = g1.Peers()
	if len(peers) != 0 {
		t.Fatal("g1 has peers after disconnect:", peers)
	}
}

func TestNew(t *testing.T) {
	if _, err := New("", ""); err == nil {
		t.Fatal("expecting persistDir error, got nil")
	}
	if _, err := New("localhost:0", ""); err == nil {
		t.Fatal("expecting persistDir error, got nil")
	}
	if g, err := New("foo", build.TempDir("gateway", "TestNew1")); err == nil {
		t.Fatal("expecting listener error, got nil", g.myAddr)
	}
	// create corrupted nodes.json
	dir := build.TempDir("gateway", "TestNew2")
	os.MkdirAll(dir, 0700)
	err := ioutil.WriteFile(filepath.Join(dir, "nodes.json"), []byte{1, 2, 3}, 0660)
	if err != nil {
		t.Fatal("couldn't create corrupted file:", err)
	}
	if _, err := New("localhost:0", dir); err == nil {
		t.Fatal("expected load error, got nil")
	}
}
<commit_msg>Change TestAddress to make it pass with loopback addresses<commit_after>package gateway

import (
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"testing"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
)

// newTestingGateway returns a gateway read to use in a testing environment.
func newTestingGateway(name string, t *testing.T) *Gateway {
	g, err := New("localhost:0", build.TempDir("gateway", name))
	if err != nil {
		t.Fatal(err)
	}
	// Manually add myAddr as a node. This is necessary because g.addNode
	// rejects loopback addresses.
	id := g.mu.Lock()
	g.nodes[g.myAddr] = struct{}{}
	g.mu.Unlock(id)
	return g
}

// TestAddress tests that Gateway.Address returns the address of its listener.
// Also tests that the address is not unspecified and is a loopback address.
// The address must be a loopback address for testing.
func TestAddress(t *testing.T) {
	g := newTestingGateway("TestAddress", t)
	defer g.Close()
	if g.Address() != g.myAddr {
		t.Fatal("Address does not return g.myAddr")
	}
	if g.Address() != modules.NetAddress(g.listener.Addr().String()) {
		t.Fatalf("wrong address: expected %v, got %v", g.listener.Addr(), g.Address())
	}
	host := modules.NetAddress(g.listener.Addr().String()).Host()
	ip := net.ParseIP(host)
	if ip.IsUnspecified() {
		t.Fatal("expected a non-unspecified address")
	}
	if !ip.IsLoopback() {
		t.Fatal("expected a loopback address")
	}
}

func TestPeers(t *testing.T) {
	g1 := newTestingGateway("TestRPC1", t)
	defer g1.Close()
	g2 := newTestingGateway("TestRPC2", t)
	defer g2.Close()
	err := g1.Connect(g2.Address())
	if err != nil {
		t.Fatal("failed to connect:", err)
	}
	peers := g1.Peers()
	if len(peers) != 1 || peers[0].NetAddress != g2.Address() {
		t.Fatal("g1 has bad peer list:", peers)
	}
	err = g1.Disconnect(g2.Address())
	if err != nil {
		t.Fatal("failed to disconnect:", err)
	}
	peers = g1.Peers()
	if len(peers) != 0 {
		t.Fatal("g1 has peers after disconnect:", peers)
	}
}

func TestNew(t *testing.T) {
	if _, err := New("", ""); err == nil {
		t.Fatal("expecting persistDir error, got nil")
	}
	if _, err := New("localhost:0", ""); err == nil {
		t.Fatal("expecting persistDir error, got nil")
	}
	if g, err := New("foo", build.TempDir("gateway", "TestNew1")); err == nil {
		t.Fatal("expecting listener error, got nil", g.myAddr)
	}
	// create corrupted nodes.json
	dir := build.TempDir("gateway", "TestNew2")
	os.MkdirAll(dir, 0700)
	err := ioutil.WriteFile(filepath.Join(dir, "nodes.json"), []byte{1, 2, 3}, 0660)
	if err != nil {
		t.Fatal("couldn't create corrupted file:", err)
	}
	if _, err := New("localhost:0", dir); err == nil {
		t.Fatal("expected load error, got nil")
	}
}
<|endoftext|>