<commit_before>package main

import (
	"expvar"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"os/user"
	"runtime"
	"strconv"
	"syscall"

	"github.com/lomik/zapwriter"
	daemon "github.com/sevlyar/go-daemon"
	"go.uber.org/zap"

	"github.com/lomik/go-carbon/carbon"
	"github.com/lomik/go-carbon/points"

	_ "net/http/pprof"
)

// Version of go-carbon
const Version = "0.12.0-rc1"

var BuildVersion = "(development version)"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")

	printVersion := flag.Bool("version", false, "Print version")

	isDaemon := flag.Bool("daemon", false, "Run in background")
	pidfile := flag.String("pidfile", "", "Pidfile path (only for daemon)")

	cat := flag.String("cat", "", "Print cache dump file")

	flag.Parse()

	if *printVersion {
		fmt.Println(Version)
		return
	}

	if *cat != "" {
		err = points.ReadFromFile(*cat, func(p *points.Points) {
			for _, d := range p.Data { // every metric point
				fmt.Printf("%s %v %v\n", p.Metric, d.Value, d.Timestamp)
			}
		})
		if err != nil {
			log.Fatal(err)
		}
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	cfg := app.Config

	var runAsUser *user.User
	if cfg.Common.User != "" {
		runAsUser, err = user.Lookup(cfg.Common.User)
		if err != nil {
			log.Fatal(err)
		}
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	for i := 0; i < len(cfg.Logging); i++ {
		if err := zapwriter.PrepareFileForUser(cfg.Logging[i].File, runAsUser); err != nil {
			log.Fatal(err)
		}
	}

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	if *isDaemon {
		runtime.LockOSThread()

		context := new(daemon.Context)
		if *pidfile != "" {
			context.PidFileName = *pidfile
			context.PidFilePerm = 0644
		}

		if runAsUser != nil {
			uid, err := strconv.ParseInt(runAsUser.Uid, 10, 0)
			if err != nil {
				mainLogger.Fatal(err.Error())
			}

			gid, err := strconv.ParseInt(runAsUser.Gid, 10, 0)
			if err != nil {
				mainLogger.Fatal(err.Error())
			}

			context.Credential = &syscall.Credential{
				Uid: uint32(uid),
				Gid: uint32(gid),
			}
		}

		child, _ := context.Reborn()

		if child != nil {
			return
		}
		defer context.Release()

		runtime.UnlockOSThread()
	}
	/* CONFIG end */

	// pprof
	// httpStop := func() {}
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal(err.Error())
		}

		expvar.NewString("GoVersion").Set(runtime.Version())
		expvar.NewString("BuildVersion").Set(BuildVersion)
		expvar.Publish("Config", expvar.Func(func() interface{} { return cfg }))
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal(err.Error())
	} else {
		mainLogger.Info("started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR2)
		for {
			<-c
			app.DumpStop()
			os.Exit(0)
		}
	}()

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGHUP)
		for {
			<-c
			mainLogger.Info("HUP received. Reload config")
			if err := app.ReloadConfig(); err != nil {
				mainLogger.Error("config reload failed", zap.Error(err))
			} else {
				mainLogger.Info("config successfully reloaded")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("stopped")
}
<commit_msg>version 0.12.0<commit_after>package main

import (
	"expvar"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"os/user"
	"runtime"
	"strconv"
	"syscall"

	"github.com/lomik/zapwriter"
	daemon "github.com/sevlyar/go-daemon"
	"go.uber.org/zap"

	"github.com/lomik/go-carbon/carbon"
	"github.com/lomik/go-carbon/points"

	_ "net/http/pprof"
)

// Version of go-carbon
const Version = "0.12.0"

var BuildVersion = "(development version)"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")

	printVersion := flag.Bool("version", false, "Print version")

	isDaemon := flag.Bool("daemon", false, "Run in background")
	pidfile := flag.String("pidfile", "", "Pidfile path (only for daemon)")

	cat := flag.String("cat", "", "Print cache dump file")

	flag.Parse()

	if *printVersion {
		fmt.Println(Version)
		return
	}

	if *cat != "" {
		err = points.ReadFromFile(*cat, func(p *points.Points) {
			for _, d := range p.Data { // every metric point
				fmt.Printf("%s %v %v\n", p.Metric, d.Value, d.Timestamp)
			}
		})
		if err != nil {
			log.Fatal(err)
		}
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	cfg := app.Config

	var runAsUser *user.User
	if cfg.Common.User != "" {
		runAsUser, err = user.Lookup(cfg.Common.User)
		if err != nil {
			log.Fatal(err)
		}
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	for i := 0; i < len(cfg.Logging); i++ {
		if err := zapwriter.PrepareFileForUser(cfg.Logging[i].File, runAsUser); err != nil {
			log.Fatal(err)
		}
	}

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	if *isDaemon {
		runtime.LockOSThread()

		context := new(daemon.Context)
		if *pidfile != "" {
			context.PidFileName = *pidfile
			context.PidFilePerm = 0644
		}

		if runAsUser != nil {
			uid, err := strconv.ParseInt(runAsUser.Uid, 10, 0)
			if err != nil {
				mainLogger.Fatal(err.Error())
			}

			gid, err := strconv.ParseInt(runAsUser.Gid, 10, 0)
			if err != nil {
				mainLogger.Fatal(err.Error())
			}

			context.Credential = &syscall.Credential{
				Uid: uint32(uid),
				Gid: uint32(gid),
			}
		}

		child, _ := context.Reborn()

		if child != nil {
			return
		}
		defer context.Release()

		runtime.UnlockOSThread()
	}
	/* CONFIG end */

	// pprof
	// httpStop := func() {}
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal(err.Error())
		}

		expvar.NewString("GoVersion").Set(runtime.Version())
		expvar.NewString("BuildVersion").Set(BuildVersion)
		expvar.Publish("Config", expvar.Func(func() interface{} { return cfg }))
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal(err.Error())
	} else {
		mainLogger.Info("started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR2)
		for {
			<-c
			app.DumpStop()
			os.Exit(0)
		}
	}()

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGHUP)
		for {
			<-c
			mainLogger.Info("HUP received. Reload config")
			if err := app.ReloadConfig(); err != nil {
				mainLogger.Error("config reload failed", zap.Error(err))
			} else {
				mainLogger.Info("config successfully reloaded")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("stopped")
}
<|endoftext|>


<commit_before>package sxchange

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"sync"
	"time"
	"unsafe"
)

const (
	verHI = 1
	verLO = 0
)

// CB type for callback function for each type of transfered data
type CB func([]byte, *Connection)

// DataTypeCB describes
type DataTypeCB struct {
	SizeBytes int8  // -1 for fixed size, 0 for no data (ping?), 1 for 0-255 bytes, 2 for 0-65535 bytes...
	FixedSize int32 // if some struct has fixed fize no other header needed
	Callback  CB    // which function to run after data received
}

// Connection informaion both for client or server
type Connection struct {
	conn         *net.TCPConn         // real connection
	writeMutex   sync.Mutex           // we need to lock on write operations
	Types        map[uint8]DataTypeCB // map of type->params&callback
	KeepAlive    time.Duration        // tcp keep-alive
	ReadTimeout  time.Duration        // we need to receive new message at least once per this duration
	WriteTimeout time.Duration        // maximal duration for every write operation
	MaxSize      uint32               // maximum number of bytes for one record/message
}

const (
	initialPacketSize = 1292 // we need to compare it manualy to be sure that compiler doesn't change aligment!
)

type initialPacket struct {
	Header    [8]uint8 // sxchngXY where X.Y is version of protocol
	MaxSize   uint32
	SizeBytes [256]int8
	FixedSize [256]int32
}

func readAll(conn *net.TCPConn, buf []byte, size int, timeout time.Duration) error {

	totalDeadline := time.Now().Add(timeout)
	conn.SetReadDeadline(totalDeadline)
	i, err := io.ReadFull(conn, buf[:size])
	if nil != err {
		return err
	}
	if i != size {
		return errors.New("Partial read")
	}

	return nil
}

func writeAll(conn *net.TCPConn, buf []byte, size int, timeout time.Duration) error {
	totalDeadline := time.Now().Add(timeout)

	for i := 0; i < size; i++ {
		conn.SetWriteDeadline(totalDeadline)
		j, err := conn.Write(buf[i:size])
		i += j
		if nil != err {
			return err
		}
	}

	return nil
}

func (c *Connection) initConnection() error {
	if c.KeepAlive != 0 {
		c.conn.SetKeepAlivePeriod(c.KeepAlive)
		c.conn.SetKeepAlive(true)
	}

	// initial packets exchange
	initial := c.prepareInitialPacket()
	initialBuf := (*[initialPacketSize]byte)(unsafe.Pointer(&initial))[:]
	received := initialPacket{}
	receivedBuf := (*[initialPacketSize]byte)(unsafe.Pointer(&received))[:]

	// send out version
	err := writeAll(c.conn, initialBuf, initialPacketSize, c.WriteTimeout)
	if nil != err {
		return err
	}

	err = readAll(c.conn, receivedBuf, initialPacketSize, c.ReadTimeout)
	if nil != err {
		return err
	}

	if received.Header != initial.Header {
		return errors.New("HS error - different protocol version")
	}

	if received.MaxSize != initial.MaxSize {
		return errors.New("HS error - different message maximal size")
	}

	if (received.FixedSize != initial.FixedSize) || (received.MaxSize != received.MaxSize) {
		return errors.New("HS error - different set of datatypes")
	}

	return nil
}

// internal run function
func (c *Connection) run() error {

	defer c.conn.Close()

	var (
		sbuf [4]byte
		t    [1]byte
		msg  = make([]byte, 0, c.MaxSize)
	)

	for {
		c.conn.SetReadDeadline(time.Now().Add(c.ReadTimeout))
		i, err := c.conn.Read(t[:])
		if nil != err {
			return err
		}
		if 0 == i { // TODO: check if this is an error?..
			continue
		}

		dt, ok := c.Types[t[0]]

		if !ok {
			return fmt.Errorf("Structure type %d is not defined", t[0])
		}

		var size2read int
		if dt.SizeBytes > 0 {
			err = readAll(c.conn, sbuf[0:dt.SizeBytes], int(dt.SizeBytes), c.ReadTimeout)
			if nil != err {
				return err
			}
		}

		switch dt.SizeBytes {
		case -1:
			size2read = int(dt.FixedSize)
		case 0:
			size2read = 0
		case 1:
			size2read = int(sbuf[0])
		case 2:
			size2read = int(sbuf[0]) | (int(sbuf[1]) << 8)
		case 3:
			size2read = int(sbuf[0]) | (int(sbuf[1]) << 8) | (int(sbuf[2]) << 16)
		default:
			return fmt.Errorf("Structure type %d has invalid SizeBytes setting (%d)", t[0], dt.SizeBytes)
		}

		err = readAll(c.conn, msg, size2read, c.ReadTimeout)
		if nil != err {
			return err
		}

		// we need to allow types that only can be send by one of the side, but also defined
		if nil != dt.Callback {
			dt.Callback(msg[0:size2read], c)
		}
	}
}

// WriteMsg creates header and then writes msg buffer via TCP connection
func (c *Connection) WriteMsg(msgType uint8, msg []byte) error {

	dt, ok := c.Types[msgType]
	if !ok {
		return fmt.Errorf("Unknown msg type %d", msgType)
	}

	header := [5]byte{}
	header[0] = msgType

	size2write := 0
	headerSize := 1

	switch dt.SizeBytes {
	case -1:
		size2write = int(dt.FixedSize)
	case 0:
		size2write = 0
	case 1:
		if len(msg) > 255 {
			return fmt.Errorf("Structure type %d has 1-byte size header but %d bytes givven", msgType, len(msg))
		}
		header[1] = byte(len(msg))
		size2write = len(msg)
		headerSize = 2
	case 2:
		if len(msg) > 255*255 {
			return fmt.Errorf("Structure type %d has 2-byte size header but %d bytes givven", msgType, len(msg))
		}
		header[1] = byte(len(msg) & 0xff)
		header[2] = byte((len(msg) & 0xff00) >> 8)
		size2write = len(msg)
		headerSize = 3

	case 3:
		if len(msg) > 255*255*255 {
			return fmt.Errorf("Structure type %d has 2-byte size header but %d bytes givven", msgType, len(msg))
		}
		header[1] = byte(len(msg) & 0xff)
		header[2] = byte((len(msg) & 0xff00) >> 8)
		header[3] = byte((len(msg) & 0xff0000) >> 16)
		size2write = len(msg)
		headerSize = 4

	default:
		return fmt.Errorf("Structure type %d has invalid SizeBytes setting (%d)", msgType, dt.SizeBytes)
	}

	if size2write > len(msg) {
		return fmt.Errorf("Unable to write %d bytes for structure %d, have only %d in buffer", msgType, size2write, len(msg))
	}

	c.writeMutex.Lock()
	defer c.writeMutex.Unlock()

	err := writeAll(c.conn, header[:], headerSize, c.WriteTimeout)
	if nil != err {
		return err
	}

	err = writeAll(c.conn, msg, size2write, c.WriteTimeout)
	if nil != err {
		return err
	}

	return nil
}

// Remote address retrive
func (c *Connection) Remote() net.Addr {
	if nil == c.conn {
		return nil
	}

	return c.conn.RemoteAddr()
}

func (c *Connection) prepareInitialPacket() initialPacket {
	result := initialPacket{MaxSize: c.MaxSize, Header: [8]byte{'s', 'x', 'c', 'h', 'n', 'g', verHI, verLO}}

	for i := 0; i < 256; i++ {
		result.FixedSize[i] = c.Types[uint8(i)].FixedSize
		result.SizeBytes[i] = c.Types[uint8(i)].SizeBytes
	}

	return result
}

func init() {
	if initialPacketSize != unsafe.Sizeof(initialPacket{}) {
		log.Fatalln("Golang uses different aligment withing structure, please modify sxchnge code!")
	}
}
<commit_msg>Don't need loop for write<commit_after>package sxchange

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"sync"
	"time"
	"unsafe"
)

const (
	verHI = 1
	verLO = 0
)

// CB type for callback function for each type of transfered data
type CB func([]byte, *Connection)

// DataTypeCB describes
type DataTypeCB struct {
	SizeBytes int8  // -1 for fixed size, 0 for no data (ping?), 1 for 0-255 bytes, 2 for 0-65535 bytes...
	FixedSize int32 // if some struct has fixed fize no other header needed
	Callback  CB    // which function to run after data received
}

// Connection informaion both for client or server
type Connection struct {
	conn         *net.TCPConn         // real connection
	writeMutex   sync.Mutex           // we need to lock on write operations
	Types        map[uint8]DataTypeCB // map of type->params&callback
	KeepAlive    time.Duration        // tcp keep-alive
	ReadTimeout  time.Duration        // we need to receive new message at least once per this duration
	WriteTimeout time.Duration        // maximal duration for every write operation
	MaxSize      uint32               // maximum number of bytes for one record/message
}

const (
	initialPacketSize = 1292 // we need to compare it manualy to be sure that compiler doesn't change aligment!
)

type initialPacket struct {
	Header    [8]uint8 // sxchngXY where X.Y is version of protocol
	MaxSize   uint32
	SizeBytes [256]int8
	FixedSize [256]int32
}

func readAll(conn *net.TCPConn, buf []byte, size int, timeout time.Duration) error {

	totalDeadline := time.Now().Add(timeout)
	conn.SetReadDeadline(totalDeadline)
	i, err := io.ReadFull(conn, buf[:size])
	if nil != err {
		return err
	}
	if i != size {
		return errors.New("Partial read")
	}

	return nil
}

func writeAll(conn *net.TCPConn, buf []byte, size int, timeout time.Duration) error {
	totalDeadline := time.Now().Add(timeout)
	conn.SetWriteDeadline(totalDeadline)
	i, err := conn.Write(buf[:size])

	if nil != err {
		return err
	}
	if i != size {
		return errors.New("Partial write")
	}

	return nil
}

func (c *Connection) initConnection() error {
	if c.KeepAlive != 0 {
		c.conn.SetKeepAlivePeriod(c.KeepAlive)
		c.conn.SetKeepAlive(true)
	}

	// initial packets exchange
	initial := c.prepareInitialPacket()
	initialBuf := (*[initialPacketSize]byte)(unsafe.Pointer(&initial))[:]
	received := initialPacket{}
	receivedBuf := (*[initialPacketSize]byte)(unsafe.Pointer(&received))[:]

	// send out version
	err := writeAll(c.conn, initialBuf, initialPacketSize, c.WriteTimeout)
	if nil != err {
		return err
	}

	err = readAll(c.conn, receivedBuf, initialPacketSize, c.ReadTimeout)
	if nil != err {
		return err
	}

	if received.Header != initial.Header {
		return errors.New("HS error - different protocol version")
	}

	if received.MaxSize != initial.MaxSize {
		return errors.New("HS error - different message maximal size")
	}

	if (received.FixedSize != initial.FixedSize) || (received.MaxSize != received.MaxSize) {
		return errors.New("HS error - different set of datatypes")
	}

	return nil
}

// internal run function
func (c *Connection) run() error {

	defer c.conn.Close()

	var (
		sbuf [4]byte
		t    [1]byte
		msg  = make([]byte, 0, c.MaxSize)
	)

	for {
		c.conn.SetReadDeadline(time.Now().Add(c.ReadTimeout))
		i, err := c.conn.Read(t[:])
		if nil != err {
			return err
		}
		if 0 == i { // TODO: check if this is an error?..
			continue
		}

		dt, ok := c.Types[t[0]]

		if !ok {
			return fmt.Errorf("Structure type %d is not defined", t[0])
		}

		var size2read int
		if dt.SizeBytes > 0 {
			err = readAll(c.conn, sbuf[0:dt.SizeBytes], int(dt.SizeBytes), c.ReadTimeout)
			if nil != err {
				return err
			}
		}

		switch dt.SizeBytes {
		case -1:
			size2read = int(dt.FixedSize)
		case 0:
			size2read = 0
		case 1:
			size2read = int(sbuf[0])
		case 2:
			size2read = int(sbuf[0]) | (int(sbuf[1]) << 8)
		case 3:
			size2read = int(sbuf[0]) | (int(sbuf[1]) << 8) | (int(sbuf[2]) << 16)
		default:
			return fmt.Errorf("Structure type %d has invalid SizeBytes setting (%d)", t[0], dt.SizeBytes)
		}

		err = readAll(c.conn, msg, size2read, c.ReadTimeout)
		if nil != err {
			return err
		}

		// we need to allow types that only can be send by one of the side, but also defined
		if nil != dt.Callback {
			dt.Callback(msg[0:size2read], c)
		}
	}
}

// WriteMsg creates header and then writes msg buffer via TCP connection
func (c *Connection) WriteMsg(msgType uint8, msg []byte) error {

	dt, ok := c.Types[msgType]
	if !ok {
		return fmt.Errorf("Unknown msg type %d", msgType)
	}

	header := [5]byte{}
	header[0] = msgType

	size2write := 0
	headerSize := 1

	switch dt.SizeBytes {
	case -1:
		size2write = int(dt.FixedSize)
	case 0:
		size2write = 0
	case 1:
		if len(msg) > 255 {
			return fmt.Errorf("Structure type %d has 1-byte size header but %d bytes givven", msgType, len(msg))
		}
		header[1] = byte(len(msg))
		size2write = len(msg)
		headerSize = 2
	case 2:
		if len(msg) > 255*255 {
			return fmt.Errorf("Structure type %d has 2-byte size header but %d bytes givven", msgType, len(msg))
		}
		header[1] = byte(len(msg) & 0xff)
		header[2] = byte((len(msg) & 0xff00) >> 8)
		size2write = len(msg)
		headerSize = 3

	case 3:
		if len(msg) > 255*255*255 {
			return fmt.Errorf("Structure type %d has 2-byte size header but %d bytes givven", msgType, len(msg))
		}
		header[1] = byte(len(msg) & 0xff)
		header[2] = byte((len(msg) & 0xff00) >> 8)
		header[3] = byte((len(msg) & 0xff0000) >> 16)
		size2write = len(msg)
		headerSize = 4

	default:
		return fmt.Errorf("Structure type %d has invalid SizeBytes setting (%d)", msgType, dt.SizeBytes)
	}

	if size2write > len(msg) {
		return fmt.Errorf("Unable to write %d bytes for structure %d, have only %d in buffer", msgType, size2write, len(msg))
	}

	c.writeMutex.Lock()
	defer c.writeMutex.Unlock()

	err := writeAll(c.conn, header[:], headerSize, c.WriteTimeout)
	if nil != err {
		return err
	}

	err = writeAll(c.conn, msg, size2write, c.WriteTimeout)
	if nil != err {
		return err
	}

	return nil
}

// Remote address retrive
func (c *Connection) Remote() net.Addr {
	if nil == c.conn {
		return nil
	}

	return c.conn.RemoteAddr()
}

func (c *Connection) prepareInitialPacket() initialPacket {
	result := initialPacket{MaxSize: c.MaxSize, Header: [8]byte{'s', 'x', 'c', 'h', 'n', 'g', verHI, verLO}}

	for i := 0; i < 256; i++ {
		result.FixedSize[i] = c.Types[uint8(i)].FixedSize
		result.SizeBytes[i] = c.Types[uint8(i)].SizeBytes
	}

	return result
}

func init() {
	if initialPacketSize != unsafe.Sizeof(initialPacket{}) {
		log.Fatalln("Golang uses different aligment withing structure, please modify sxchnge code!")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"unicode"
	"unicode/utf8"
)

type GhScanner struct {
	*bufio.Scanner
	varKinds  map[string]string
	tvarKinds map[string]string
	lastEntry *Entry
}

type sexp struct {
	Leaf string
	Kids []*sexp
	mama *sexp
}

// TODO: not sure whether this is necessary??
func (s *sexp) destroy() {
	s.mama = nil
	for _, k := range s.Kids {
		k.destroy()
	}
}

func (s *sexp) toString() string {
	if len(s.Leaf) > 0 {
		return s.Leaf
	}
	out := "("
	first := true
	for _, k := range s.Kids {
		if !first {
			out += ","
		}
		first = false
		out += k.toString()
	}
	out += ")"
	return out
}

func indexOf(list []string, t string) (i int, l []string) {
	l = list
	var k string
	for i, k = range l {
		if k == t {
			return
		}
	}
	return len(l), append(l, t)
}

// returns a JSON-like string wrapping
func bracketize(ss []string) string {
	out := "["
	first := true
	for _, s := range ss {
		if !first {
			out += ","
		}
		out += s
		first = false
	}
	out += "]"
	return out
}
func mapify(ss []*sexp, f func(s *sexp) string) []string {
	out := make([]string, len(ss))
	for i, k := range ss {
		out[i] = f(k)
	}
	return out
}

// Turns a sexp into a string.  Assumes all leafs are vars.
// Augments the fields of this.lastEntry as we go.
func (this *GhScanner) stringify(s *sexp) string {
	if len(s.Leaf) > 0 {
		var kind, pfix string
		var ok bool
		if kind, ok = this.tvarKinds[s.Leaf]; ok {
			pfix = "T"
		} else if kind, ok = this.varKinds[s.Leaf]; ok {
			pfix = "V"
		} else {
			panic(errors.New(fmt.Sprintf("Unknown var %s", s.Leaf)))
		}
		oldLen := len(this.lastEntry.Fact.Meat.Kinds)
		var kindI int
		kindI, this.lastEntry.Fact.Meat.Kinds =
			indexOf(this.lastEntry.Fact.Meat.Kinds, kind)
		if kindI >= oldLen {
			// new kind; augment V and T
			this.lastEntry.Fact.Skin.T = append(this.lastEntry.Fact.Skin.T,
				make([]string, 0))
			this.lastEntry.Fact.Skin.V = append(this.lastEntry.Fact.Skin.V,
				make([]string, 0))
		}
		var varList [][]string
		if pfix == "T" {
			varList = this.lastEntry.Fact.Skin.T
		} else {
			varList = this.lastEntry.Fact.Skin.V
		}
		var varI int
		varI, varList[kindI] = indexOf(varList[kindI], s.Leaf)
		return fmt.Sprintf("%s%d.%d", pfix, kindI, varI)
	} else {
		return bracketize(mapify(s.Kids, this.stringify))
	}
}

// Turns a sexp into a string.  Assumes all leafs are vars, except the first,
// which is a term.  Augments the fields of this.lastEntry as we go.
func (this *GhScanner) stringifyTerm(s *sexp) string {
	if len(s.Leaf) > 0 {
		return this.stringify(s)
	} else {
		if len(s.Kids) < 1 {
			msg := ""
			for s.mama != nil {
				msg += s.toString() + "\n"
				s = s.mama
			}
			panic("Empty isTerm sexp! " + msg)
		}
		var termI int
		termI, this.lastEntry.Fact.Meat.Terms =
			indexOf(this.lastEntry.Fact.Meat.Terms, s.Kids[0].Leaf)
		ss := make([]string, 1)
		ss[0] = fmt.Sprintf("%d", termI)
		kids := mapify(s.Kids[1:], this.stringifyTerm)
		return bracketize(append(ss, kids...))
	}
}

func (this *GhScanner) ghSplit(data []byte, atEOF bool) (
	advance int, token []byte, err error) {
	i := 0
	var r rune
	var n int
	var EOF, UTF8 interface{}
	var eofError error
	eatUntil := func(f func() bool) {
		for {
			if i >= len(data) {
				advance = 0
				if atEOF {
					err = eofError
				}
				panic(&EOF)
			}
			r, n = utf8.DecodeRune(data[i:])
			if r == utf8.RuneError {
				err = errors.New("Bad UTF8 encoding!")
				panic(&UTF8)
			}
			i += n
			if f() {
				return
			}
		}
	}
	defer func() {
		r := recover()
		if r == &EOF {
		} else if r == &UTF8 {
		} else if r != nil {
			panic(r)
		}
	}()

	eatUntil(func() bool { return !unicode.IsSpace(r) })
	for r == '#' {
		eatUntil(func() bool { return r == '\n' })
		eatUntil(func() bool { return !unicode.IsSpace(r) })
	}
	eofError = errors.New("Unexpected EOF") // eof not allowed until end sexp
	cmdStart := i - n
	eatUntil(func() bool { return unicode.IsSpace(r) })
	cmd := string(data[cmdStart : i-n])
	eatUntil(func() bool { return !unicode.IsSpace(r) })
	i -= n
	var s *sexp
	tokStart := i
	// eat sexp  TODO: needs testing
	eatUntil(func() bool {
		isSpace := unicode.IsSpace(r)
		if r == ')' || isSpace {
			if i-n > tokStart {
				tok := new(sexp)
				tok.Leaf = string(data[tokStart : i-n])
				tok.mama = s
				s.Kids = append(s.Kids, tok)
			}
			tokStart = i
		}

		if r == '(' {
			kid := new(sexp)
			kid.mama = s
			if s != nil {
				s.Kids = append(s.Kids, kid)
			}
			s = kid
			tokStart = i
		} else if r == ')' {
			if s.mama == nil {
				return true
			}
			s = s.mama
		}
		return false
	})

	if cmd == "stmt" {
		// Emit the token as flat text. Access to the parsed fact is through
		// Entry()
		token = data[cmdStart:i]
		if len(s.Kids) != 4 {
			err = errors.New(fmt.Sprintf("Bad stmt command: %s\n",
				data[cmdStart:i]))
			panic(err)
		}
		label, dvs, hyps, conc := s.Kids[0], s.Kids[1], s.Kids[2], s.Kids[3]
		this.lastEntry = new(Entry)
		this.lastEntry.Fact.Skin.Name = label.Leaf

		key := bracketize([]string{
			bracketize([]string{
				this.stringifyTerm(conc),
				bracketize(mapify(hyps.Kids, this.stringifyTerm)),
				bracketize(mapify(dvs.Kids, this.stringify)),
			}),
			bracketize([]string{
				bracketize(this.lastEntry.Fact.Meat.Terms),
				bracketize(this.lastEntry.Fact.Meat.Kinds),
			}),
		})
		this.lastEntry.Key = key
	} else if cmd == "tvar" || cmd == "var" {
		kind := s.Kids[0].Leaf
		for _, vars := range s.Kids[1:] {
			varName := vars.Leaf
			if cmd == "tvar" {
				this.tvarKinds[varName] = kind
			} else {
				this.varKinds[varName] = kind
			}
		}
		token = make([]byte, 0)
	} else {
		// other commands (kind, term) we skip.
		token = make([]byte, 0)
	}
	advance = i
	s.destroy()
	return
}

// NB: ghSplit emits empty tokens on ignored text. This wrapper
// eliminates those.
func (this *GhScanner) splitWrap(data []byte, atEOF bool) (
	advance int, token []byte, err error) {
	var a int
	for len(token) == 0 && err == nil {
		a, token, err = this.ghSplit(data[advance:], atEOF)
		if a == 0 {
			return 0, nil, err
		}
		advance += a
	}
	return
}

func (this *GhScanner) Entry() *Entry {
	return this.lastEntry
}

func NewScanner(r io.Reader) *GhScanner {
	scanner := GhScanner{
		bufio.NewScanner(r),
		make(map[string]string),
		make(map[string]string),
		nil,
	}
	scanner.Split(scanner.splitWrap)
	return &scanner
}
<commit_msg>use case statment<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"unicode"
	"unicode/utf8"
)

type GhScanner struct {
	*bufio.Scanner
	varKinds  map[string]string
	tvarKinds map[string]string
	lastEntry *Entry
}

type sexp struct {
	Leaf string
	Kids []*sexp
	mama *sexp
}

// TODO: not sure whether this is necessary??
func (s *sexp) destroy() {
	s.mama = nil
	for _, k := range s.Kids {
		k.destroy()
	}
}

func (s *sexp) toString() string {
	if len(s.Leaf) > 0 {
		return s.Leaf
	}
	out := "("
	first := true
	for _, k := range s.Kids {
		if !first {
			out += ","
		}
		first = false
		out += k.toString()
	}
	out += ")"
	return out
}

func indexOf(list []string, t string) (i int, l []string) {
	l = list
	var k string
	for i, k = range l {
		if k == t {
			return
		}
	}
	return len(l), append(l, t)
}

// returns a JSON-like string wrapping
func bracketize(ss []string) string {
	out := "["
	first := true
	for _, s := range ss {
		if !first {
			out += ","
		}
		out += s
		first = false
	}
	out += "]"
	return out
}
func mapify(ss []*sexp, f func(s *sexp) string) []string {
	out := make([]string, len(ss))
	for i, k := range ss {
		out[i] = f(k)
	}
	return out
}

// Turns a sexp into a string.  Assumes all leafs are vars.
// Augments the fields of this.lastEntry as we go.
func (this *GhScanner) stringify(s *sexp) string {
	if len(s.Leaf) > 0 {
		var kind, pfix string
		var ok bool
		if kind, ok = this.tvarKinds[s.Leaf]; ok {
			pfix = "T"
		} else if kind, ok = this.varKinds[s.Leaf]; ok {
			pfix = "V"
		} else {
			panic(errors.New(fmt.Sprintf("Unknown var %s", s.Leaf)))
		}
		oldLen := len(this.lastEntry.Fact.Meat.Kinds)
		var kindI int
		kindI, this.lastEntry.Fact.Meat.Kinds =
			indexOf(this.lastEntry.Fact.Meat.Kinds, kind)
		if kindI >= oldLen {
			// new kind; augment V and T
			this.lastEntry.Fact.Skin.T = append(this.lastEntry.Fact.Skin.T,
				make([]string, 0))
			this.lastEntry.Fact.Skin.V = append(this.lastEntry.Fact.Skin.V,
				make([]string, 0))
		}
		var varList [][]string
		if pfix == "T" {
			varList = this.lastEntry.Fact.Skin.T
		} else {
			varList = this.lastEntry.Fact.Skin.V
		}
		var varI int
		varI, varList[kindI] = indexOf(varList[kindI], s.Leaf)
		return fmt.Sprintf("%s%d.%d", pfix, kindI, varI)
	} else {
		return bracketize(mapify(s.Kids, this.stringify))
	}
}

// Turns a sexp into a string.  Assumes all leafs are vars, except the first,
// which is a term.  Augments the fields of this.lastEntry as we go.
func (this *GhScanner) stringifyTerm(s *sexp) string {
	if len(s.Leaf) > 0 {
		return this.stringify(s)
	} else {
		if len(s.Kids) < 1 {
			msg := ""
			for s.mama != nil {
				msg += s.toString() + "\n"
				s = s.mama
			}
			panic("Empty isTerm sexp! " + msg)
		}
		var termI int
		termI, this.lastEntry.Fact.Meat.Terms =
			indexOf(this.lastEntry.Fact.Meat.Terms, s.Kids[0].Leaf)
		ss := make([]string, 1)
		ss[0] = fmt.Sprintf("%d", termI)
		kids := mapify(s.Kids[1:], this.stringifyTerm)
		return bracketize(append(ss, kids...))
	}
}

func (this *GhScanner) ghSplit(data []byte, atEOF bool) (
	advance int, token []byte, err error) {
	i := 0
	var r rune
	var n int
	var EOF, UTF8 interface{}
	var eofError error
	eatUntil := func(f func() bool) {
		for {
			if i >= len(data) {
				advance = 0
				if atEOF {
					err = eofError
				}
				panic(&EOF)
			}
			r, n = utf8.DecodeRune(data[i:])
			if r == utf8.RuneError {
				err = errors.New("Bad UTF8 encoding!")
				panic(&UTF8)
			}
			i += n
			if f() {
				return
			}
		}
	}
	defer func() {
		r := recover()
		if r == &EOF {
		} else if r == &UTF8 {
		} else if r != nil {
			panic(r)
		}
	}()

	eatUntil(func() bool { return !unicode.IsSpace(r) })
	for r == '#' {
		eatUntil(func() bool { return r == '\n' })
		eatUntil(func() bool { return !unicode.IsSpace(r) })
	}
	eofError = errors.New("Unexpected EOF") // eof not allowed until end sexp
	cmdStart := i - n
	eatUntil(func() bool { return unicode.IsSpace(r) })
	cmd := string(data[cmdStart : i-n])
	eatUntil(func() bool { return !unicode.IsSpace(r) })
	i -= n
	var s *sexp
	tokStart := i
	// eat sexp  TODO: needs testing
	eatUntil(func() bool {
		isSpace := unicode.IsSpace(r)
		if r == ')' || isSpace {
			if i-n > tokStart {
				tok := new(sexp)
				tok.Leaf = string(data[tokStart : i-n])
				tok.mama = s
				s.Kids = append(s.Kids, tok)
			}
			tokStart = i
		}

		if r == '(' {
			kid := new(sexp)
			kid.mama = s
			if s != nil {
				s.Kids = append(s.Kids, kid)
			}
			s = kid
			tokStart = i
		} else if r == ')' {
			if s.mama == nil {
				return true
			}
			s = s.mama
		}
		return false
	})

	switch cmd {
	case "stmt":
		// Emit the token as flat text. Access to the parsed fact is through
		// Entry()
		token = data[cmdStart:i]
		if len(s.Kids) != 4 {
			err = errors.New(fmt.Sprintf("Bad stmt command: %s\n",
				data[cmdStart:i]))
			panic(err)
		}
		label, dvs, hyps, conc := s.Kids[0], s.Kids[1], s.Kids[2], s.Kids[3]
		this.lastEntry = new(Entry)
		this.lastEntry.Fact.Skin.Name = label.Leaf

		key := bracketize([]string{
			bracketize([]string{
				this.stringifyTerm(conc),
				bracketize(mapify(hyps.Kids, this.stringifyTerm)),
				bracketize(mapify(dvs.Kids, this.stringify)),
			}),
			bracketize([]string{
				bracketize(this.lastEntry.Fact.Meat.Terms),
				bracketize(this.lastEntry.Fact.Meat.Kinds),
			}),
		})
		this.lastEntry.Key = key
	case "tvar", "var":
		kind := s.Kids[0].Leaf
		for _, vars := range s.Kids[1:] {
			varName := vars.Leaf
			if cmd == "tvar" {
				this.tvarKinds[varName] = kind
			} else {
				this.varKinds[varName] = kind
			}
		}
		token = make([]byte, 0)
	case "param":
		//PICKUP : how to handle?
	default:
		// other commands (kind, term) we skip.
		token = make([]byte, 0)
	}
	advance = i
	s.destroy()
	return
}

// NB: ghSplit emits empty tokens on ignored text. This wrapper
// eliminates those.
func (this *GhScanner) splitWrap(data []byte, atEOF bool) (
	advance int, token []byte, err error) {
	var a int
	for len(token) == 0 && err == nil {
		a, token, err = this.ghSplit(data[advance:], atEOF)
		if a == 0 {
			return 0, nil, err
		}
		advance += a
	}
	return
}

func (this *GhScanner) Entry() *Entry {
	return this.lastEntry
}

func NewScanner(r io.Reader) *GhScanner {
	scanner := GhScanner{
		bufio.NewScanner(r),
		make(map[string]string),
		make(map[string]string),
		nil,
	}
	scanner.Split(scanner.splitWrap)
	return &scanner
}
<|endoftext|>


<commit_before>package gocaptcha

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"text/template"
)

var captchaHtml *template.Template

func init() {
	var err error
	captchaHtml, err = template.New("CaptchaHtml").Parse(`

	<textarea name="recaptcha_challenge_field" rows="3" cols="40"></textarea>
	<input type="hidden" name="recaptcha_response_field" value="manual_challenge">

`)
	if err != nil {
		fmt.Printf("Error parsing CaptchaHtml template.")
		os.Exit(-1)
	}
}

// A GoCaptcha object identifies a single reCAPTCHA session (for one client).
// It is possible to store this object on a session activity and re-use it lateron for verification.
// This object keeps track of faulty verifications and makes sure any newly generated html contains an error message for the end-user, as provided by reCAPTCHA.
// Once a reCAPTCHA response was successfully verified this object should be discarded.
type GoCaptcha struct {
	publickey     string
	privatekey    string
	lastErrorCode string
	lastResult    bool
}

// NewGoCaptha creates a new GoCaptcha object.
// Privatekey is the api key to be used with reCAPTCHA.
func NewGoCaptcha(publickey string, privatekey string) *GoCaptcha {
	gc := &GoCaptcha{
		publickey:  publickey,
		privatekey: privatekey,
	}
	return gc
}

// Generate the reCAPTCHA html for this session and write it to the given io.Writer.
func (gc *GoCaptcha) WriteHtml(w io.Writer) error {
	err := captchaHtml.Execute(w, struct {
		PublicKey string
		ErrorCode string
	}{gc.publickey, gc.lastErrorCode})
	return err
}

// Generate the reCAPTCHA html for this session and return it as string.
// If error is not nil then something went wrong and string is empty.
func (gc *GoCaptcha) HtmlString() (string, error) {
	buf := new(bytes.Buffer)
	err := gc.WriteHtml(buf)
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}

// Generate the reCAPTCHA html for this session and return it as byteslice.
// If error is not nil then something went wrong and the byteslice is empty.
func (gc *GoCaptcha) HtmlByteSlice() ([]byte, error) {
	buf := new(bytes.Buffer)
	err := gc.WriteHtml(buf)
	if err != nil {
		return []byte{}, err
	}
	return buf.Bytes(), nil
}

// Verify calls the reCAPTCHA API to verify if the given response by end-user is correct.
// Any returned error indicates a unsuccessfull api call. It does not indicate that the reCAPTCHA response by the end-user was faulty.
// Any returned error value is not to be shown to the end-user.
// When the error is nil, then response=false indicates that the reCAPTCHA response by the end-user was faulty.
// End-user will be notified of a faulty reCAPTCHA response when re-using this GoCaptcha object to generate html code again.
//
// Expected parameters:
// challenge string, form value as sent by the http request. (Set by the reCAPTCHA in the end-users browser.)
// response string, form value as sent by the http request. (The answer given by the end-user.)
// remoteaddr string, The ip address (i.e. "127.0.0.1") or http.Request.RemoteAddr (i.e. "127.0.0.1:45435") from the client's endpoint.
// Warning: at this moment only IPv4 remoteaddr is supported. Major bug!!!
func (gc *GoCaptcha) Verify(challenge string, response string, remoteaddr string) (bool, error) {
	if gc.lastResult {
		return false, errors.New("This GoCaptcha session has already been successfully verified. Please create a new GoCaptcha session.")
	}

	//++ WARNING: This will NOT work with IPv6. Major bug!
	//++ TODO(GeertJohan): pickup the ip properly.
	remoteip := strings.SplitN(remoteaddr, ":", 2)[0]

	apiRequestValues := url.Values{}
	apiRequestValues.Set("privatekey", gc.privatekey)
	apiRequestValues.Set("remoteip", remoteip)
	apiRequestValues.Set("challenge", challenge)
	apiRequestValues.Set("response", response)
	apiResponse, err := http.PostForm("https://www.google.com/recaptcha/api/verify", apiRequestValues)
	if err != nil {
		panic(err)
	}
	defer apiResponse.Body.Close()
	reader := bufio.NewReader(apiResponse.Body)

	// read first line
	line1, err := reader.ReadString('\n')
	if err != nil {
		return false, errors.New("Received unexpected result value from reCAPTCHA API.")
	}
	switch line1 {
	case "true\n":
		gc.lastResult = true
		gc.lastErrorCode = ""
		return true, nil

	case "false\n":
		// read second line
		line2, err := reader.ReadString('\n')
		if err != nil && err != io.EOF {
			return false, errors.New("Received unexpected result value from reCAPTCHA API.")
		}
		gc.lastErrorCode = line2

	default:
		return false, errors.New("Received unexpected result value from reCAPTCHA API.")
	}

	return false, nil
}
<commit_msg>Removed panic call. Simply returning error now.<commit_after>package gocaptcha

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"text/template"
)

var captchaHtml *template.Template

func init() {
	var err error
	captchaHtml, err = template.New("CaptchaHtml").Parse(`

	<textarea name="recaptcha_challenge_field" rows="3" cols="40"></textarea>
	<input type="hidden" name="recaptcha_response_field" value="manual_challenge">

`)
	if err != nil {
		fmt.Printf("Error parsing CaptchaHtml template.")
		os.Exit(-1)
	}
}

// A GoCaptcha object identifies a single reCAPTCHA session (for one client).
// It is possible to store this object on a session activity and re-use it lateron for verification.
// This object keeps track of faulty verifications and makes sure any newly generated html contains an error message for the end-user, as provided by reCAPTCHA.
// Once a reCAPTCHA response was successfully verified this object should be discarded.
type GoCaptcha struct {
	publickey     string
	privatekey    string
	lastErrorCode string
	lastResult    bool
}

// NewGoCaptha creates a new GoCaptcha object.
// Privatekey is the api key to be used with reCAPTCHA.
func NewGoCaptcha(publickey string, privatekey string) *GoCaptcha {
	gc := &GoCaptcha{
		publickey:  publickey,
		privatekey: privatekey,
	}
	return gc
}

// Generate the reCAPTCHA html for this session and write it to the given io.Writer.
func (gc *GoCaptcha) WriteHtml(w io.Writer) error {
	err := captchaHtml.Execute(w, struct {
		PublicKey string
		ErrorCode string
	}{gc.publickey, gc.lastErrorCode})
	return err
}

// Generate the reCAPTCHA html for this session and return it as string.
// If error is not nil then something went wrong and string is empty.
func (gc *GoCaptcha) HtmlString() (string, error) {
	buf := new(bytes.Buffer)
	err := gc.WriteHtml(buf)
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}

// Generate the reCAPTCHA html for this session and return it as byteslice.
// If error is not nil then something went wrong and the byteslice is empty.
func (gc *GoCaptcha) HtmlByteSlice() ([]byte, error) {
	buf := new(bytes.Buffer)
	err := gc.WriteHtml(buf)
	if err != nil {
		return []byte{}, err
	}
	return buf.Bytes(), nil
}

// Verify calls the reCAPTCHA API to verify if the given response by end-user is correct.
// Any returned error indicates a unsuccessfull api call. It does not indicate that the reCAPTCHA response by the end-user was faulty.
// Any returned error value is not to be shown to the end-user.
// When the error is nil, then response=false indicates that the reCAPTCHA response by the end-user was faulty.
// End-user will be notified of a faulty reCAPTCHA response when re-using this GoCaptcha object to generate html code again.
//
// Expected parameters:
// challenge string, form value as sent by the http request. (Set by the reCAPTCHA in the end-users browser.)
// response string, form value as sent by the http request. (The answer given by the end-user.)
// remoteaddr string, The ip address (i.e. "127.0.0.1") or http.Request.RemoteAddr (i.e. "127.0.0.1:45435") from the client's endpoint.
// Warning: at this moment only IPv4 remoteaddr is supported. Major bug!!!
func (gc *GoCaptcha) Verify(challenge string, response string, remoteaddr string) (bool, error) {
	if gc.lastResult {
		return false, errors.New("This GoCaptcha session has already been successfully verified. Please create a new GoCaptcha session.")
	}

	//++ WARNING: This will NOT work with IPv6. Major bug!
	//++ TODO(GeertJohan): pickup the ip properly.
	remoteip := strings.SplitN(remoteaddr, ":", 2)[0]

	apiRequestValues := url.Values{}
	apiRequestValues.Set("privatekey", gc.privatekey)
	apiRequestValues.Set("remoteip", remoteip)
	apiRequestValues.Set("challenge", challenge)
	apiRequestValues.Set("response", response)
	apiResponse, err := http.PostForm("https://www.google.com/recaptcha/api/verify", apiRequestValues)
	if err != nil {
		return false, err
	}
	defer apiResponse.Body.Close()
	reader := bufio.NewReader(apiResponse.Body)

	// read first line
	line1, err := reader.ReadString('\n')
	if err != nil {
		return false, errors.New("Received unexpected result value from reCAPTCHA API.")
	}
	switch line1 {
	case "true\n":
		gc.lastResult = true
		gc.lastErrorCode = ""
		return true, nil

	case "false\n":
		// read second line
		line2, err := reader.ReadString('\n')
		if err != nil && err != io.EOF {
			return false, errors.New("Received unexpected result value from reCAPTCHA API.")
		}
		gc.lastErrorCode = line2

	default:
		return false, errors.New("Received unexpected result value from reCAPTCHA API.")
	}

	return false, nil
}
<|endoftext|>


<commit_before>// An EventEmitter package for go 1.2 +
// Author: Mohammed Al Ashaal
// License: MIT License
// Version: v1.0.0
package Emitter

// Emitter - our listeners container
type Emitter struct {
	listeners	map[interface{}][]Listener
}

// Listener - our callback container and whether it will run once or not
type Listener struct {
	callback	func(...interface{})
	once		bool
}

// Construct() - create a new instance of Emitter
func Construct() *Emitter {
	emitter := new(Emitter)
	emitter.listeners = make(map[interface{}][]Listener)
	return emitter
}

// Destruct() - free memory from an emitter instance
func(self *Emitter) Destruct() {
	self = nil
}

// AddListener() - register a new listener on the specified event
func(self *Emitter) AddListener(event string, callback func(...interface{})) *Emitter {
	return self.On(event, callback)
}

// On() - register a new listener on the specified event
func(self *Emitter) On(event string, callback func(...interface{})) *Emitter {
	if _, ok := self.listeners[event]; ! ok {
		self.listeners[event] = []Listener{}
	}
	self.listeners[event] = append(self.listeners[event], Listener{callback, false})
	self.EmitSync("newListener", []interface{}{event, callback})
	return self
}

// Once() - register a new one-time listener on the specified event
func(self *Emitter) Once(event string, callback func(...interface{})) *Emitter {
	if _, ok := self.listeners[event]; ! ok {
		self.listeners[event] = []Listener{}
	}
	self.listeners[event] = append(self.listeners[event], Listener{callback, true})
	self.EmitSync("newListener", []interface{}{event, callback})
	return self
}

// RemoveListeners() - remove the specified callback from the specified events' listeners
func(self *Emitter) RemoveListener(event string, callback func(...interface{})) *Emitter {
	if _, ok := self.listeners[event]; ! ok {
		return self
	}
	for k,v := range(self.listeners[event]) {
		if &v.callback == &callback {
			self.listeners[event] = append(self.listeners[event][:k], self.listeners[event][k + 1:]...)
			self.EmitSync("removeListener", []interface{}{event, callback})
			return self
		}
	}
	return self
}

// RemoveAllListeners() - remove all listeners from (all/event)
func(self *Emitter) RemoveAllListeners(event interface{}) *Emitter {
	if event == nil {
		self.listeners = make(map[interface{}][]Listener)
		return self
	}
	if _, ok := self.listeners[event]; ! ok {
		return self
	}
	delete(self.listeners, event)
	return self
}

// Listeners() - return an array with the registered listeners in the specified event
func(self *Emitter) Listeners(event string) []Listener {
	if _, ok := self.listeners[event]; ! ok {
		return nil
	}
	return self.listeners[event]
}

// ListenersCount() - return the count of listeners in the speicifed event
func(self *Emitter) ListenersCount(event string) int {
	return len(self.Listeners(event))
}

// EmitSync() - run all listeners of the specified event in synchronous mode
func(self *Emitter) EmitSync(event string, args ...interface{}) *Emitter {
	for _,v := range(self.Listeners(event)) {
		if v.once {
			self.RemoveListener(event, v.callback)
		}
		v.callback(args...)
	}
	return self
}

// EmitAsync() - run all listeners of the specified event in asynchronous mode using goroutines
func(self *Emitter) EmitAsync(event string, args []interface{}) *Emitter {
	for _,v := range(self.Listeners(event)) {
		if v.once {
			self.RemoveListener(event, v.callback)
		}
		go v.callback(args...)
	}
	return self
}
<commit_msg>feat (*mutex) make it concurrent  safe<commit_after>// An EventEmitter package for go 1.2 +
// Author: Mohammed Al Ashaal
// License: MIT License
// Version: v1.0.0
package Emitter

import "sync"

// Emitter - our listeners container
type Emitter struct {
	listeners map[interface{}][]Listener
	mutex     *sync.Mutex
}

// Listener - our callback container and whether it will run once or not
type Listener struct {
	callback func(...interface{})
	once     bool
}

// Construct() - create a new instance of Emitter
func Construct() *Emitter {
	return &Emitter{
		make(map[interface{}][]Listener),
		&sync.Mutex{},
	}
}

// Destruct() - free memory from an emitter instance
func (self *Emitter) Destruct() {
	self = nil
}

// AddListener() - register a new listener on the specified event
func (self *Emitter) AddListener(event string, callback func(...interface{})) *Emitter {
	return self.On(event, callback)
}

// On() - register a new listener on the specified event
func (self *Emitter) On(event string, callback func(...interface{})) *Emitter {
	self.mutex.Lock()
	if _, ok := self.listeners[event]; !ok {
		self.listeners[event] = []Listener{}
	}
	self.listeners[event] = append(self.listeners[event], Listener{callback, false})
	self.mutex.Unlock()

	self.EmitSync("newListener", []interface{}{event, callback})
	return self
}

// Once() - register a new one-time listener on the specified event
func (self *Emitter) Once(event string, callback func(...interface{})) *Emitter {
	self.mutex.Lock()
	if _, ok := self.listeners[event]; !ok {
		self.listeners[event] = []Listener{}
	}
	self.listeners[event] = append(self.listeners[event], Listener{callback, true})
	self.mutex.Unlock()

	self.EmitSync("newListener", []interface{}{event, callback})
	return self
}

// RemoveListeners() - remove the specified callback from the specified events' listeners
func (self *Emitter) RemoveListener(event string, callback func(...interface{})) *Emitter {
	if _, ok := self.listeners[event]; !ok {
		return self
	}
	for k, v := range self.listeners[event] {
		if &v.callback == &callback {
			self.listeners[event] = append(self.listeners[event][:k], self.listeners[event][k+1:]...)
			self.EmitSync("removeListener", []interface{}{event, callback})
			return self
		}
	}
	return self
}

// RemoveAllListeners() - remove all listeners from (all/event)
func (self *Emitter) RemoveAllListeners(event interface{}) *Emitter {
	self.mutex.Lock()
	defer self.mutex.Unlock()

	if event == nil {
		self.listeners = make(map[interface{}][]Listener)
		return self
	}
	if _, ok := self.listeners[event]; !ok {
		return self
	}
	delete(self.listeners, event)
	return self
}

// Listeners() - return an array with the registered listeners in the specified event
func (self *Emitter) Listeners(event string) []Listener {
	if _, ok := self.listeners[event]; !ok {
		return nil
	}
	return self.listeners[event]
}

// ListenersCount() - return the count of listeners in the speicifed event
func (self *Emitter) ListenersCount(event string) int {
	return len(self.Listeners(event))
}

// EmitSync() - run all listeners of the specified event in synchronous mode
func (self *Emitter) EmitSync(event string, args ...interface{}) *Emitter {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	for _, v := range self.Listeners(event) {
		if v.once {
			self.RemoveListener(event, v.callback)
		}
		v.callback(args...)
	}

	return self
}

// EmitAsync() - run all listeners of the specified event in asynchronous mode using goroutines
func (self *Emitter) EmitAsync(event string, args []interface{}) *Emitter {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	for _, v := range self.Listeners(event) {
		if v.once {
			self.RemoveListener(event, v.callback)
		}
		go v.callback(args...)
	}
	return self
}
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gnmi_1_8_metadata_annotation_test

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"testing"

	gpb "github.com/openconfig/gnmi/proto/gnmi"
	"google.golang.org/protobuf/types/known/anypb"
	"google.golang.org/protobuf/types/known/timestamppb"

	"github.com/google/go-cmp/cmp"
	"github.com/openconfig/featureprofiles/internal/fptest"
	"github.com/openconfig/ondatra"
	"github.com/openconfig/ygot/util"
	"google.golang.org/protobuf/encoding/prototext"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/testing/protocmp"
)

func TestMain(m *testing.M) {
	fptest.RunTests(m)
}

// Test cases:
//  - Set and get metadata annotation and verify annotation is configured.
//  - Set metadata annotation with different one and verify new annotation is configured.
//
// Test notes:
//   - https://github.com/openconfig/public/blob/master/release/models/extensions/openconfig-metadata.yang
//   - Documentation on the @ path: https://datatracker.ietf.org/doc/html/rfc7952#section-5.2.2
//   - Metadata is not supported by telemgen generated telemetry APIs, so we need to use
//     RawAPIs() to set request and get response data.
//   - Metadata annotation is a proto message with base64 encoded.
//   - Although RFC 7952 allows metadata annotation on any path and leaves,
//     this test only configures it at root per WBB requirement.
//   - Steps to create SetRequest and get response data.
//     1) Marshal proto message and encode it with base64.
//        - Arista does not accept SetRequest paths with "@" in it - b/216674592.
//     2) Create a json format config for metadata annotation.
//     3) Marshal the metadata annotation config.
//     4) Use gnmiClient to send SetRequest with Update operation.
//     5) For GetResponse data, it is a reverse process to SetRequest.
//
//  Sample metadata annotation in json and flat format:
//   1) json format
//        {
//          "@": {
//             "openconfig-metadata:protobuf-metadata": "CNyR0gk="
//          }
//        }
//   2) flat format
//        /@/openconfig-metadata:protobuf-metadata: CNyR0gk=
//

func TestGNMIMetadataAnnotation(t *testing.T) {
	dut := ondatra.DUT(t, "dut")
	cases := []struct {
		desc        string
		protoMsg    proto.Message
		receivedMsg proto.Message
	}{
		{
			// This case is designed to set an empty field.
			desc:        "Set and get metadata annotation and verify annotation",
			protoMsg:    &timestamppb.Timestamp{Seconds: 1643261234},
			receivedMsg: &timestamppb.Timestamp{},
		},
		{
			// This case is designed to reset an existing field configured from the previous test.
			// It should also pass if it is run independently
			desc:        "Set metadata annotation with different value and verify new annotation",
			protoMsg:    &gpb.ModelData{Name: "Test model data", Organization: "Google", Version: "1.0"},
			receivedMsg: &gpb.ModelData{},
		},
	}

	for _, tc := range cases {
		t.Log(tc.desc)
		gnmiClient := dut.RawAPIs().GNMI().Default(t)

		gpbSetRequest, err := buildMetadataAnnotation(tc.protoMsg)
		if err != nil {
			t.Errorf("Cannot build a gNMI SetRequest from proto message: %v", err)
		}

		t.Log("gnmiClient Set metadata annotation")
		if _, err = gnmiClient.Set(context.Background(), gpbSetRequest); err != nil {
			t.Errorf("gnmi.Set unexpected error: %v", err)
		}

		t.Log("gnmiClient Get metadata annotation")
		getResponse, err := gnmiClient.Get(context.Background(), &gpb.GetRequest{
			Path: []*gpb.Path{{
				Elem: []*gpb.PathElem{},
			}},
			Type: gpb.GetRequest_CONFIG,
		})
		if err != nil {
			t.Fatalf("Cannot fetch metadata annotation from the DUT: %v", err)
		}

		receivedProtoMsg := tc.receivedMsg
		if err := extractMetadataAnnotation(getResponse, receivedProtoMsg); err != nil {
			t.Errorf("Extracts metadata protobuf message from getResponse with error: %v", err)
		}
		if diff := cmp.Diff(tc.protoMsg, receivedProtoMsg, protocmp.Transform()); diff != "" {
			t.Errorf("MyFunction(%s) diff (-want +got):\n%s", tc.protoMsg, diff)
		}
	}
}

// buildMetadataAnnotation builds a gNMI SetRequest by encoding a protobuf message as metadata.
func buildMetadataAnnotation(m proto.Message) (*gpb.SetRequest, error) {
	b, err := proto.Marshal(m)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal proto msg - error: %v", err)
	}
	anyMsg := &anypb.Any{TypeUrl: "example.com", Value: b}
	a, err := proto.Marshal(anyMsg)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal proto any msg - error: %v", err)
	}

	b64 := base64.StdEncoding.EncodeToString(a)
	fmt.Printf("Encoded proto msg: %s\n", b64)

	j := map[string]interface{}{
		"@": map[string]interface{}{
			"openconfig-metadata:protobuf-metadata": b64,
		},
	}
	v, err := json.Marshal(j)
	if err != nil {
		return nil, fmt.Errorf("marshal metadata annotation failed with unexpected error: %v", err)
	}

	// TODO: extract the GetRequest construction to a buildGetRequest
	gpbSetRequest := &gpb.SetRequest{
		Update: []*gpb.Update{{
			Path: &gpb.Path{
				Elem: []*gpb.PathElem{},
			},
			Val: &gpb.TypedValue{
				Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: v},
			},
		}},
	}
	return gpbSetRequest, nil
}

// extractMetadataAnnotation extracts the metadata protobuf message from a gNMI GetResponse.
func extractMetadataAnnotation(getResponse *gpb.GetResponse, m proto.Message) error {
	if got := len(getResponse.GetNotification()); got != 1 {
		return fmt.Errorf("number of notifications got %d, want 1", got)
	}

	var encoded string
	n := getResponse.GetNotification()[0]
	u := n.Update[0]
	path, err := util.JoinPaths(n.GetPrefix(), u.GetPath())
	if err != nil {
		return err
	}
	if len(path.GetElem()) > 0 {
		return fmt.Errorf("update path, got: %v, want root", prototext.Format(path))
	}
	var v map[string]interface{}
	if err = json.Unmarshal(u.GetVal().GetJsonIetfVal(), &v); err != nil {
		return fmt.Errorf("cannot unmarshal the json content, err: %v", err)
	}
	metav, ok := v["@"]
	if !ok {
		return fmt.Errorf("did not receive metadata value from root data: %#v", v)
	}
	metaObj, ok := metav.(map[string]interface{})
	if !ok {
		return fmt.Errorf("received metadata object is not a JSON object: %v", metav)
	}
	annotation, ok := metaObj["openconfig-metadata:protobuf-metadata"]
	if !ok {
		return fmt.Errorf("received metadata object does not contain expected annotation: %#v", metav)
	}
	if encoded, ok = annotation.(string); !ok {
		return fmt.Errorf("got %T type for annotation, expected string type", annotation)
	}
	fmt.Println("Got the encoded annotation", encoded)

	decoded, err := base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return fmt.Errorf("cannot decode base64 string, err: %v", err)
	}

	anyMsg := &anypb.Any{}
	if err := proto.Unmarshal(decoded, anyMsg); err != nil {
		return fmt.Errorf("cannot unmarshal received proto any msg, err: %v", err)
	}
	if err := proto.Unmarshal(anyMsg.GetValue(), m); err != nil {
		return fmt.Errorf("cannot unmarshal received proto msg, err: %v", err)
	}
	return nil
}
<commit_msg>Removing reference to test plan (#359)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gnmi_1_8_metadata_annotation_test

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"testing"

	gpb "github.com/openconfig/gnmi/proto/gnmi"
	"google.golang.org/protobuf/types/known/anypb"
	"google.golang.org/protobuf/types/known/timestamppb"

	"github.com/google/go-cmp/cmp"
	"github.com/openconfig/featureprofiles/internal/fptest"
	"github.com/openconfig/ondatra"
	"github.com/openconfig/ygot/util"
	"google.golang.org/protobuf/encoding/prototext"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/testing/protocmp"
)

func TestMain(m *testing.M) {
	fptest.RunTests(m)
}

// Test cases:
//  - Set and get metadata annotation and verify annotation is configured.
//  - Set metadata annotation with different one and verify new annotation is configured.
//
// Test notes:
//   - https://github.com/openconfig/public/blob/master/release/models/extensions/openconfig-metadata.yang
//   - Documentation on the @ path: https://datatracker.ietf.org/doc/html/rfc7952#section-5.2.2
//   - Metadata is not supported by telemgen generated telemetry APIs, so we need to use
//     RawAPIs() to set request and get response data.
//   - Metadata annotation is a proto message with base64 encoded.
//   - Although RFC 7952 allows metadata annotation on any path and leaves,
//     this test only configures it at root.
//   - Steps to create SetRequest and get response data.
//     1) Marshal proto message and encode it with base64.
//        - Arista does not accept SetRequest paths with "@" in it - b/216674592.
//     2) Create a json format config for metadata annotation.
//     3) Marshal the metadata annotation config.
//     4) Use gnmiClient to send SetRequest with Update operation.
//     5) For GetResponse data, it is a reverse process to SetRequest.
//
//  Sample metadata annotation in json and flat format:
//   1) json format
//        {
//          "@": {
//             "openconfig-metadata:protobuf-metadata": "CNyR0gk="
//          }
//        }
//   2) flat format
//        /@/openconfig-metadata:protobuf-metadata: CNyR0gk=
//

func TestGNMIMetadataAnnotation(t *testing.T) {
	dut := ondatra.DUT(t, "dut")
	cases := []struct {
		desc        string
		protoMsg    proto.Message
		receivedMsg proto.Message
	}{
		{
			// This case is designed to set an empty field.
			desc:        "Set and get metadata annotation and verify annotation",
			protoMsg:    &timestamppb.Timestamp{Seconds: 1643261234},
			receivedMsg: &timestamppb.Timestamp{},
		},
		{
			// This case is designed to reset an existing field configured from the previous test.
			// It should also pass if it is run independently
			desc:        "Set metadata annotation with different value and verify new annotation",
			protoMsg:    &gpb.ModelData{Name: "Test model data", Organization: "Google", Version: "1.0"},
			receivedMsg: &gpb.ModelData{},
		},
	}

	for _, tc := range cases {
		t.Log(tc.desc)
		gnmiClient := dut.RawAPIs().GNMI().Default(t)

		gpbSetRequest, err := buildMetadataAnnotation(tc.protoMsg)
		if err != nil {
			t.Errorf("Cannot build a gNMI SetRequest from proto message: %v", err)
		}

		t.Log("gnmiClient Set metadata annotation")
		if _, err = gnmiClient.Set(context.Background(), gpbSetRequest); err != nil {
			t.Errorf("gnmi.Set unexpected error: %v", err)
		}

		t.Log("gnmiClient Get metadata annotation")
		getResponse, err := gnmiClient.Get(context.Background(), &gpb.GetRequest{
			Path: []*gpb.Path{{
				Elem: []*gpb.PathElem{},
			}},
			Type: gpb.GetRequest_CONFIG,
		})
		if err != nil {
			t.Fatalf("Cannot fetch metadata annotation from the DUT: %v", err)
		}

		receivedProtoMsg := tc.receivedMsg
		if err := extractMetadataAnnotation(getResponse, receivedProtoMsg); err != nil {
			t.Errorf("Extracts metadata protobuf message from getResponse with error: %v", err)
		}
		if diff := cmp.Diff(tc.protoMsg, receivedProtoMsg, protocmp.Transform()); diff != "" {
			t.Errorf("MyFunction(%s) diff (-want +got):\n%s", tc.protoMsg, diff)
		}
	}
}

// buildMetadataAnnotation builds a gNMI SetRequest by encoding a protobuf message as metadata.
func buildMetadataAnnotation(m proto.Message) (*gpb.SetRequest, error) {
	b, err := proto.Marshal(m)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal proto msg - error: %v", err)
	}
	anyMsg := &anypb.Any{TypeUrl: "example.com", Value: b}
	a, err := proto.Marshal(anyMsg)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal proto any msg - error: %v", err)
	}

	b64 := base64.StdEncoding.EncodeToString(a)
	fmt.Printf("Encoded proto msg: %s\n", b64)

	j := map[string]interface{}{
		"@": map[string]interface{}{
			"openconfig-metadata:protobuf-metadata": b64,
		},
	}
	v, err := json.Marshal(j)
	if err != nil {
		return nil, fmt.Errorf("marshal metadata annotation failed with unexpected error: %v", err)
	}

	// TODO: extract the GetRequest construction to a buildGetRequest
	gpbSetRequest := &gpb.SetRequest{
		Update: []*gpb.Update{{
			Path: &gpb.Path{
				Elem: []*gpb.PathElem{},
			},
			Val: &gpb.TypedValue{
				Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: v},
			},
		}},
	}
	return gpbSetRequest, nil
}

// extractMetadataAnnotation extracts the metadata protobuf message from a gNMI GetResponse.
func extractMetadataAnnotation(getResponse *gpb.GetResponse, m proto.Message) error {
	if got := len(getResponse.GetNotification()); got != 1 {
		return fmt.Errorf("number of notifications got %d, want 1", got)
	}

	var encoded string
	n := getResponse.GetNotification()[0]
	u := n.Update[0]
	path, err := util.JoinPaths(n.GetPrefix(), u.GetPath())
	if err != nil {
		return err
	}
	if len(path.GetElem()) > 0 {
		return fmt.Errorf("update path, got: %v, want root", prototext.Format(path))
	}
	var v map[string]interface{}
	if err = json.Unmarshal(u.GetVal().GetJsonIetfVal(), &v); err != nil {
		return fmt.Errorf("cannot unmarshal the json content, err: %v", err)
	}
	metav, ok := v["@"]
	if !ok {
		return fmt.Errorf("did not receive metadata value from root data: %#v", v)
	}
	metaObj, ok := metav.(map[string]interface{})
	if !ok {
		return fmt.Errorf("received metadata object is not a JSON object: %v", metav)
	}
	annotation, ok := metaObj["openconfig-metadata:protobuf-metadata"]
	if !ok {
		return fmt.Errorf("received metadata object does not contain expected annotation: %#v", metav)
	}
	if encoded, ok = annotation.(string); !ok {
		return fmt.Errorf("got %T type for annotation, expected string type", annotation)
	}
	fmt.Println("Got the encoded annotation", encoded)

	decoded, err := base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return fmt.Errorf("cannot decode base64 string, err: %v", err)
	}

	anyMsg := &anypb.Any{}
	if err := proto.Unmarshal(decoded, anyMsg); err != nil {
		return fmt.Errorf("cannot unmarshal received proto any msg, err: %v", err)
	}
	if err := proto.Unmarshal(anyMsg.GetValue(), m); err != nil {
		return fmt.Errorf("cannot unmarshal received proto msg, err: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package analytics

import (
	"github.com/lonelycode/tyk-pump/logger"
	"reflect"
	"strconv"
	"strings"
	"time"
)

var log = logger.GetLogger()

// AnalyticsRecord encodes the details of a request
type AnalyticsRecord struct {
	Method        string
	Path          string
	ContentLength int64
	UserAgent     string
	Day           int
	Month         time.Month
	Year          int
	Hour          int
	ResponseCode  int
	APIKey        string
	TimeStamp     time.Time
	APIVersion    string
	APIName       string
	APIID         string
	OrgID         string
	OauthID       string
	RequestTime   int64
	RawRequest    string
	RawResponse   string
	Tags          []string
	ExpireAt      time.Time `bson:"expireAt" json:"expireAt"`
}

func (a *AnalyticsRecord) GetFieldNames() []string {
	val := reflect.ValueOf(a).Elem()
	fields := []string{}

	for i := 0; i < val.NumField(); i++ {
		typeField := val.Type().Field(i)
		fields = append(fields, typeField.Name)
	}

	return fields
}

func (a *AnalyticsRecord) GetLineValues() []string {
	val := reflect.ValueOf(a).Elem()
	fields := []string{}

	for i := 0; i < val.NumField(); i++ {
		valueField := val.Field(i)
		typeField := val.Type().Field(i)
		thisVal := ""
		switch typeField.Type.String() {
		case "int":
			thisVal = strconv.Itoa(int(valueField.Int()))
		case "int64":
			thisVal = strconv.Itoa(int(valueField.Int()))
		case "[]string":
			tmpVal := valueField.Interface().([]string)
			thisVal = strings.Join(tmpVal, ";")
		case "time.Time":
			tmpVal := valueField.Interface().(time.Time)
			thisVal = tmpVal.String()
		case "time.Month":
			tmpVal := valueField.Interface().(time.Month)
			thisVal = tmpVal.String()
		default:
			thisVal = valueField.String()
		}

		fields = append(fields, thisVal)
	}

	return fields
}
<commit_msg>Added IP address to analytics<commit_after>package analytics

import (
	"github.com/lonelycode/tyk-pump/logger"
	"reflect"
	"strconv"
	"strings"
	"time"
)

var log = logger.GetLogger()

// AnalyticsRecord encodes the details of a request
type AnalyticsRecord struct {
	Method        string
	Path          string
	ContentLength int64
	UserAgent     string
	Day           int
	Month         time.Month
	Year          int
	Hour          int
	ResponseCode  int
	APIKey        string
	TimeStamp     time.Time
	APIVersion    string
	APIName       string
	APIID         string
	OrgID         string
	OauthID       string
	RequestTime   int64
	RawRequest    string
	RawResponse   string
	IPAddress     string
	Tags          []string
	ExpireAt      time.Time `bson:"expireAt" json:"expireAt"`
}

func (a *AnalyticsRecord) GetFieldNames() []string {
	val := reflect.ValueOf(a).Elem()
	fields := []string{}

	for i := 0; i < val.NumField(); i++ {
		typeField := val.Type().Field(i)
		fields = append(fields, typeField.Name)
	}

	return fields
}

func (a *AnalyticsRecord) GetLineValues() []string {
	val := reflect.ValueOf(a).Elem()
	fields := []string{}

	for i := 0; i < val.NumField(); i++ {
		valueField := val.Field(i)
		typeField := val.Type().Field(i)
		thisVal := ""
		switch typeField.Type.String() {
		case "int":
			thisVal = strconv.Itoa(int(valueField.Int()))
		case "int64":
			thisVal = strconv.Itoa(int(valueField.Int()))
		case "[]string":
			tmpVal := valueField.Interface().([]string)
			thisVal = strings.Join(tmpVal, ";")
		case "time.Time":
			tmpVal := valueField.Interface().(time.Time)
			thisVal = tmpVal.String()
		case "time.Month":
			tmpVal := valueField.Interface().(time.Month)
			thisVal = tmpVal.String()
		default:
			thisVal = valueField.String()
		}

		fields = append(fields, thisVal)
	}

	return fields
}
<|endoftext|>


<commit_before>package anticaptcha

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

//CaptchaService handles communication with the captcha actions of the
//anti-captcha API
type CaptchaService service

//checkResponse represents check response for have OK message
func checkResponse(body []byte) (string, error) {
	response := string(body)

	if strings.Contains(response, "OK") {
		list := strings.Split(response, "|")
		for i := range list {
			list[i] = strings.TrimSpace(list[i])
		}

		return list[1], nil
	}

	return "", errors.New(response)
}

//UploadCaptchaFromFile represents updaload image(jpg, gif, png) to http://anti-captcha.com API
//and get capthca ID
func (s *CaptchaService) UploadCaptchaFromFile(path string) (int, error) {
	file, err := os.Open(path)
	if err != nil {
		return 0, errors.New("File does not exist")
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile("file", filepath.Base(path))
	if err != nil {
		return 0, err
	}
	_, err = io.Copy(part, file)
	_ = writer.WriteField("key", s.client.APIKey)
	_ = writer.WriteField("method", "post")
	err = writer.Close()
	if err != nil {
		return 0, err
	}
	req, err := s.client.NewRequest("POST", "in.php", body)
	if err != nil {
		return 0, err
	}
	req.Header.Set("Content-Type", writer.FormDataContentType())

	data, err := s.client.Do(req)
	if err != nil {
		return 0, err
	}

	res, err := checkResponse(data)
	if err != nil {
		return 0, err
	}
	captchaID, err := strconv.Atoi(res)
	if err != nil {
		return 0, err
	}
	return captchaID, nil
}

//UploadCaptchaFromBase64 represents updaload base64 string to http://anti-captcha.com API
//and get capthca ID
func (s *CaptchaService) UploadCaptchaFromBase64(base64 string) (int, error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	_ = writer.WriteField("body", base64)
	_ = writer.WriteField("key", s.client.APIKey)
	_ = writer.WriteField("method", "base64")
	err := writer.Close()
	if err != nil {
		return 0, err
	}

	req, err := s.client.NewRequest("POST", "in.php", body)
	if err != nil {
		return 0, err
	}
	req.Header.Set("Content-Type", writer.FormDataContentType())
	data, err := s.client.Do(req)
	if err != nil {
		return 0, err
	}

	res, err := checkResponse(data)
	if err != nil {
		return 0, err
	}
	captchaID, err := strconv.Atoi(res)
	if err != nil {
		return 0, err
	}
	return captchaID, nil
}

//GetText represents return captcha text by ID
func (s *CaptchaService) GetText(id int) (string, error) {
	reqURL := fmt.Sprintf("res.php?key=%s&action=get&id=%d", s.client.APIKey, id)

	req, err := s.client.NewRequest("GET", reqURL, nil)
	if err != nil {
		return "", err
	}

	res, err := s.client.Do(req)
	if err != nil {
		return "", err
	}

	resOk, err := checkResponse(res)
	if err != nil {
		if fmt.Sprintf("%s", err) == "CAPCHA_NOT_READY" {
			time.Sleep(5 * time.Second)
			return s.GetText(id)
		}
		return "", err
	}

	return resOk, err
}
<commit_msg>refact upload captcha<commit_after>package anticaptcha

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"strconv"
	"strings"
	"time"
)

//CaptchaService handles communication with the captcha actions of the
//anti-captcha API
type CaptchaService service

//UploadCaptchaFromFile represents updaload image(jpg, gif, png) to http://anti-captcha.com API
//and get capthca ID
func (s *CaptchaService) UploadCaptchaFromFile(path string) (int, error) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, errors.New("File does not exist")
	}
	strBase64 := base64.StdEncoding.EncodeToString(content)
	captchaID, err := s.uploadCaptcha(strBase64)
	if err != nil {
		return 0, err
	}
	return captchaID, err
}

//UploadCaptchaFromBase64 represents updaload base64 string to http://anti-captcha.com API
//and get capthca ID
func (s *CaptchaService) UploadCaptchaFromBase64(base64 string) (int, error) {
	captchaID, err := s.uploadCaptcha(base64)
	if err != nil {
		return 0, err
	}
	return captchaID, err
}

//GetText represents return captcha text by ID
func (s *CaptchaService) GetText(id int) (string, error) {
	reqURL := fmt.Sprintf("res.php?key=%s&action=get&id=%d", s.client.APIKey, id)

	req, err := s.client.NewRequest("GET", reqURL, nil)
	if err != nil {
		return "", err
	}

	res, err := s.client.Do(req)
	if err != nil {
		return "", err
	}

	resOk, err := checkResponse(res)
	if err != nil {
		if fmt.Sprintf("%s", err) == "CAPCHA_NOT_READY" {
			time.Sleep(5 * time.Second)
			return s.GetText(id)
		}
		return "", err
	}

	return resOk, err
}

//checkResponse represents check response for have OK message
func checkResponse(body []byte) (string, error) {
	response := string(body)

	if strings.Contains(response, "OK") {
		list := strings.Split(response, "|")
		for i := range list {
			list[i] = strings.TrimSpace(list[i])
		}

		return list[1], nil
	}

	return "", errors.New(response)
}

//uploadCaptcha represents updaload base64 string to http://anti-captcha.com API
//and get capthca ID
func (s CaptchaService) uploadCaptcha(base64 string) (int, error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	_ = writer.WriteField("body", base64)
	_ = writer.WriteField("key", s.client.APIKey)
	_ = writer.WriteField("method", "base64")
	err := writer.Close()
	if err != nil {
		return 0, err
	}

	req, err := s.client.NewRequest("POST", "in.php", body)
	if err != nil {
		return 0, err
	}
	req.Header.Set("Content-Type", writer.FormDataContentType())
	data, err := s.client.Do(req)
	if err != nil {
		return 0, err
	}

	res, err := checkResponse(data)
	if err != nil {
		return 0, err
	}
	captchaID, err := strconv.Atoi(res)
	if err != nil {
		return 0, err
	}
	return captchaID, nil
}
<|endoftext|>


<commit_before>// Unit is a non-static game object
package game

import (
	"fmt"
)

const (
	DEFAULT_HP = 10
)

type Unit struct {
	*GameObject
	*Inventory

	Level     int
	Hp, HpMax int
}

func NewUnit(name string) *Unit {
	u := &Unit{Level: 1,
		Hp:    DEFAULT_HP,
		HpMax: DEFAULT_HP,
	}
	u.Inventory = NewInventory()
	u.GameObject = NewGameObject(name)

	return u
}

func (u Unit) String() string {
	return fmt.Sprintf("%s: Hp: %d(%d) %s", u.Name, u.Hp, u.HpMax, u.GameObject)
}
<commit_msg>Added Unit.HasItem(*Item)<commit_after>// Unit is a non-static game object
package game

import (
	"fmt"
)

const (
	DEFAULT_HP = 10
)

type Unit struct {
	*GameObject
	*Inventory

	Level     int
	Hp, HpMax int
}

func NewUnit(name string) *Unit {
	u := &Unit{Level: 1,
		Hp:    DEFAULT_HP,
		HpMax: DEFAULT_HP,
	}
	u.Inventory = NewInventory()
	u.GameObject = NewGameObject(name)

	return u
}

// Checks if a Unit HasItem *Item
func (u Unit) HasItem (i *Item) bool {
	if u.Inventory.ContainsItem(i) {
		return true
	}
	return false
}

func (u Unit) String() string {
	return fmt.Sprintf("%s: Hp: %d(%d) %s", u.Name, u.Hp, u.HpMax, u.GameObject)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/url"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/api/googleapi"
)

// A bucket that wraps another, calling its methods in a retry loop with
// randomized exponential backoff.
type retryBucket struct {
	maxSleep time.Duration
	wrapped  Bucket
}

func newRetryBucket(
	maxSleep time.Duration,
	wrapped Bucket) (b Bucket) {
	b = &retryBucket{
		maxSleep: maxSleep,
		wrapped:  wrapped,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func shouldRetry(err error) (b bool) {
	// HTTP 50x errors.
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code >= 500 && typed.Code < 600 {
			b = true
			return
		}
	}

	// HTTP 429 errors (GCS uses these for rate limiting).
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code == 429 {
			b = true
			return
		}
	}

	// Network errors, which tend to show up transiently when doing lots of
	// operations in parallel. For example:
	//
	//     dial tcp 74.125.203.95:443: too many open files
	//
	if _, ok := err.(*net.OpError); ok {
		b = true
		return
	}

	// Sometimes the HTTP package helpfully encapsulates the real error in a URL
	// error.
	if urlErr, ok := err.(*url.Error); ok {
		b = shouldRetry(urlErr.Err)
		return
	}

	return
}

// Choose an appropriate delay for exponential backoff, given that we have
// already slept the given number of times for this logical request.
func chooseDelay(prevSleepCount uint) (d time.Duration) {
	const baseDelay = time.Millisecond

	// Choose a a delay in [0, 2^prevSleepCount * baseDelay).
	d = (1 << prevSleepCount) * baseDelay
	d = time.Duration(float64(d) * rand.Float64())

	return
}

// Exponential backoff for a function that might fail.
//
// This is essentially what is described in the "Best practices" section of the
// "Upload Objects" docs:
//
//     https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
//
// with the following exceptions:
//
//  *  We perform backoff for all operations.
//
//  *  The random component scales with the delay, so that the first sleep
//     cannot be as long as one second. The algorithm used matches the
//     description at http://en.wikipedia.org/wiki/Exponential_backoff.
//
//  *  We retry more types of errors; see shouldRetry above.
//
// State for total sleep time and number of previous sleeps is housed outside
// of this function to allow it to be "resumed" by multiple invocations of
// retryObjectReader.Read.
func expBackoff(
	ctx context.Context,
	desc string,
	maxSleep time.Duration,
	f func() error,
	prevSleepCount *uint,
	prevSleepDuration *time.Duration) (err error) {
	for {
		// Make an attempt. Stop if successful.
		err = f()
		if err == nil {
			return
		}

		// Do we want to retry?
		if !shouldRetry(err) {
			// Special case: don't spam up the logs for EOF, which io.Reader returns
			// in the normal course of things.
			if err != io.EOF {
				log.Printf(
					"Not retrying %s after error of type %T (%q): %#v",
					desc,
					err,
					err.Error(),
					err)
			}

			return
		}

		// Choose a a delay.
		d := chooseDelay(*prevSleepCount)
		*prevSleepCount++

		// Are we out of credit?
		if *prevSleepDuration+d > maxSleep {
			// Return the most recent error.
			return
		}

		// Sleep, returning early if cancelled.
		log.Printf(
			"Retrying %s after error of type %T (%q) in %v",
			desc,
			err,
			err,
			d)

		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case <-time.After(d):
			*prevSleepDuration += d
			continue
		}
	}
}

// Like expBackoff, but assumes that we've never slept before (and won't need
// to sleep again).
func oneShotExpBackoff(
	ctx context.Context,
	desc string,
	maxSleep time.Duration,
	f func() error) (err error) {
	var prevSleepCount uint
	var prevSleepDuration time.Duration

	err = expBackoff(
		ctx,
		desc,
		maxSleep,
		f,
		&prevSleepCount,
		&prevSleepDuration)

	return
}

////////////////////////////////////////////////////////////////////////
// Read support
////////////////////////////////////////////////////////////////////////

type retryObjectReader struct {
	bucket *retryBucket

	// The context we should watch when sleeping for retries.
	ctx context.Context

	// What we are trying to read.
	name       string
	generation int64
	byteRange  ByteRange

	// nil when we start or have seen a permanent error.
	wrapped io.ReadCloser

	// If we've seen an error that we shouldn't retry for, this will be non-nil
	// and should be returned permanently.
	permanentErr error

	// The number of times we've slept so far, and the total amount of time we've
	// spent sleeping.
	sleepCount    uint
	sleepDuration time.Duration
}

// Set up the wrapped reader.
func (rc *retryObjectReader) setUpWrapped() (err error) {
	// Call through to create the reader.
	req := &ReadObjectRequest{
		Name:       rc.name,
		Generation: rc.generation,
		Range:      &rc.byteRange,
	}

	wrapped, err := rc.bucket.wrapped.NewReader(rc.ctx, req)
	if err != nil {
		return
	}

	rc.wrapped = wrapped
	return
}

// Set up the wrapped reader if necessary, and make one attempt to read through
// it.
//
// Clears the wrapped reader on error.
func (rc *retryObjectReader) readOnce(p []byte) (n int, err error) {
	// Set up the wrapped reader if it's not already around.
	if rc.wrapped == nil {
		err = rc.setUpWrapped()
		if err != nil {
			return
		}
	}

	// Attempt to read from it.
	n, err = rc.wrapped.Read(p)
	if err != nil {
		rc.wrapped.Close()
		rc.wrapped = nil
		return
	}

	return
}

// Invariant: we never return an error from this function unless we've given up
// on retrying. In particular, we won't return a short read because the wrapped
// reader returned a short read and an error.
func (rc *retryObjectReader) Read(p []byte) (n int, err error) {
	// Whatever we do, accumulate the bytes that we're returning to the user.
	defer func() {
		if n < 0 {
			panic(fmt.Sprintf("Negative byte count: %d", n))
		}

		rc.byteRange.Start += uint64(n)
	}()

	// If we've already decided on a permanent error, return that.
	if rc.permanentErr != nil {
		err = rc.permanentErr
		return
	}

	// If we let an error escape below, it must be a permanent one.
	defer func() {
		if err != nil {
			rc.permanentErr = err
		}
	}()

	// We will repeatedly make single attempts until we get a successful request.
	// Don't forget to accumulate the result each time.
	tryOnce := func() (err error) {
		var bytesRead int
		bytesRead, err = rc.readOnce(p)
		n += bytesRead
		p = p[bytesRead:]

		return
	}

	err = expBackoff(
		rc.ctx,
		fmt.Sprintf("Read(%q, %d)", rc.name, rc.generation),
		rc.bucket.maxSleep,
		tryOnce,
		&rc.sleepCount,
		&rc.sleepDuration)

	return
}

func (rc *retryObjectReader) Close() (err error) {
	// If we don't have a wrapped reader, there is nothing useful that we can or
	// need to do here.
	if rc.wrapped == nil {
		return
	}

	// Call through.
	err = rc.wrapped.Close()

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (rb *retryBucket) Name() (name string) {
	name = rb.wrapped.Name()
	return
}

func (rb *retryBucket) NewReader(
	ctx context.Context,
	req *ReadObjectRequest) (rc io.ReadCloser, err error) {
	// If the user specified the latest generation, we need to figure out what
	// that is so that we can create a reader that knows how to keep a stable
	// generation despite retrying repeatedly.
	var generation int64 = req.Generation
	var sleepCount uint
	var sleepDuration time.Duration

	if generation == 0 {
		findGeneration := func() (err error) {
			o, err := rb.wrapped.StatObject(
				ctx,
				&StatObjectRequest{
					Name: req.Name,
				})

			if err != nil {
				return
			}

			generation = o.Generation
			return
		}

		err = expBackoff(
			ctx,
			fmt.Sprintf("FindLatestGeneration(%q)", req.Name),
			rb.maxSleep,
			findGeneration,
			&sleepCount,
			&sleepDuration)

		if err != nil {
			return
		}
	}

	// Choose an appropriate byte range.
	byteRange := ByteRange{0, math.MaxUint64}
	if req.Range != nil {
		byteRange = *req.Range
	}

	// Now that we know what generation we're looking for, return an appropriate
	// reader that knows how to retry when the connection fails. Make sure to
	// inherit the time spent sleeping above.
	rc = &retryObjectReader{
		bucket: rb,
		ctx:    ctx,

		name:       req.Name,
		generation: generation,
		byteRange:  byteRange,

		sleepCount:    sleepCount,
		sleepDuration: sleepDuration,
	}

	return
}

func (rb *retryBucket) CreateObject(
	ctx context.Context,
	req *CreateObjectRequest) (o *Object, err error) {
	// We can't simply replay the request multiple times, because the first
	// attempt might exhaust some of the req.Contents reader, leaving missing
	// contents for the second attempt.
	//
	// So, copy out all contents and create a modified request that serves from
	// memory.
	contents, err := ioutil.ReadAll(req.Contents)
	if err != nil {
		err = fmt.Errorf("ioutil.ReadAll: %v", err)
		return
	}

	reqCopy := *req
	reqCopy.Contents = bytes.NewReader(contents)

	// Call through with that request.
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("CreateObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.CreateObject(ctx, &reqCopy)
			return
		})

	return
}

func (rb *retryBucket) CopyObject(
	ctx context.Context,
	req *CopyObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("CopyObject(%q, %q)", req.SrcName, req.DstName),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.CopyObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) StatObject(
	ctx context.Context,
	req *StatObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("StatObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.StatObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) ListObjects(
	ctx context.Context,
	req *ListObjectsRequest) (listing *Listing, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("ListObjects(%q)", req.Prefix),
		rb.maxSleep,
		func() (err error) {
			listing, err = rb.wrapped.ListObjects(ctx, req)
			return
		})
	return
}

func (rb *retryBucket) UpdateObject(
	ctx context.Context,
	req *UpdateObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("UpdateObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.UpdateObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) DeleteObject(
	ctx context.Context,
	name string) (err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("DeleteObject(%q)", name),
		rb.maxSleep,
		func() (err error) {
			err = rb.wrapped.DeleteObject(ctx, name)
			return
		})

	return
}
<commit_msg>Fixed CancellationTest.ReadObject with retry enabled.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/url"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/api/googleapi"
)

// A bucket that wraps another, calling its methods in a retry loop with
// randomized exponential backoff.
type retryBucket struct {
	maxSleep time.Duration
	wrapped  Bucket
}

func newRetryBucket(
	maxSleep time.Duration,
	wrapped Bucket) (b Bucket) {
	b = &retryBucket{
		maxSleep: maxSleep,
		wrapped:  wrapped,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func shouldRetry(err error) (b bool) {
	// HTTP 50x errors.
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code >= 500 && typed.Code < 600 {
			b = true
			return
		}
	}

	// HTTP 429 errors (GCS uses these for rate limiting).
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code == 429 {
			b = true
			return
		}
	}

	// Network errors, which tend to show up transiently when doing lots of
	// operations in parallel. For example:
	//
	//     dial tcp 74.125.203.95:443: too many open files
	//
	if _, ok := err.(*net.OpError); ok {
		b = true
		return
	}

	// Sometimes the HTTP package helpfully encapsulates the real error in a URL
	// error.
	if urlErr, ok := err.(*url.Error); ok {
		b = shouldRetry(urlErr.Err)
		return
	}

	return
}

// Choose an appropriate delay for exponential backoff, given that we have
// already slept the given number of times for this logical request.
func chooseDelay(prevSleepCount uint) (d time.Duration) {
	const baseDelay = time.Millisecond

	// Choose a a delay in [0, 2^prevSleepCount * baseDelay).
	d = (1 << prevSleepCount) * baseDelay
	d = time.Duration(float64(d) * rand.Float64())

	return
}

// Exponential backoff for a function that might fail.
//
// This is essentially what is described in the "Best practices" section of the
// "Upload Objects" docs:
//
//     https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
//
// with the following exceptions:
//
//  *  We perform backoff for all operations.
//
//  *  The random component scales with the delay, so that the first sleep
//     cannot be as long as one second. The algorithm used matches the
//     description at http://en.wikipedia.org/wiki/Exponential_backoff.
//
//  *  We retry more types of errors; see shouldRetry above.
//
// State for total sleep time and number of previous sleeps is housed outside
// of this function to allow it to be "resumed" by multiple invocations of
// retryObjectReader.Read.
func expBackoff(
	ctx context.Context,
	desc string,
	maxSleep time.Duration,
	f func() error,
	prevSleepCount *uint,
	prevSleepDuration *time.Duration) (err error) {
	for {
		// Make an attempt. Stop if successful.
		err = f()
		if err == nil {
			return
		}

		// Do we want to retry?
		if !shouldRetry(err) {
			// Special case: don't spam up the logs for EOF, which io.Reader returns
			// in the normal course of things.
			if err != io.EOF {
				log.Printf(
					"Not retrying %s after error of type %T (%q): %#v",
					desc,
					err,
					err.Error(),
					err)
			}

			return
		}

		// Choose a a delay.
		d := chooseDelay(*prevSleepCount)
		*prevSleepCount++

		// Are we out of credit?
		if *prevSleepDuration+d > maxSleep {
			// Return the most recent error.
			return
		}

		// Sleep, returning early if cancelled.
		log.Printf(
			"Retrying %s after error of type %T (%q) in %v",
			desc,
			err,
			err,
			d)

		select {
		case <-ctx.Done():
			// On cancellation, return the last error we saw.
			return

		case <-time.After(d):
			*prevSleepDuration += d
			continue
		}
	}
}

// Like expBackoff, but assumes that we've never slept before (and won't need
// to sleep again).
func oneShotExpBackoff(
	ctx context.Context,
	desc string,
	maxSleep time.Duration,
	f func() error) (err error) {
	var prevSleepCount uint
	var prevSleepDuration time.Duration

	err = expBackoff(
		ctx,
		desc,
		maxSleep,
		f,
		&prevSleepCount,
		&prevSleepDuration)

	return
}

////////////////////////////////////////////////////////////////////////
// Read support
////////////////////////////////////////////////////////////////////////

type retryObjectReader struct {
	bucket *retryBucket

	// The context we should watch when sleeping for retries.
	ctx context.Context

	// What we are trying to read.
	name       string
	generation int64
	byteRange  ByteRange

	// nil when we start or have seen a permanent error.
	wrapped io.ReadCloser

	// If we've seen an error that we shouldn't retry for, this will be non-nil
	// and should be returned permanently.
	permanentErr error

	// The number of times we've slept so far, and the total amount of time we've
	// spent sleeping.
	sleepCount    uint
	sleepDuration time.Duration
}

// Set up the wrapped reader.
func (rc *retryObjectReader) setUpWrapped() (err error) {
	// Call through to create the reader.
	req := &ReadObjectRequest{
		Name:       rc.name,
		Generation: rc.generation,
		Range:      &rc.byteRange,
	}

	wrapped, err := rc.bucket.wrapped.NewReader(rc.ctx, req)
	if err != nil {
		return
	}

	rc.wrapped = wrapped
	return
}

// Set up the wrapped reader if necessary, and make one attempt to read through
// it.
//
// Clears the wrapped reader on error.
func (rc *retryObjectReader) readOnce(p []byte) (n int, err error) {
	// Set up the wrapped reader if it's not already around.
	if rc.wrapped == nil {
		err = rc.setUpWrapped()
		if err != nil {
			return
		}
	}

	// Attempt to read from it.
	n, err = rc.wrapped.Read(p)
	if err != nil {
		rc.wrapped.Close()
		rc.wrapped = nil
		return
	}

	return
}

// Invariant: we never return an error from this function unless we've given up
// on retrying. In particular, we won't return a short read because the wrapped
// reader returned a short read and an error.
func (rc *retryObjectReader) Read(p []byte) (n int, err error) {
	// Whatever we do, accumulate the bytes that we're returning to the user.
	defer func() {
		if n < 0 {
			panic(fmt.Sprintf("Negative byte count: %d", n))
		}

		rc.byteRange.Start += uint64(n)
	}()

	// If we've already decided on a permanent error, return that.
	if rc.permanentErr != nil {
		err = rc.permanentErr
		return
	}

	// If we let an error escape below, it must be a permanent one.
	defer func() {
		if err != nil {
			rc.permanentErr = err
		}
	}()

	// We will repeatedly make single attempts until we get a successful request.
	// Don't forget to accumulate the result each time.
	tryOnce := func() (err error) {
		var bytesRead int
		bytesRead, err = rc.readOnce(p)
		n += bytesRead
		p = p[bytesRead:]

		return
	}

	err = expBackoff(
		rc.ctx,
		fmt.Sprintf("Read(%q, %d)", rc.name, rc.generation),
		rc.bucket.maxSleep,
		tryOnce,
		&rc.sleepCount,
		&rc.sleepDuration)

	return
}

func (rc *retryObjectReader) Close() (err error) {
	// If we don't have a wrapped reader, there is nothing useful that we can or
	// need to do here.
	if rc.wrapped == nil {
		return
	}

	// Call through.
	err = rc.wrapped.Close()

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (rb *retryBucket) Name() (name string) {
	name = rb.wrapped.Name()
	return
}

func (rb *retryBucket) NewReader(
	ctx context.Context,
	req *ReadObjectRequest) (rc io.ReadCloser, err error) {
	// If the user specified the latest generation, we need to figure out what
	// that is so that we can create a reader that knows how to keep a stable
	// generation despite retrying repeatedly.
	var generation int64 = req.Generation
	var sleepCount uint
	var sleepDuration time.Duration

	if generation == 0 {
		findGeneration := func() (err error) {
			o, err := rb.wrapped.StatObject(
				ctx,
				&StatObjectRequest{
					Name: req.Name,
				})

			if err != nil {
				return
			}

			generation = o.Generation
			return
		}

		err = expBackoff(
			ctx,
			fmt.Sprintf("FindLatestGeneration(%q)", req.Name),
			rb.maxSleep,
			findGeneration,
			&sleepCount,
			&sleepDuration)

		if err != nil {
			return
		}
	}

	// Choose an appropriate byte range.
	byteRange := ByteRange{0, math.MaxUint64}
	if req.Range != nil {
		byteRange = *req.Range
	}

	// Now that we know what generation we're looking for, return an appropriate
	// reader that knows how to retry when the connection fails. Make sure to
	// inherit the time spent sleeping above.
	rc = &retryObjectReader{
		bucket: rb,
		ctx:    ctx,

		name:       req.Name,
		generation: generation,
		byteRange:  byteRange,

		sleepCount:    sleepCount,
		sleepDuration: sleepDuration,
	}

	return
}

func (rb *retryBucket) CreateObject(
	ctx context.Context,
	req *CreateObjectRequest) (o *Object, err error) {
	// We can't simply replay the request multiple times, because the first
	// attempt might exhaust some of the req.Contents reader, leaving missing
	// contents for the second attempt.
	//
	// So, copy out all contents and create a modified request that serves from
	// memory.
	contents, err := ioutil.ReadAll(req.Contents)
	if err != nil {
		err = fmt.Errorf("ioutil.ReadAll: %v", err)
		return
	}

	reqCopy := *req
	reqCopy.Contents = bytes.NewReader(contents)

	// Call through with that request.
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("CreateObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.CreateObject(ctx, &reqCopy)
			return
		})

	return
}

func (rb *retryBucket) CopyObject(
	ctx context.Context,
	req *CopyObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("CopyObject(%q, %q)", req.SrcName, req.DstName),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.CopyObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) StatObject(
	ctx context.Context,
	req *StatObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("StatObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.StatObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) ListObjects(
	ctx context.Context,
	req *ListObjectsRequest) (listing *Listing, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("ListObjects(%q)", req.Prefix),
		rb.maxSleep,
		func() (err error) {
			listing, err = rb.wrapped.ListObjects(ctx, req)
			return
		})
	return
}

func (rb *retryBucket) UpdateObject(
	ctx context.Context,
	req *UpdateObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("UpdateObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.UpdateObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) DeleteObject(
	ctx context.Context,
	name string) (err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("DeleteObject(%q)", name),
		rb.maxSleep,
		func() (err error) {
			err = rb.wrapped.DeleteObject(ctx, name)
			return
		})

	return
}
<|endoftext|>