<commit_before>package utp

/*
#include "utp.h"
*/
import "C"
import (
	"bytes"
	"context"
	"errors"
	"io"
	"net"
	"sync"
	"syscall"
	"time"
	"unsafe"
)

type Conn struct {
	s          *C.utp_socket
	cond       sync.Cond
	readBuf    bytes.Buffer
	gotEOF     bool
	gotConnect bool
	// Set on state changed to UTP_STATE_DESTROYING. Not valid to refer to the
	// socket after getting this.
	destroyed bool
	// Conn.Close was called.
	closed   bool
	libError error

	writeDeadline      time.Time
	writeDeadlineTimer *time.Timer
	readDeadline       time.Time
	readDeadlineTimer  *time.Timer

	numBytesRead    int64
	numBytesWritten int64
}

func (c *Conn) onLibError(codeName string) {
	c.libError = errors.New(codeName)
	c.cond.Broadcast()
}

func (c *Conn) setConnected() {
	c.gotConnect = true
	c.cond.Broadcast()
}

func (c *Conn) waitForConnect(ctx context.Context) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	go func() {
		<-ctx.Done()
		c.cond.Broadcast()
	}()
	for {
		if c.libError != nil {
			return c.libError
		}
		if c.gotConnect {
			return nil
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
		c.cond.Wait()
	}
}

func (c *Conn) Close() error {
	mu.Lock()
	defer mu.Unlock()
	c.close()
	return nil
}

func (c *Conn) close() {
	// log.Printf("conn %p: closed", c)
	if c.closed {
		return
	}
	if !c.destroyed {
		C.utp_close(c.s)
		// C.utp_issue_deferred_acks(C.utp_get_context(c.s))
	}
	c.closed = true
	c.cond.Broadcast()
}

func (c *Conn) LocalAddr() net.Addr {
	mu.Lock()
	defer mu.Unlock()
	return getSocketForLibContext(C.utp_get_context(c.s)).pc.LocalAddr()
}

func (c *Conn) readNoWait(b []byte) (n int, err error) {
	n, _ = c.readBuf.Read(b)
	if n != 0 && c.readBuf.Len() == 0 {
		// Can we call this if the utp_socket is closed, destroyed or errored?
		if c.s != nil {
			C.utp_read_drained(c.s)
			// C.utp_issue_deferred_acks(C.utp_get_context(c.s))
		}
	}
	if c.readBuf.Len() != 0 {
		return
	}
	err = func() error {
		switch {
		case c.gotEOF:
			return io.EOF
		case c.libError != nil:
			return c.libError
		case c.destroyed:
			return errors.New("destroyed")
		case c.closed:
			return errors.New("closed")
		case !c.readDeadline.IsZero() && !time.Now().Before(c.readDeadline):
			return errDeadlineExceeded{}
		default:
			return nil
		}
	}()
	return
}

func (c *Conn) Read(b []byte) (int, error) {
	mu.Lock()
	defer mu.Unlock()
	for {
		n, err := c.readNoWait(b)
		c.numBytesRead += int64(n)
		// log.Printf("read %d bytes", c.numBytesRead)
		if n != 0 || len(b) == 0 || err != nil {
			// log.Printf("conn %p: read %d bytes: %s", c, n, err)
			return n, err
		}
		c.cond.Wait()
	}
}

func (c *Conn) writeNoWait(b []byte) (n int, err error) {
	err = func() error {
		switch {
		case c.libError != nil:
			return c.libError
		case c.closed:
			return errors.New("closed")
		case c.destroyed:
			return errors.New("destroyed")
		case !c.writeDeadline.IsZero() && !time.Now().Before(c.writeDeadline):
			return errDeadlineExceeded{}
		default:
			return nil
		}
	}()
	if err != nil {
		return
	}
	n = int(C.utp_write(c.s, unsafe.Pointer(&b[0]), C.size_t(len(b))))
	if n < 0 {
		panic(n)
	}
	// log.Print(n)
	// C.utp_issue_deferred_acks(C.utp_get_context(c.s))
	return
}

func (c *Conn) Write(b []byte) (n int, err error) {
	// defer func() { log.Printf("wrote %d bytes: %s", n, err) }()
	// log.Print(len(b))
	mu.Lock()
	defer mu.Unlock()
	for len(b) != 0 {
		var n1 int
		n1, err = c.writeNoWait(b)
		b = b[n1:]
		n += n1
		if err != nil {
			break
		}
		if n1 != 0 {
			continue
		}
		c.cond.Wait()
	}
	c.numBytesWritten += int64(n)
	// log.Printf("wrote %d bytes", c.numBytesWritten)
	return
}

func (c *Conn) RemoteAddr() net.Addr {
	var rsa syscall.RawSockaddrAny
	var addrlen C.socklen_t = C.socklen_t(unsafe.Sizeof(rsa))
	C.utp_getpeername(c.s, (*C.struct_sockaddr)(unsafe.Pointer(&rsa)), &addrlen)
	sa, err := anyToSockaddr(&rsa)
	if err != nil {
		panic(err)
	}
	return sockaddrToUDP(sa)
}

func (c *Conn) SetDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.readDeadline = t
	c.writeDeadline = t
	if t.IsZero() {
		c.readDeadlineTimer.Stop()
		c.writeDeadlineTimer.Stop()
	} else {
		d := t.Sub(time.Now())
		c.readDeadlineTimer.Reset(d)
		c.writeDeadlineTimer.Reset(d)
	}
	c.cond.Broadcast()
	return nil
}
func (c *Conn) SetReadDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.readDeadline = t
	if t.IsZero() {
		c.readDeadlineTimer.Stop()
	} else {
		d := t.Sub(time.Now())
		c.readDeadlineTimer.Reset(d)
	}
	c.cond.Broadcast()
	return nil
}
func (c *Conn) SetWriteDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.writeDeadline = t
	if t.IsZero() {
		c.writeDeadlineTimer.Stop()
	} else {
		d := t.Sub(time.Now())
		c.writeDeadlineTimer.Reset(d)
	}
	c.cond.Broadcast()
	return nil
}

func (c *Conn) setGotEOF() {
	c.gotEOF = true
	c.cond.Broadcast()
}

func (c *Conn) onDestroyed() {
	c.destroyed = true
	c.s = nil
	c.cond.Broadcast()
}

func (c *Conn) WriteBufferLen() int {
	mu.Lock()
	defer mu.Unlock()
	return int(C.utp_getsockopt(c.s, C.UTP_SNDBUF))
}

func (c *Conn) SetWriteBufferLen(len int) {
	mu.Lock()
	defer mu.Unlock()
	i := C.utp_setsockopt(c.s, C.UTP_SNDBUF, C.int(len))
	if i != 0 {
		panic(i)
	}
}
<commit_msg>Add fix for #6<commit_after>package utp

/*
#include "utp.h"
*/
import "C"
import (
	"bytes"
	"context"
	"errors"
	"io"
	"net"
	"sync"
	"syscall"
	"time"
	"unsafe"
)

type Conn struct {
	s          *C.utp_socket
	cond       sync.Cond
	readBuf    bytes.Buffer
	gotEOF     bool
	gotConnect bool
	// Set on state changed to UTP_STATE_DESTROYING. Not valid to refer to the
	// socket after getting this.
	destroyed bool
	// Conn.Close was called.
	closed   bool
	libError error

	writeDeadline      time.Time
	writeDeadlineTimer *time.Timer
	readDeadline       time.Time
	readDeadlineTimer  *time.Timer

	numBytesRead    int64
	numBytesWritten int64
}

func (c *Conn) onLibError(codeName string) {
	c.libError = errors.New(codeName)
	c.cond.Broadcast()
}

func (c *Conn) setConnected() {
	c.gotConnect = true
	c.cond.Broadcast()
}

func (c *Conn) waitForConnect(ctx context.Context) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	go func() {
		<-ctx.Done()
		c.cond.Broadcast()
	}()
	for {
		if c.libError != nil {
			return c.libError
		}
		if c.gotConnect {
			return nil
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
		c.cond.Wait()
	}
}

func (c *Conn) Close() error {
	mu.Lock()
	defer mu.Unlock()
	c.close()
	return nil
}

func (c *Conn) close() {
	// log.Printf("conn %p: closed", c)
	if c.closed {
		return
	}
	if !c.destroyed {
		C.utp_close(c.s)
		// C.utp_issue_deferred_acks(C.utp_get_context(c.s))
	}
	c.closed = true
	c.cond.Broadcast()
}

func (c *Conn) LocalAddr() net.Addr {
	mu.Lock()
	defer mu.Unlock()
	if c.s == nil {
		return nil
	}
	return getSocketForLibContext(C.utp_get_context(c.s)).pc.LocalAddr()
}

func (c *Conn) readNoWait(b []byte) (n int, err error) {
	n, _ = c.readBuf.Read(b)
	if n != 0 && c.readBuf.Len() == 0 {
		// Can we call this if the utp_socket is closed, destroyed or errored?
		if c.s != nil {
			C.utp_read_drained(c.s)
			// C.utp_issue_deferred_acks(C.utp_get_context(c.s))
		}
	}
	if c.readBuf.Len() != 0 {
		return
	}
	err = func() error {
		switch {
		case c.gotEOF:
			return io.EOF
		case c.libError != nil:
			return c.libError
		case c.destroyed:
			return errors.New("destroyed")
		case c.closed:
			return errors.New("closed")
		case !c.readDeadline.IsZero() && !time.Now().Before(c.readDeadline):
			return errDeadlineExceeded{}
		default:
			return nil
		}
	}()
	return
}

func (c *Conn) Read(b []byte) (int, error) {
	mu.Lock()
	defer mu.Unlock()
	for {
		n, err := c.readNoWait(b)
		c.numBytesRead += int64(n)
		// log.Printf("read %d bytes", c.numBytesRead)
		if n != 0 || len(b) == 0 || err != nil {
			// log.Printf("conn %p: read %d bytes: %s", c, n, err)
			return n, err
		}
		c.cond.Wait()
	}
}

func (c *Conn) writeNoWait(b []byte) (n int, err error) {
	err = func() error {
		switch {
		case c.libError != nil:
			return c.libError
		case c.closed:
			return errors.New("closed")
		case c.destroyed:
			return errors.New("destroyed")
		case !c.writeDeadline.IsZero() && !time.Now().Before(c.writeDeadline):
			return errDeadlineExceeded{}
		default:
			return nil
		}
	}()
	if err != nil {
		return
	}
	n = int(C.utp_write(c.s, unsafe.Pointer(&b[0]), C.size_t(len(b))))
	if n < 0 {
		panic(n)
	}
	// log.Print(n)
	// C.utp_issue_deferred_acks(C.utp_get_context(c.s))
	return
}

func (c *Conn) Write(b []byte) (n int, err error) {
	// defer func() { log.Printf("wrote %d bytes: %s", n, err) }()
	// log.Print(len(b))
	mu.Lock()
	defer mu.Unlock()
	for len(b) != 0 {
		var n1 int
		n1, err = c.writeNoWait(b)
		b = b[n1:]
		n += n1
		if err != nil {
			break
		}
		if n1 != 0 {
			continue
		}
		c.cond.Wait()
	}
	c.numBytesWritten += int64(n)
	// log.Printf("wrote %d bytes", c.numBytesWritten)
	return
}

func (c *Conn) RemoteAddr() net.Addr {
	var rsa syscall.RawSockaddrAny
	var addrlen C.socklen_t = C.socklen_t(unsafe.Sizeof(rsa))
	if c.s == nil {
		return nil
	}
	C.utp_getpeername(c.s, (*C.struct_sockaddr)(unsafe.Pointer(&rsa)), &addrlen)
	sa, err := anyToSockaddr(&rsa)
	if err != nil {
		panic(err)
	}
	return sockaddrToUDP(sa)
}

func (c *Conn) SetDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.readDeadline = t
	c.writeDeadline = t
	if t.IsZero() {
		c.readDeadlineTimer.Stop()
		c.writeDeadlineTimer.Stop()
	} else {
		d := t.Sub(time.Now())
		c.readDeadlineTimer.Reset(d)
		c.writeDeadlineTimer.Reset(d)
	}
	c.cond.Broadcast()
	return nil
}
func (c *Conn) SetReadDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.readDeadline = t
	if t.IsZero() {
		c.readDeadlineTimer.Stop()
	} else {
		d := t.Sub(time.Now())
		c.readDeadlineTimer.Reset(d)
	}
	c.cond.Broadcast()
	return nil
}
func (c *Conn) SetWriteDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.writeDeadline = t
	if t.IsZero() {
		c.writeDeadlineTimer.Stop()
	} else {
		d := t.Sub(time.Now())
		c.writeDeadlineTimer.Reset(d)
	}
	c.cond.Broadcast()
	return nil
}

func (c *Conn) setGotEOF() {
	c.gotEOF = true
	c.cond.Broadcast()
}

func (c *Conn) onDestroyed() {
	c.destroyed = true
	c.s = nil
	c.cond.Broadcast()
}

func (c *Conn) WriteBufferLen() int {
	mu.Lock()
	defer mu.Unlock()
	return int(C.utp_getsockopt(c.s, C.UTP_SNDBUF))
}

func (c *Conn) SetWriteBufferLen(len int) {
	mu.Lock()
	defer mu.Unlock()
	i := C.utp_setsockopt(c.s, C.UTP_SNDBUF, C.int(len))
	if i != 0 {
		panic(i)
	}
}
<|endoftext|>


<commit_before>package rhynock

import (
	"github.com/gorilla/websocket"
	"net/http"
	"time"
	"log"
)

// Some defaults for pinging
// Needs to be settable from outside
const (
	writeWait = 10 * time.Second
	pongWait = 60 * time.Second
	pingPeriod = (pongWait * 9) / 10
	maxMessageSize = 512
)

// Conn encapsulates our websocket
type Conn struct {
	// Exported so everything can be messed with from outside
	Ws      *websocket.Conn
	Send    chan []byte
	Dst     BottleDst
	Quit    chan []byte
	wait    chan bool
}

//
// Convenience function so you dont have to use the Send channel
//
func (c *Conn) SendMsg(message string) {
	// Basically just typecasting for convenience
	c.Send <- []byte(message)
}

//
// Convenience function to call the quit channel with a message
//
func (c *Conn) CloseMsg(message string) {
	c.Quit <- []byte(message)
}

//
// This function chews through the power cables
//
func (c *Conn) Close() {
	// Send ourself the quit signal with no message
	c.Quit <- []byte("")
}

//
// Used to write a single message to the client and report any errors
//
func (c *Conn) write(t int, payload []byte) error {
	c.Ws.SetWriteDeadline(time.Now().Add(writeWait))
	return c.Ws.WriteMessage(t, payload)
}

//
// Maintains both a reader and a writer, cleans up both if one fails
//
func (c *Conn) read_write() {
	// Ping timer
	ticker := time.NewTicker(pingPeriod)

	// Clean up Connection and Connection resources
	defer func() {
		ticker.Stop()
		close(c.Send)
		close(c.Quit)
		c.Ws.Close()
		close(c.wait) // Unblocks the ConnectionHandler
	}()

	// Config websocket settings
	c.Ws.SetReadLimit(maxMessageSize)
	c.Ws.SetReadDeadline(time.Now().Add(pongWait))
	c.Ws.SetPongHandler(func(string) error {
		// Give each client pongWait seconds after the ping to respond
		c.Ws.SetReadDeadline(time.Now().Add(pongWait))
		return nil
	})

	// Start a reading goroutine
	// The reader will stop when the c.Ws.Close is called at
	// in the defered cleanup function, so we do not manually
	// have to close the reader
	go func() {
		for {
			// This blocks until it reads EOF or an error
			// occurs trying to read, the error can be
			// used to detect when the client closes the Connection
			_, message, err := c.Ws.ReadMessage()
			if err != nil {
				break // If we get an error escape the loop
			}

			// Bottle the message with its sender
			bottle := &Bottle{
				Sender: c,
				Message: message,
			}

			// Send to the destination for processing
			c.Dst.GetBottleChan() <- bottle
		}
		// The reader has been terminated, alert the writer
		c.Close()
	}()

	// Main writing loop
	for {
		select {
		case message, ok := <- c.Send:
			// Our send channel has something in it or the channel closed
			if !ok {
				// Our channel was closed, gracefully close socket Conn
				c.write(websocket.CloseMessage, []byte{})
				return
			}
			// Attempt to write the message to the websocket
			if err := c.write(websocket.TextMessage, message); err != nil {
				// If we get an error we can no longer communcate with client
				// return, no need to send CloseMessage since that would
				// just yield another error
				return
			}

		case <- ticker.C:
			// Ping ticker went off. We need to ping to check for connectivity.
			if err := c.write(websocket.PingMessage, []byte{}); err != nil {
				// We got an error pinging, return and call defer
				// defer will close the socket which will kill the reader
				return
			}

		case bytes := <- c.Quit:
			// Close connection and send a final message
			c.write(websocket.TextMessage, bytes)
			c.write(websocket.CloseMessage, []byte{})
			return
		}
	}
}

var upgrader = &websocket.Upgrader{ReadBufferSize: 1024, WriteBufferSize: 1024, CheckOrigin: func(r* http.Request) bool { return true }}

//
// Hanlder function to start a websocket connection
//
func ConnectionHandler(w http.ResponseWriter, r *http.Request, dst BottleDst) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println(err)
		return
	}

	// Create new connection object
	c := &Conn{
		Send: make(chan []byte, 256),
		Ws: ws,
		Dst: dst,
		Quit: make(chan []byte),
		wait: make(chan bool),
	}

	// Alert the destination that a new connection has opened
	dst.ConnectionOpened(c)

	// Make sure connections that are opened will get closed
	defer func() {
		dst.ConnectionClosed(c)
	}()

	// Start read write loop blocking
	c.read_write()
}
<commit_msg>Include validation checks<commit_after>package rhynock

import (
	"github.com/gorilla/websocket"
	"net/http"
	"time"
	"log"
)

// Some defaults for pinging
// Needs to be settable from outside
const (
	writeWait = 10 * time.Second
	pongWait = 60 * time.Second
	pingPeriod = (pongWait * 9) / 10
	maxMessageSize = 512
)

// Conn encapsulates our websocket
type Conn struct {
	// Exported so everything can be messed with from outside
	Ws      *websocket.Conn
	Send    chan []byte
	Dst     BottleDst
	Quit    chan []byte
	wait    chan bool
	valid   bool
}

//
// Convenience function so you dont have to use the Send channel
//
func (c *Conn) SendMsg(message string) {
	// Basically just typecasting for convenience
	if c.valid {
		c.Send <- []byte(message)
	} else {
		log.Println("rhynock.Conn.SendMsg: Socket closed.")
	}
}

//
// Convenience function to call the quit channel with a message
//
func (c *Conn) CloseMsg(message string) {
	if c.valid {
		c.Quit <- []byte(message)
	} else {
		log.Println("rhynock.Conn.CloseMsg: Socket already closed.")
	}
}

//
// This function chews through the power cables
//
func (c *Conn) Close() {
	// Send ourself the quit signal with no message
	if c.valid {
		c.Quit <- []byte("")
	} else {
		log.Println("rhynock.Conn.Close: Socket already closed.")
	}
}

//
// Used to write a single message to the client and report any errors
//
func (c *Conn) write(t int, payload []byte) error {
	c.Ws.SetWriteDeadline(time.Now().Add(writeWait))
	return c.Ws.WriteMessage(t, payload)
}

//
// Maintains both a reader and a writer, cleans up both if one fails
//
func (c *Conn) read_write() {
	// Ping timer
	ticker := time.NewTicker(pingPeriod)

	// Clean up Connection and Connection resources
	defer func() {
		ticker.Stop()
		c.valid = false
		close(c.Send)
		close(c.Quit)
		c.Ws.Close()
		close(c.wait) // Unblocks the ConnectionHandler
	}()

	// Config websocket settings
	c.Ws.SetReadLimit(maxMessageSize)
	c.Ws.SetReadDeadline(time.Now().Add(pongWait))
	c.Ws.SetPongHandler(func(string) error {
		// Give each client pongWait seconds after the ping to respond
		c.Ws.SetReadDeadline(time.Now().Add(pongWait))
		return nil
	})

	// Start a reading goroutine
	// The reader will stop when the c.Ws.Close is called at
	// in the defered cleanup function, so we do not manually
	// have to close the reader
	go func() {
		for {
			// This blocks until it reads EOF or an error
			// occurs trying to read, the error can be
			// used to detect when the client closes the Connection
			_, message, err := c.Ws.ReadMessage()
			if err != nil {
				break // If we get an error escape the loop
			}

			// Bottle the message with its sender
			bottle := &Bottle{
				Sender: c,
				Message: message,
			}

			// Send to the destination for processing
			c.Dst.GetBottleChan() <- bottle
		}
		// The reader has been terminated, alert the writer if needed
		if c.valid {
			c.Close()
		}
	}()

	// Main writing loop
	for {
		select {
		case message, ok := <- c.Send:
			// Our send channel has something in it or the channel closed
			if !ok {
				// Our channel was closed, gracefully close socket Conn
				c.write(websocket.CloseMessage, []byte{})
				return
			}
			// Attempt to write the message to the websocket
			if err := c.write(websocket.TextMessage, message); err != nil {
				// If we get an error we can no longer communcate with client
				// return, no need to send CloseMessage since that would
				// just yield another error
				return
			}

		case <- ticker.C:
			// Ping ticker went off. We need to ping to check for connectivity.
			if err := c.write(websocket.PingMessage, []byte{}); err != nil {
				// We got an error pinging, return and call defer
				// defer will close the socket which will kill the reader
				return
			}

		case bytes := <- c.Quit:
			// Close connection and send a final message
			c.write(websocket.TextMessage, bytes)
			c.write(websocket.CloseMessage, []byte{})
			return
		}
	}
}

var upgrader = &websocket.Upgrader{ReadBufferSize: 1024, WriteBufferSize: 1024, CheckOrigin: func(r* http.Request) bool { return true }}

//
// Hanlder function to start a websocket connection
//
func ConnectionHandler(w http.ResponseWriter, r *http.Request, dst BottleDst) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println(err)
		return
	}

	// Create new connection object
	c := &Conn{
		Send: make(chan []byte, 256),
		Ws: ws,
		Dst: dst,
		Quit: make(chan []byte),
		valid: true,
		wait: make(chan bool),
	}

	// Alert the destination that a new connection has opened
	dst.ConnectionOpened(c)

	// Make sure connections that are opened will get closed
	defer func() {
		dst.ConnectionClosed(c)
	}()

	// Start read write loop blocking
	c.read_write()
}
<|endoftext|>


<commit_before>package goboots

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/gabstv/i18ngo"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"text/template"
	"time"
)

const (
	ErrNil = 0
	// TLS
	ErrTLSNil = 100
)

var (
	sessionDbs   map[string]ISessionDBEngine
	curSessionDb ISessionDBEngine
	controllers  []IController
)

func RegisterController(controller IController) {
	//TODO: error if controller with same name exists!
	if controllers == nil {
		controllers = make([]IController, 0)
	}
	controllers = append(controllers, controller)
}

func RegisterSessionStorageDriver(name string, engine ISessionDBEngine) {
	if sessionDbs == nil {
		sessionDbs = make(map[string]ISessionDBEngine, 0)
	}
	sessionDbs[name] = engine
}

func (app *App) InitSessionStorage(driver string) error {
	if sessionDbs == nil {
		return errors.New("No session storage registered.")
	}
	if _, ok := sessionDbs[driver]; !ok {
		return errors.New("The session storage driver " + driver + " does not exist.")
	}
	curSessionDb = sessionDbs[driver]
	curSessionDb.SetApp(app)
	return nil
}

func CloseSessionStorage() {
	if curSessionDb != nil {
		curSessionDb.Close()
	}
}

func initAnySessionStorage() {
	if curSessionDb != nil {
		return
	}
	if sessionDbs == nil {
		//TODO: should panic
		return
	}
	for _, v := range sessionDbs {
		curSessionDb = v
		break
	}
}

type AppError struct {
	Id      int
	Message string
}

func (e *AppError) Error() string {
	return e.Message
}

type env struct {
	path       string
	controller string
	action     string
	params     []string
	request    *http.Request
	body       string
	form       map[string][]string
}

type ISession interface {
	GetData() (string, map[string]interface{}, time.Time)
}

type Session struct {
	SID     string
	Data    map[string]interface{}
	Flash   SessFlash `json:"-" bson:"-"` // never save flash
	Time    time.Time
	Updated time.Time
}

type SessFlash struct {
	vals map[string]interface{}
}

func (s *SessFlash) init() {
	if s.vals == nil {
		s.vals = make(map[string]interface{})
	}
}

func (s *SessFlash) Set(key string, val interface{}) {
	s.init()
	s.vals[key] = val
}

func (s *SessFlash) Get(key string) interface{} {
	s.init()
	return s.vals[key]
}

func (s *SessFlash) Get2(key string) (interface{}, bool) {
	s.init()
	v, ok := s.vals[key]
	return v, ok
}

func (s *SessFlash) Del(key string) {
	s.init()
	delete(s.vals, key)
}

func (s *SessFlash) Clear() {
	s.vals = nil
}

func (s *SessFlash) All() map[string]interface{} {
	s.init()
	return s.vals
}

func (s *Session) GetData() (string, map[string]interface{}, time.Time) {
	return s.SID, s.Data, s.Time
}

func (s *Session) Flush() {
	FlushSession(s)
}

func (s *Session) GetBool(key string) (bool, bool) {
	iface, ok := s.Data[key]
	if !ok {
		return false, false
	}
	return bool(reflect.ValueOf(iface).Bool()), true
}

func (s *Session) GetBoolD(key string, defaultValue bool) bool {
	iface, ok := s.Data[key]
	if !ok {
		return defaultValue
	}
	return bool(reflect.ValueOf(iface).Bool())
}

func (s *Session) GetInt32(key string) (int, bool) {
	iface, ok := s.Data[key]
	if !ok {
		return 0, false
	}
	return int(reflect.ValueOf(iface).Int()), true
}

func (s *Session) GetInt32D(key string, defaultValue int) int {
	iface, ok := s.Data[key]
	if !ok {
		return defaultValue
	}
	return int(reflect.ValueOf(iface).Int())
}

func (s *Session) GetString(key string) (string, bool) {
	iface, ok := s.Data[key]
	if !ok {
		return "", false
	}
	return reflect.ValueOf(iface).String(), true
}

func (s *Session) GetStringD(key string, defaultValue string) string {
	iface, ok := s.Data[key]
	if !ok {
		return defaultValue
	}
	return reflect.ValueOf(iface).String()
}

func (s *Session) Expire(w http.ResponseWriter, t time.Time) {
	SetCookieAdv(w, "goboots_sessid", s.SID, "/", "", t, 0, false, true)
}

func GetSession(w http.ResponseWriter, r *http.Request) *Session {
	initAnySessionStorage()
	var cookie *http.Cookie
	var sid string
	var err error
	//LOAD
	cookie, err = r.Cookie("goboots_sessid")
	if err == nil {
		sid = cookie.Value
		if _validateSidString(sid) {
			var msession *Session
			// get from a saved location
			msession, err = curSessionDb.GetSession(sid)
			if err == nil {
				return msession
			}
			fmt.Printf("SESSION ERROR :( [%s] %s\n", sid, err.Error())
			// not found, generate a new sid
		} else {
			fmt.Printf("COULD NOT VALIDATE :( [%s]\n", sid)
		}
	}
	//TODO: improve session generation because it created a duplicate key!
	// gen session
	rand.Seed(time.Now().UnixNano())
	var uuid [16]byte
	for i := 0; i < 16; i++ {
		uuid[i] = byte(rand.Intn(255))
	}
	// secrets
	sid = fmt.Sprintf("%x", uuid)

	session := &Session{
		SID:     sid,
		Data:    make(map[string]interface{}),
		Time:    time.Now(),
		Updated: time.Now(),
	}
	err = curSessionDb.NewSession(session)
	__panic(err)
	SetCookieAdv(w, "goboots_sessid", sid, "/", "", time.Now().AddDate(0, 1, 0), 0, false, true)
	return session
}

func GetUserLang(w http.ResponseWriter, r *http.Request) string {
	var l string
	// get lang from cookies
	cookie, err := r.Cookie("lang")
	if err != nil && cookie != nil {
		l = cookie.Value
		return l
	}
	// get lang from http request
	alh := r.Header.Get("Accept-Language")
	// 2013-07-29 : not all clients actually send this header (googlebot/wget etc)
	if len(alh) > 1 {
		validlangs := i18ngo.GetLanguageCodes()
		// break alh
		alh0 := strings.Split(alh, ",")
		for _, v := range alh0 {
			lc := strings.Split(v, ";")[0][0:2]
			if StringIndexOf(validlangs, lc) != -1 {
				l = lc
				break
			}
		}
		if len(l) > 0 {
			// just update the cookies
			if w != nil {
				SetCookieSimple(w, "lang", l)
			}
		} else {
			l = i18ngo.GetDefaultLanguageCode()
		}
		return l
	}
	return i18ngo.GetDefaultLanguageCode()
}

func SetUserLang(w http.ResponseWriter, r *http.Request, langcode string) {
	SetCookieSimple(w, "lang", langcode)
}

func FlushSession(s *Session) error {
	return curSessionDb.PutSession(s)
}

func DestroySession(w http.ResponseWriter, r *http.Request, s *Session) {
	curSessionDb.RemoveSession(s)
	SetCookieAdv(w, "goboots_sessid", "", "/", "", time.Now(), 1, false, true)
}

func _validateSidString(sid string) bool {
	if len(sid) != 32 {
		return false
	}
	for _, c := range sid {
		// check if sid contains invalid chars
		if !((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
			return false
		}
	}
	return true
}

type templateInfo struct {
	path       string
	data       *template.Template
	lastUpdate time.Time
}

func SetCookieAdv(w http.ResponseWriter, name string, value string, path string, domain string, expires time.Time, maxage int, secure bool, httpOnly bool) {
	cookie := &http.Cookie{
		Name:     name,
		Value:    value,
		Path:     path,
		Domain:   domain,
		Expires:  expires,
		MaxAge:   maxage,
		Secure:   secure,
		HttpOnly: httpOnly,
	}
	http.SetCookie(w, cookie)
}

func SetCookieSimple(w http.ResponseWriter, key string, value string) {
	SetCookieAdv(w, key, value, "/", "", time.Now().AddDate(100, 0, 0), 0, false, false)
}

func StrConcat(strings ...string) string {
	var buffer bytes.Buffer
	for _, str := range strings {
		buffer.WriteString(str)
	}
	return buffer.String()
}

func DeleteItem(slice interface{}, i int) {
	v := reflect.ValueOf(slice).Elem()
	v.Set(reflect.AppendSlice(v.Slice(0, i), v.Slice(i+1, v.Len())))
}

func InsertItem(slice interface{}, i int, val interface{}) {
	v := reflect.ValueOf(slice).Elem()
	v.Set(reflect.AppendSlice(v.Slice(0, i+1), v.Slice(i, v.Len())))
	v.Index(i).Set(reflect.ValueOf(val))
}

func StringIndexOf(haystack []string, needle string) int {
	for i, v := range haystack {
		if v == needle {
			return i
		}
	}
	return -1
}

func IntMax(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func __panic(err error) {
	if err != nil {
		log.Panicln(err)
	}
}

func FormatPath(rawpath string) string {
	cwd, _ := os.Getwd()
	if strings.HasPrefix(rawpath, "./") {
		return filepath.Clean(cwd + rawpath[1:])
	} else if strings.HasPrefix(rawpath, "../") {
		return filepath.Clean(cwd + "/" + rawpath)
	}
	return filepath.Clean(rawpath)
}
<commit_msg>fix lang bug<commit_after>package goboots

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/gabstv/i18ngo"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"text/template"
	"time"
)

const (
	ErrNil = 0
	// TLS
	ErrTLSNil = 100
)

var (
	sessionDbs   map[string]ISessionDBEngine
	curSessionDb ISessionDBEngine
	controllers  []IController
)

func RegisterController(controller IController) {
	//TODO: error if controller with same name exists!
	if controllers == nil {
		controllers = make([]IController, 0)
	}
	controllers = append(controllers, controller)
}

func RegisterSessionStorageDriver(name string, engine ISessionDBEngine) {
	if sessionDbs == nil {
		sessionDbs = make(map[string]ISessionDBEngine, 0)
	}
	sessionDbs[name] = engine
}

func (app *App) InitSessionStorage(driver string) error {
	if sessionDbs == nil {
		return errors.New("No session storage registered.")
	}
	if _, ok := sessionDbs[driver]; !ok {
		return errors.New("The session storage driver " + driver + " does not exist.")
	}
	curSessionDb = sessionDbs[driver]
	curSessionDb.SetApp(app)
	return nil
}

func CloseSessionStorage() {
	if curSessionDb != nil {
		curSessionDb.Close()
	}
}

func initAnySessionStorage() {
	if curSessionDb != nil {
		return
	}
	if sessionDbs == nil {
		//TODO: should panic
		return
	}
	for _, v := range sessionDbs {
		curSessionDb = v
		break
	}
}

type AppError struct {
	Id      int
	Message string
}

func (e *AppError) Error() string {
	return e.Message
}

type env struct {
	path       string
	controller string
	action     string
	params     []string
	request    *http.Request
	body       string
	form       map[string][]string
}

type ISession interface {
	GetData() (string, map[string]interface{}, time.Time)
}

type Session struct {
	SID     string
	Data    map[string]interface{}
	Flash   SessFlash `json:"-" bson:"-"` // never save flash
	Time    time.Time
	Updated time.Time
}

type SessFlash struct {
	vals map[string]interface{}
}

func (s *SessFlash) init() {
	if s.vals == nil {
		s.vals = make(map[string]interface{})
	}
}

func (s *SessFlash) Set(key string, val interface{}) {
	s.init()
	s.vals[key] = val
}

func (s *SessFlash) Get(key string) interface{} {
	s.init()
	return s.vals[key]
}

func (s *SessFlash) Get2(key string) (interface{}, bool) {
	s.init()
	v, ok := s.vals[key]
	return v, ok
}

func (s *SessFlash) Del(key string) {
	s.init()
	delete(s.vals, key)
}

func (s *SessFlash) Clear() {
	s.vals = nil
}

func (s *SessFlash) All() map[string]interface{} {
	s.init()
	return s.vals
}

func (s *Session) GetData() (string, map[string]interface{}, time.Time) {
	return s.SID, s.Data, s.Time
}

func (s *Session) Flush() {
	FlushSession(s)
}

func (s *Session) GetBool(key string) (bool, bool) {
	iface, ok := s.Data[key]
	if !ok {
		return false, false
	}
	return bool(reflect.ValueOf(iface).Bool()), true
}

func (s *Session) GetBoolD(key string, defaultValue bool) bool {
	iface, ok := s.Data[key]
	if !ok {
		return defaultValue
	}
	return bool(reflect.ValueOf(iface).Bool())
}

func (s *Session) GetInt32(key string) (int, bool) {
	iface, ok := s.Data[key]
	if !ok {
		return 0, false
	}
	return int(reflect.ValueOf(iface).Int()), true
}

func (s *Session) GetInt32D(key string, defaultValue int) int {
	iface, ok := s.Data[key]
	if !ok {
		return defaultValue
	}
	return int(reflect.ValueOf(iface).Int())
}

func (s *Session) GetString(key string) (string, bool) {
	iface, ok := s.Data[key]
	if !ok {
		return "", false
	}
	return reflect.ValueOf(iface).String(), true
}

func (s *Session) GetStringD(key string, defaultValue string) string {
	iface, ok := s.Data[key]
	if !ok {
		return defaultValue
	}
	return reflect.ValueOf(iface).String()
}

func (s *Session) Expire(w http.ResponseWriter, t time.Time) {
	SetCookieAdv(w, "goboots_sessid", s.SID, "/", "", t, 0, false, true)
}

func GetSession(w http.ResponseWriter, r *http.Request) *Session {
	initAnySessionStorage()
	var cookie *http.Cookie
	var sid string
	var err error
	//LOAD
	cookie, err = r.Cookie("goboots_sessid")
	if err == nil {
		sid = cookie.Value
		if _validateSidString(sid) {
			var msession *Session
			// get from a saved location
			msession, err = curSessionDb.GetSession(sid)
			if err == nil {
				return msession
			}
			fmt.Printf("SESSION ERROR :( [%s] %s\n", sid, err.Error())
			// not found, generate a new sid
		} else {
			fmt.Printf("COULD NOT VALIDATE :( [%s]\n", sid)
		}
	}
	//TODO: improve session generation because it created a duplicate key!
	// gen session
	rand.Seed(time.Now().UnixNano())
	var uuid [16]byte
	for i := 0; i < 16; i++ {
		uuid[i] = byte(rand.Intn(255))
	}
	// secrets
	sid = fmt.Sprintf("%x", uuid)

	session := &Session{
		SID:     sid,
		Data:    make(map[string]interface{}),
		Time:    time.Now(),
		Updated: time.Now(),
	}
	err = curSessionDb.NewSession(session)
	__panic(err)
	SetCookieAdv(w, "goboots_sessid", sid, "/", "", time.Now().AddDate(0, 1, 0), 0, false, true)
	return session
}

func GetUserLang(w http.ResponseWriter, r *http.Request) string {
	var l string
	// get lang from cookies
	cookie, err := r.Cookie("lang")
	if err != nil && cookie != nil {
		l = cookie.Value
		return l
	}
	// get lang from http request
	alh := r.Header.Get("Accept-Language")
	// 2013-07-29 : not all clients actually send this header (googlebot/wget etc)
	if len(alh) > 1 {
		validlangs := i18ngo.GetLanguageCodes()
		// break alh
		alh0 := strings.Split(alh, ",")
		for _, v := range alh0 {
			lcd := strings.Split(v, ";")[0]
			var lc string
			if len(lcd) >= 2 {
				lc = lcd[0:2]
			}
			lc = lcd
			if StringIndexOf(validlangs, lc) != -1 {
				l = lc
				break
			}
		}
		if len(l) > 0 {
			// just update the cookies
			if w != nil {
				SetCookieSimple(w, "lang", l)
			}
		} else {
			l = i18ngo.GetDefaultLanguageCode()
		}
		return l
	}
	return i18ngo.GetDefaultLanguageCode()
}

func SetUserLang(w http.ResponseWriter, r *http.Request, langcode string) {
	SetCookieSimple(w, "lang", langcode)
}

func FlushSession(s *Session) error {
	return curSessionDb.PutSession(s)
}

func DestroySession(w http.ResponseWriter, r *http.Request, s *Session) {
	curSessionDb.RemoveSession(s)
	SetCookieAdv(w, "goboots_sessid", "", "/", "", time.Now(), 1, false, true)
}

func _validateSidString(sid string) bool {
	if len(sid) != 32 {
		return false
	}
	for _, c := range sid {
		// check if sid contains invalid chars
		if !((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
			return false
		}
	}
	return true
}

type templateInfo struct {
	path       string
	data       *template.Template
	lastUpdate time.Time
}

func SetCookieAdv(w http.ResponseWriter, name string, value string, path string, domain string, expires time.Time, maxage int, secure bool, httpOnly bool) {
	cookie := &http.Cookie{
		Name:     name,
		Value:    value,
		Path:     path,
		Domain:   domain,
		Expires:  expires,
		MaxAge:   maxage,
		Secure:   secure,
		HttpOnly: httpOnly,
	}
	http.SetCookie(w, cookie)
}

func SetCookieSimple(w http.ResponseWriter, key string, value string) {
	SetCookieAdv(w, key, value, "/", "", time.Now().AddDate(100, 0, 0), 0, false, false)
}

func StrConcat(strings ...string) string {
	var buffer bytes.Buffer
	for _, str := range strings {
		buffer.WriteString(str)
	}
	return buffer.String()
}

func DeleteItem(slice interface{}, i int) {
	v := reflect.ValueOf(slice).Elem()
	v.Set(reflect.AppendSlice(v.Slice(0, i), v.Slice(i+1, v.Len())))
}

func InsertItem(slice interface{}, i int, val interface{}) {
	v := reflect.ValueOf(slice).Elem()
	v.Set(reflect.AppendSlice(v.Slice(0, i+1), v.Slice(i, v.Len())))
	v.Index(i).Set(reflect.ValueOf(val))
}

func StringIndexOf(haystack []string, needle string) int {
	for i, v := range haystack {
		if v == needle {
			return i
		}
	}
	return -1
}

func IntMax(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func __panic(err error) {
	if err != nil {
		log.Panicln(err)
	}
}

func FormatPath(rawpath string) string {
	cwd, _ := os.Getwd()
	if strings.HasPrefix(rawpath, "./") {
		return filepath.Clean(cwd + rawpath[1:])
	} else if strings.HasPrefix(rawpath, "../") {
		return filepath.Clean(cwd + "/" + rawpath)
	}
	return filepath.Clean(rawpath)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/varnamproject/libvarnam-golang"
)

type syncDispatcher struct {
	quit   chan struct{}
	ticker *time.Ticker
}

func newSyncDispatcher(intervalInSeconds time.Duration) *syncDispatcher {
	return &syncDispatcher{ticker: time.NewTicker(intervalInSeconds), quit: make(chan struct{})}
}

func (s *syncDispatcher) start() {
	err := createSyncMetadataDir()
	if err != nil {
		fmt.Printf("Failed to create sync metadata directory. Sync will be disabled.\nActual error: %s\n", err.Error())
		return
	}
	for s := range varnamdConfig.SchemesToSync {
		// download cache directory for each of the languages
		err = createLearnQueueDir(s)
		if err != nil {
			fmt.Printf("Failed to create learn queue directory for '%s'. Sync will be disabled.\nActual error: %s\n", err.Error())
			return
		}
	}

	go func() {
		for {
			select {
			case <-s.ticker.C:
				log.Println("---SYNC BEGIN---")
				log.Printf("Config: %v\n", varnamdConfig)

				syncWordsFromUpstream()

				log.Println("---SYNC DONE---")
			case <-s.quit:
				s.ticker.Stop()
				return
			}
		}
	}()
}

func (s *syncDispatcher) stop() {
	close(s.quit)
}

func syncWordsFromUpstream() {
	for langCode := range varnamdConfig.SchemesToSync {
		log.Printf("Sync: %s\n", langCode)
		syncWordsFromUpstreamFor(langCode)
	}
}

func syncWordsFromUpstreamFor(langCode string) {
	corpusDetails, err := getCorpusDetails(langCode)
	if err != nil {
		log.Printf("Error getting corpus details for '%s'. %s\n", langCode, err.Error())
		return
	}

	localFilesToLearn := make(chan string, 100)
	downloadedFilesToLearn := make(chan string, 100)
	done := make(chan bool)

	// adding files which are remaining to learn in the local learn queue
	go addFilesFromLocalLearnQueue(langCode, localFilesToLearn)
	go downloadAllWords(langCode, corpusDetails.WordsCount, downloadedFilesToLearn)
	go func() {
		learnAll(langCode, localFilesToLearn)
		learnAll(langCode, downloadedFilesToLearn)
		done <- true
	}()

	<-done
}

func addFilesFromLocalLearnQueue(langCode string, filesToLearn chan string) {
	files := getFilesFromLearnQueue(langCode)
	if files != nil {
		log.Printf("Adding %d files to learn from local learn queue\n", len(files))
		for _, f := range files {
			filesToLearn <- f
		}
	} else {
		log.Printf("Local learn queue for '%s' is empty", langCode)
	}
	close(filesToLearn)
}

func downloadAllWords(langCode string, corpusSize int, output chan string) {
	for {
		offset := getDownloadOffset(langCode)
		log.Printf("Offset: %d\n", offset)
		if offset >= corpusSize {
			break
		}
		filePath, err := downloadWordsAndUpdateOffset(langCode, offset)
		if err != nil {
			break
		}
		output <- filePath
	}
	log.Println("Local copy is upto date. No need to download from upstream")
	close(output)
}

func learnAll(langCode string, filesToLearn chan string) {
	for fileToLearn := range filesToLearn {
		learnFromFile(langCode, fileToLearn)
	}
}

func learnFromFile(langCode, fileToLearn string) {
	log.Printf("Learning from %s\n", fileToLearn)
	start := time.Now()
	getOrCreateHandler(langCode, func(handle *libvarnam.Varnam) (data interface{}, err error) {
		learnStatus, err := handle.LearnFromFile(fileToLearn)
		end := time.Now()
		if err != nil {
			log.Printf("Error learning from '%s'\n", err.Error())
		} else {
			log.Printf("Learned from '%s'. TotalWords: %d, Failed: %d. Took %s\n", fileToLearn, learnStatus.TotalWords, learnStatus.Failed, end.Sub(start))
		}
		return
	})

	err := os.Remove(fileToLearn)
	if err != nil {
		log.Printf("Error deleting '%s'. %s\n", fileToLearn, err.Error())
	}

}

func downloadWordsAndUpdateOffset(langCode string, offset int) (string, error) {
	count, filePath, err := downloadWords(langCode, offset)
	if err != nil {
		log.Printf("Error downloading words for '%s'. %s\n", langCode, err.Error())
		return "", err
	}

	err = setDownloadOffset(langCode, offset+count)
	if err != nil {
		log.Printf("Error setting download offset for '%s'. %s\n", langCode, err.Error())
		return "", err
	}

	return filePath, nil
}

func getCorpusDetails(langCode string) (*libvarnam.CorpusDetails, error) {
	url := fmt.Sprintf("%s/meta/%s", varnamdConfig.Upstream, langCode)
	log.Printf("Fetching corpus details for '%s'\n", langCode)
	var corpusDetails libvarnam.CorpusDetails
	err := getJSONResponse(url, &corpusDetails)
	if err != nil {
		return nil, err
	}
	log.Printf("Corpus size: %d\n", corpusDetails.WordsCount)
	return &corpusDetails, nil
}

// Downloads words from upstream starting from the specified offset and stores it locally in the learn queue
// Returns the number of words downloaded, local file path and error if any
func downloadWords(langCode string, offset int) (totalWordsDownloaded int, downloadedFilePath string, err error) {
	url := fmt.Sprintf("%s/download/%s/%d", varnamdConfig.Upstream, langCode, offset)
	var response downloadResponse
	err = getJSONResponse(url, &response)
	if err != nil {
		return 0, "", err
	}
	downloadedFilePath, err = transformAndPersistWords(langCode, &response)
	if err != nil {
		log.Printf("Download was successful, but failed to persist to local learn queue. %s\n", err.Error())
		return 0, "", err
	}

	return response.Count, downloadedFilePath, nil
}

func transformAndPersistWords(langCode string, dresp *downloadResponse) (string, error) {
	learnQueueDir := getLearnQueueDir(langCode)
	targetFile, err := ioutil.TempFile(learnQueueDir, langCode)
	if err != nil {
		return "", err
	}
	defer targetFile.Close()

	for _, word := range dresp.Words {
		_, err = targetFile.WriteString(fmt.Sprintf("%s %d\n", word.Word, word.Confidence))
		if err != nil {
			return "", err
		}
	}
	return targetFile.Name(), nil
}

func getFilesFromLearnQueue(langCode string) []string {
	var files []string
	learnQueueDir := getLearnQueueDir(langCode)
	queueContents, err := ioutil.ReadDir(learnQueueDir)
	if err != nil {
		return nil
	}

	for _, c := range queueContents {
		if !c.IsDir() {
			files = append(files, path.Join(learnQueueDir, c.Name()))
		}
	}

	return files
}

func getJSONResponse(url string, output interface{}) error {
	log.Printf("GET: '%s'\n", url)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	jsonDecoder := json.NewDecoder(resp.Body)
	err = jsonDecoder.Decode(output)
	if err != nil {
		return err
	}
	return nil
}

func getDownloadOffset(langCode string) int {
	filePath := getDownloadOffsetMetadataFile(langCode)
	content, err := ioutil.ReadFile(filePath)
	if err != nil {
		return 0
	}

	offset, err := strconv.Atoi(strings.TrimSpace(string(content)))
	if err != nil {
		return 0
	}

	return offset
}

func setDownloadOffset(langCode string, offset int) error {
	filePath := getDownloadOffsetMetadataFile(langCode)
	return ioutil.WriteFile(filePath, []byte(fmt.Sprintf("%d", offset)), 0666)
}

func getDownloadOffsetMetadataFile(langCode string) string {
	syncDir := getSyncMetadataDir()
	return path.Join(syncDir, fmt.Sprintf("%s.download.offset", langCode))
}

func createLearnQueueDir(langCode string) error {
	queueDir := getLearnQueueDir(langCode)
	err := os.MkdirAll(queueDir, 0777)
	if err != nil {
		return err
	}
	return nil
}

func getLearnQueueDir(langCode string) string {
	syncDir := getSyncMetadataDir()
	queueDir := path.Join(syncDir, fmt.Sprintf("%s.learn.queue", langCode))
	return queueDir
}

func createSyncMetadataDir() error {
	syncDir := getSyncMetadataDir()
	err := os.MkdirAll(syncDir, 0777)
	if err != nil {
		return err
	}
	return nil
}

func getSyncMetadataDir() string {
	configDir := getConfigDir()
	syncDir := path.Join(configDir, "sync")
	return syncDir
}
<commit_msg>Downloaded words uses langcode.offset as name<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/varnamproject/libvarnam-golang"
)

type syncDispatcher struct {
	quit   chan struct{}
	ticker *time.Ticker
}

func newSyncDispatcher(intervalInSeconds time.Duration) *syncDispatcher {
	return &syncDispatcher{ticker: time.NewTicker(intervalInSeconds), quit: make(chan struct{})}
}

func (s *syncDispatcher) start() {
	err := createSyncMetadataDir()
	if err != nil {
		fmt.Printf("Failed to create sync metadata directory. Sync will be disabled.\nActual error: %s\n", err.Error())
		return
	}
	for s := range varnamdConfig.SchemesToSync {
		// download cache directory for each of the languages
		err = createLearnQueueDir(s)
		if err != nil {
			fmt.Printf("Failed to create learn queue directory for '%s'. Sync will be disabled.\nActual error: %s\n", err.Error())
			return
		}
	}

	go func() {
		for {
			select {
			case <-s.ticker.C:
				log.Println("---SYNC BEGIN---")
				log.Printf("Config: %v\n", varnamdConfig)

				syncWordsFromUpstream()

				log.Println("---SYNC DONE---")
			case <-s.quit:
				s.ticker.Stop()
				return
			}
		}
	}()
}

func (s *syncDispatcher) stop() {
	close(s.quit)
}

func syncWordsFromUpstream() {
	for langCode := range varnamdConfig.SchemesToSync {
		log.Printf("Sync: %s\n", langCode)
		syncWordsFromUpstreamFor(langCode)
	}
}

func syncWordsFromUpstreamFor(langCode string) {
	corpusDetails, err := getCorpusDetails(langCode)
	if err != nil {
		log.Printf("Error getting corpus details for '%s'. %s\n", langCode, err.Error())
		return
	}

	localFilesToLearn := make(chan string, 100)
	downloadedFilesToLearn := make(chan string, 100)
	done := make(chan bool)

	// adding files which are remaining to learn in the local learn queue
	go addFilesFromLocalLearnQueue(langCode, localFilesToLearn)
	go downloadAllWords(langCode, corpusDetails.WordsCount, downloadedFilesToLearn)
	go func() {
		learnAll(langCode, localFilesToLearn)
		learnAll(langCode, downloadedFilesToLearn)
		done <- true
	}()

	<-done
}

func addFilesFromLocalLearnQueue(langCode string, filesToLearn chan string) {
	files := getFilesFromLearnQueue(langCode)
	if files != nil {
		log.Printf("Adding %d files to learn from local learn queue\n", len(files))
		for _, f := range files {
			filesToLearn <- f
		}
	} else {
		log.Printf("Local learn queue for '%s' is empty", langCode)
	}
	close(filesToLearn)
}

func downloadAllWords(langCode string, corpusSize int, output chan string) {
	for {
		offset := getDownloadOffset(langCode)
		log.Printf("Offset: %d\n", offset)
		if offset >= corpusSize {
			break
		}
		filePath, err := downloadWordsAndUpdateOffset(langCode, offset)
		if err != nil {
			break
		}
		output <- filePath
	}
	log.Println("Local copy is upto date. No need to download from upstream")
	close(output)
}

func learnAll(langCode string, filesToLearn chan string) {
	for fileToLearn := range filesToLearn {
		learnFromFile(langCode, fileToLearn)
	}
}

func learnFromFile(langCode, fileToLearn string) {
	log.Printf("Learning from %s\n", fileToLearn)
	start := time.Now()
	getOrCreateHandler(langCode, func(handle *libvarnam.Varnam) (data interface{}, err error) {
		learnStatus, err := handle.LearnFromFile(fileToLearn)
		end := time.Now()
		if err != nil {
			log.Printf("Error learning from '%s'\n", err.Error())
		} else {
			log.Printf("Learned from '%s'. TotalWords: %d, Failed: %d. Took %s\n", fileToLearn, learnStatus.TotalWords, learnStatus.Failed, end.Sub(start))
		}
		return
	})

	err := os.Remove(fileToLearn)
	if err != nil {
		log.Printf("Error deleting '%s'. %s\n", fileToLearn, err.Error())
	}

}

func downloadWordsAndUpdateOffset(langCode string, offset int) (string, error) {
	count, filePath, err := downloadWords(langCode, offset)
	if err != nil {
		log.Printf("Error downloading words for '%s'. %s\n", langCode, err.Error())
		return "", err
	}

	err = setDownloadOffset(langCode, offset+count)
	if err != nil {
		log.Printf("Error setting download offset for '%s'. %s\n", langCode, err.Error())
		return "", err
	}

	return filePath, nil
}

func getCorpusDetails(langCode string) (*libvarnam.CorpusDetails, error) {
	url := fmt.Sprintf("%s/meta/%s", varnamdConfig.Upstream, langCode)
	log.Printf("Fetching corpus details for '%s'\n", langCode)
	var corpusDetails libvarnam.CorpusDetails
	err := getJSONResponse(url, &corpusDetails)
	if err != nil {
		return nil, err
	}
	log.Printf("Corpus size: %d\n", corpusDetails.WordsCount)
	return &corpusDetails, nil
}

// Downloads words from upstream starting from the specified offset and stores it locally in the learn queue
// Returns the number of words downloaded, local file path and error if any
func downloadWords(langCode string, offset int) (totalWordsDownloaded int, downloadedFilePath string, err error) {
	url := fmt.Sprintf("%s/download/%s/%d", varnamdConfig.Upstream, langCode, offset)
	var response downloadResponse
	err = getJSONResponse(url, &response)
	if err != nil {
		return 0, "", err
	}
	downloadedFilePath, err = transformAndPersistWords(langCode, offset, &response)
	if err != nil {
		log.Printf("Download was successful, but failed to persist to local learn queue. %s\n", err.Error())
		return 0, "", err
	}

	return response.Count, downloadedFilePath, nil
}

func transformAndPersistWords(langCode string, offset int, dresp *downloadResponse) (string, error) {
	learnQueueDir := getLearnQueueDir(langCode)
	targetFile, err := os.Create(path.Join(learnQueueDir, fmt.Sprintf("%s.%d", langCode, offset)))
	if err != nil {
		return "", err
	}
	defer targetFile.Close()

	for _, word := range dresp.Words {
		_, err = targetFile.WriteString(fmt.Sprintf("%s %d\n", word.Word, word.Confidence))
		if err != nil {
			return "", err
		}
	}
	return targetFile.Name(), nil
}

func getFilesFromLearnQueue(langCode string) []string {
	var files []string
	learnQueueDir := getLearnQueueDir(langCode)
	queueContents, err := ioutil.ReadDir(learnQueueDir)
	if err != nil {
		return nil
	}

	for _, c := range queueContents {
		if !c.IsDir() {
			files = append(files, path.Join(learnQueueDir, c.Name()))
		}
	}

	return files
}

func getJSONResponse(url string, output interface{}) error {
	log.Printf("GET: '%s'\n", url)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	jsonDecoder := json.NewDecoder(resp.Body)
	err = jsonDecoder.Decode(output)
	if err != nil {
		return err
	}
	return nil
}

func getDownloadOffset(langCode string) int {
	filePath := getDownloadOffsetMetadataFile(langCode)
	content, err := ioutil.ReadFile(filePath)
	if err != nil {
		return 0
	}

	offset, err := strconv.Atoi(strings.TrimSpace(string(content)))
	if err != nil {
		return 0
	}

	return offset
}

func setDownloadOffset(langCode string, offset int) error {
	filePath := getDownloadOffsetMetadataFile(langCode)
	return ioutil.WriteFile(filePath, []byte(fmt.Sprintf("%d", offset)), 0666)
}

func getDownloadOffsetMetadataFile(langCode string) string {
	syncDir := getSyncMetadataDir()
	return path.Join(syncDir, fmt.Sprintf("%s.download.offset", langCode))
}

func createLearnQueueDir(langCode string) error {
	queueDir := getLearnQueueDir(langCode)
	err := os.MkdirAll(queueDir, 0777)
	if err != nil {
		return err
	}
	return nil
}

func getLearnQueueDir(langCode string) string {
	syncDir := getSyncMetadataDir()
	queueDir := path.Join(syncDir, fmt.Sprintf("%s.learn.queue", langCode))
	return queueDir
}

func createSyncMetadataDir() error {
	syncDir := getSyncMetadataDir()
	err := os.MkdirAll(syncDir, 0777)
	if err != nil {
		return err
	}
	return nil
}

func getSyncMetadataDir() string {
	configDir := getConfigDir()
	syncDir := path.Join(configDir, "sync")
	return syncDir
}
<|endoftext|>


<commit_before>package csql

import (
	"database/sql"
	"database/sql/driver"
)

// Execer describes values that can execute queries without returning any rows.
type Execer interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
}

// Queryer describes values that can run queries which return 1 or many rows.
type Queryer interface {
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
}

// Preparer describes values that can create prepared statements.
type Preparer interface {
	Prepare(query string) (*sql.Stmt, error)
}

// RowScanner describes values that can scan a row of values.
type RowScanner interface {
	Scan(dest ...interface{}) error
}

// Valuer describes values that can convert themselves to a driver value.
type Valuer interface {
	Value() (driver.Value, error)
}

// Beginner describes values that can begin a transaction.
type Beginner interface {
	Begin() (*sql.Tx, error)
}

// SQLError satisfies the error interface. All panic'd errors in this package
// are SQLErrors. Errors returned by functions in this package are never
// SQLerrors.
type SQLError struct {
	error
}

func (se SQLError) Error() string {
	return se.error.Error()
}

// Safe executes any function that may panic with a SQLError safely. In
// particular, if `f` panics with a SQLError, then Safe recovers and returns
// the error wrapped by SQLError.
//
// If `f` panics with any other type of error, the panic is not recovered.
func Safe(f func()) (err error) {
	defer func() {
		if r := recover(); r != nil {
			if e, ok := r.(SQLError); !ok {
				panic(r)
			} else {
				err = e.error
				return
			}
		}
	}()
	f()
	return nil
}

// SQLPanic will wrap the given error in SQLError and pass it to panic.
// If the error is nil, this function does nothing.
func SQLPanic(err error) {
	if err != nil {
		panic(SQLError{err})
	}
}

// Tx runs the given function safely within a transaction. If the function
// panics with a SQLError, then the transaction is rolled back. Otherwise,
// the transaction is committed.
//
// The first error that occurs (including beginning and ending the transaction)
// is returned.
func Tx(db Beginner, f func()) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	if err := Safe(f); err != nil {
		tx.Rollback() // ignore this error (return the first)
		return err
	}
	return tx.Commit()
}

// Exec returns the result of a running a query that doesn't return any rows.
// If an error occurs, it is panic'd as a SQLError.
func Exec(db Execer, query string, args ...interface{}) sql.Result {
	r, err := db.Exec(query, args...)
	SQLPanic(err)
	return r
}

// Query returns the result of a query that fetches many rows.
// If an error occurs, it is panic'd as a SQLError.
func Query(db Queryer, query string, args ...interface{}) *sql.Rows {
	rs, err := db.Query(query, args...)
	SQLPanic(err)
	return rs
}

// Prepare returns a prepared statement.
// If an error occurs, it is panic'd as a SQLError.
func Prepare(db Preparer, query string) *sql.Stmt {
	stmt, err := db.Prepare(query)
	SQLPanic(err)
	return stmt
}

// Scan performs a scan on a row. If an error occurs, it is panic'd as a
// SQLError.
func Scan(scanner RowScanner, dest ...interface{}) {
	SQLPanic(scanner.Scan(dest...))
}

// Value returns the driver value.
// If an error occurs, it is panic'd as a SQLError.
func Value(v Valuer) driver.Value {
	dval, err := v.Value()
	SQLPanic(err)
	return dval
}
<commit_msg>Provide interface for Queryer + Execer.<commit_after>package csql

import (
	"database/sql"
	"database/sql/driver"
)

// Execer describes values that can execute queries without returning any rows.
type Execer interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
}

// Queryer describes values that can run queries which return 1 or many rows.
type Queryer interface {
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
}

// QExecer is the composition of the Queryer and Execer interfaces.
type QExecer interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
}

// Preparer describes values that can create prepared statements.
type Preparer interface {
	Prepare(query string) (*sql.Stmt, error)
}

// RowScanner describes values that can scan a row of values.
type RowScanner interface {
	Scan(dest ...interface{}) error
}

// Valuer describes values that can convert themselves to a driver value.
type Valuer interface {
	Value() (driver.Value, error)
}

// Beginner describes values that can begin a transaction.
type Beginner interface {
	Begin() (*sql.Tx, error)
}

// SQLError satisfies the error interface. All panic'd errors in this package
// are SQLErrors. Errors returned by functions in this package are never
// SQLerrors.
type SQLError struct {
	error
}

func (se SQLError) Error() string {
	return se.error.Error()
}

// Safe executes any function that may panic with a SQLError safely. In
// particular, if `f` panics with a SQLError, then Safe recovers and returns
// the error wrapped by SQLError.
//
// If `f` panics with any other type of error, the panic is not recovered.
func Safe(f func()) (err error) {
	defer func() {
		if r := recover(); r != nil {
			if e, ok := r.(SQLError); !ok {
				panic(r)
			} else {
				err = e.error
				return
			}
		}
	}()
	f()
	return nil
}

// SQLPanic will wrap the given error in SQLError and pass it to panic.
// If the error is nil, this function does nothing.
func SQLPanic(err error) {
	if err != nil {
		panic(SQLError{err})
	}
}

// Tx runs the given function safely within a transaction. If the function
// panics with a SQLError, then the transaction is rolled back. Otherwise,
// the transaction is committed.
//
// The first error that occurs (including beginning and ending the transaction)
// is returned.
func Tx(db Beginner, f func()) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	if err := Safe(f); err != nil {
		tx.Rollback() // ignore this error (return the first)
		return err
	}
	return tx.Commit()
}

// Exec returns the result of a running a query that doesn't return any rows.
// If an error occurs, it is panic'd as a SQLError.
func Exec(db Execer, query string, args ...interface{}) sql.Result {
	r, err := db.Exec(query, args...)
	SQLPanic(err)
	return r
}

// Query returns the result of a query that fetches many rows.
// If an error occurs, it is panic'd as a SQLError.
func Query(db Queryer, query string, args ...interface{}) *sql.Rows {
	rs, err := db.Query(query, args...)
	SQLPanic(err)
	return rs
}

// Prepare returns a prepared statement.
// If an error occurs, it is panic'd as a SQLError.
func Prepare(db Preparer, query string) *sql.Stmt {
	stmt, err := db.Prepare(query)
	SQLPanic(err)
	return stmt
}

// Scan performs a scan on a row. If an error occurs, it is panic'd as a
// SQLError.
func Scan(scanner RowScanner, dest ...interface{}) {
	SQLPanic(scanner.Scan(dest...))
}

// Value returns the driver value.
// If an error occurs, it is panic'd as a SQLError.
func Value(v Valuer) driver.Value {
	dval, err := v.Value()
	SQLPanic(err)
	return dval
}
<|endoftext|>


<commit_before>package csrf

import (
	"crypto/sha1"
	"encoding/base64"
	"errors"
	"io"

	"github.com/dchest/uniuri"
	"github.com/gin-gonic/gin"
	"github.com/tommy351/gin-sessions"
)

const (
	csrfSecret = "csrfSecret"
	csrfSalt   = "csrfSalt"
	csrfToken  = "csrfToken"
)

var defaultIgnoreMethods = []string{"GET", "HEAD", "OPTIONS"}

var defaultErrorFunc = func(c *gin.Context) {
	panic(errors.New("CSRF token mismatch"))
}

var defaultTokenGetter = func(c *gin.Context) string {
	r := c.Request

	if t := r.FormValue("_csrf"); len(t) > 0 {
		return t
	} else if t := r.URL.Query().Get("_csrf"); len(t) > 0 {
		return t
	} else if t := r.Header.Get("X-CSRF-TOKEN"); len(t) > 0 {
		return t
	} else if t := r.Header.Get("X-XSRF-TOKEN"); len(t) > 0 {
		return t
	}

	return ""
}

// Options stores configurations for a CSRF middleware.
type Options struct {
	Secret        string
	IgnoreMethods []string
	ErrorFunc     gin.HandlerFunc
	TokenGetter   func(c *gin.Context) string
}

func tokenize(secret, salt string) string {
	h := sha1.New()
	io.WriteString(h, salt+"-"+secret)
	hash := base64.URLEncoding.EncodeToString(h.Sum(nil))

	return hash
}

func inArray(arr []string, value string) bool {
	inarr := false

	for _, v := range arr {
		if v == value {
			inarr = true
			break
		}
	}

	return inarr
}

// Middleware validates CSRF token.
func Middleware(options Options) gin.HandlerFunc {
	ignoreMethods := options.IgnoreMethods
	errorFunc := options.ErrorFunc
	tokenGetter := options.TokenGetter

	if ignoreMethods == nil {
		ignoreMethods = defaultIgnoreMethods
	}

	if errorFunc == nil {
		errorFunc = defaultErrorFunc
	}

	if tokenGetter == nil {
		tokenGetter = defaultTokenGetter
	}

	return func(c *gin.Context) {
		session := sessions.Get(c)
		c.Set(csrfSecret, options.Secret)

		if inArray(ignoreMethods, c.Request.Method) {
			c.Next()
			return
		}

		var salt string

		if s, ok := session.Get(csrfSalt).(string); !ok || len(s) == 0 {
			c.Next()
			return
		} else {
			salt = s
		}

		session.Delete(csrfSalt)

		token := tokenGetter(c)

		if tokenize(options.Secret, salt) != token {
			errorFunc(c)
			return
		}

		c.Next()
	}
}

// GetToken returns a CSRF token.
func GetToken(c *gin.Context) string {
	session := sessions.Get(c)
	secret := c.MustGet(csrfSecret).(string)

	if t, err := c.Get(csrfToken); err == nil {
		return t.(string)
	}

	salt := uniuri.New()
	token := tokenize(secret, salt)
	session.Set(csrfSalt, salt)
	session.Save()
	c.Set(csrfToken, token)

	return token
}
<commit_msg>Update csrf.go<commit_after>package csrf

import (
	"crypto/sha1"
	"encoding/base64"
	"errors"
	"io"

	"github.com/dchest/uniuri"
	"github.com/gin-gonic/gin"
	"github.com/tommy351/gin-sessions"
)

const (
	csrfSecret = "csrfSecret"
	csrfSalt   = "csrfSalt"
	csrfToken  = "csrfToken"
)

var defaultIgnoreMethods = []string{"GET", "HEAD", "OPTIONS"}

var defaultErrorFunc = func(c *gin.Context) {
	panic(errors.New("CSRF token mismatch"))
}

var defaultTokenGetter = func(c *gin.Context) string {
	r := c.Request

	if t := r.FormValue("_csrf"); len(t) > 0 {
		return t
	} else if t := r.URL.Query().Get("_csrf"); len(t) > 0 {
		return t
	} else if t := r.Header.Get("X-CSRF-TOKEN"); len(t) > 0 {
		return t
	} else if t := r.Header.Get("X-XSRF-TOKEN"); len(t) > 0 {
		return t
	}

	return ""
}

// Options stores configurations for a CSRF middleware.
type Options struct {
	Secret        string
	IgnoreMethods []string
	ErrorFunc     gin.HandlerFunc
	TokenGetter   func(c *gin.Context) string
}

func tokenize(secret, salt string) string {
	h := sha1.New()
	io.WriteString(h, salt+"-"+secret)
	hash := base64.URLEncoding.EncodeToString(h.Sum(nil))

	return hash
}

func inArray(arr []string, value string) bool {
	inarr := false

	for _, v := range arr {
		if v == value {
			inarr = true
			break
		}
	}

	return inarr
}

// Middleware validates CSRF token.
func Middleware(options Options) gin.HandlerFunc {
	ignoreMethods := options.IgnoreMethods
	errorFunc := options.ErrorFunc
	tokenGetter := options.TokenGetter

	if ignoreMethods == nil {
		ignoreMethods = defaultIgnoreMethods
	}

	if errorFunc == nil {
		errorFunc = defaultErrorFunc
	}

	if tokenGetter == nil {
		tokenGetter = defaultTokenGetter
	}

	return func(c *gin.Context) {
		session := sessions.Get(c)
		c.Set(csrfSecret, options.Secret)

		if inArray(ignoreMethods, c.Request.Method) {
			c.Next()
			return
		}

		var salt string

		if s, ok := session.Get(csrfSalt).(string); !ok || len(s) == 0 {
			c.Next()
			return
		} else {
			salt = s
		}

		session.Delete(csrfSalt)

		token := tokenGetter(c)

		if tokenize(options.Secret, salt) != token {
			errorFunc(c)
			return
		}

		c.Next()
	}
}

// GetToken returns a CSRF token.
func GetToken(c *gin.Context) string {
	session := sessions.Get(c)
	secret := c.MustGet(csrfSecret).(string)

	if t, ok := c.Get(csrfToken); ok {
		return t.(string)
	}

	salt := uniuri.New()
	token := tokenize(secret, salt)
	session.Set(csrfSalt, salt)
	session.Save()
	c.Set(csrfToken, token)

	return token
}
<|endoftext|>


<commit_before>// redis cache backend implementation for persistent storage

package main

import (
	"bytes"
	"fmt"
	"os"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/garyburd/redigo/redis"
)

const prefix = "genproxy:"

type Cache struct {
	pool   *redis.Pool
	prefix string
}

// set records a key in cache (redis)
func (c *Cache) set(key string, value interface{}) error {

	if c.prefix == "" {
		c.prefix = prefix
	}

	client := c.pool.Get()
	defer client.Close()

	_, err := client.Do("SET", fmt.Sprintf(c.prefix+key), value)

	if err != nil {
		// getting payload for debugging
		var payload string
		switch v := value.(type) {
		case bytes.Buffer: // return as is
			payload = v.String() // Here v is of type bytes.Buffer
		}
		log.WithFields(log.Fields{
			"error":   err.Error(),
			"key":     fmt.Sprintf(c.prefix + key),
			"payload": payload,
		}).Error("Failed to SET key...")
	} else {
		log.WithFields(log.Fields{
			"key": fmt.Sprintf(c.prefix + key),
		}).Info("Key/value SET successfuly!")
	}

	return err
}

// getAllKeys returns all keys for specified (or default) prefix
func (c *Cache) getAllKeys() ([]string, error) {

	client := c.pool.Get()
	defer client.Close()

	values, err := redis.Strings(client.Do("KEYS", fmt.Sprintf(c.prefix+"*")))

	return values, err
}

// getAllValues returns values for specified keys
func (c *Cache) getAllValues(keys []string) ([]string, error) {

	log.WithFields(log.Fields{
		"keys": keys,
	}).Debug("Getting all supplied values")

	client := c.pool.Get()
	defer client.Close()

	// preparing keys
	var args []interface{}
	for _, k := range keys {
		args = append(args, k)
	}

	jsonStr, err := redis.Strings(client.Do("MGET", args...))

	log.WithFields(log.Fields{
		"keys": keys,
	}).Debug("Returning supplied values")

	return jsonStr, err

}

// get returns key from cache
func (c *Cache) get(key string) ([]byte, error) {

	client := c.pool.Get()
	defer client.Close()

	value, err := redis.Bytes(client.Do("GET", fmt.Sprintf(c.prefix+key)))

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"key":   fmt.Sprintf(c.prefix + key),
		}).Error("Failed to GET key...")
	} else {
		log.WithFields(log.Fields{
			"key": fmt.Sprintf(c.prefix + key),
		}).Debug("Key found!")
	}

	return value, err
}

// delete removes specified entry from cache
func (c *Cache) delete(key string) error {
	if c.prefix == "" {
		c.prefix = prefix
	}

	client := c.pool.Get()
	defer client.Close()

	_, err := client.Do("DEL", key)

	return err

}

// getRedisPool returns thread safe Redis connection pool
func getRedisPool() *redis.Pool {

	// getting redis connection
	maxConnections := 10
	mc := os.Getenv("MaxConnections")
	if mc != "" {
		maxCons, err := strconv.Atoi(mc)
		if err != nil {
			maxConnections = 10
		} else {
			maxConnections = maxCons
		}
	}

	// getting redis client for state storing
	redisPool := redis.NewPool(func() (redis.Conn, error) {
		c, err := redis.Dial("tcp", AppConfig.redisAddress)

		if err != nil {
			log.WithFields(log.Fields{
				"Error": err.Error(),
			}).Warn("Failed to create Redis connection pool!")
			return nil, err
		}
		if AppConfig.redisPassword != "" {
			if _, err := c.Do("AUTH", AppConfig.redisPassword); err != nil {
				log.WithFields(log.Fields{
					"Error":        err.Error(),
					"PasswordUsed": AppConfig.redisPassword,
				}).Warn("Failed to authenticate to Redis!")
				c.Close()
				return nil, err
			} else {
				log.Debug("Authenticated to Redis successfully! ")
			}
		}

		return c, err
	}, maxConnections)

	return redisPool
}
<commit_msg>bolt import, default config<commit_after>// redis cache backend implementation for persistent storage

package main

import (
	"bytes"
	"fmt"
	"os"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/boltdb/bolt"
	"github.com/garyburd/redigo/redis"
)

const prefix = "genproxy:"
const requestsBucketName = "rqbucket"

type Cache struct {
	pool   *redis.Pool
	prefix string

	db             *bolt.DB
	requestsBucket []byte
}

// set records a key in cache (redis)
func (c *Cache) set(key string, value interface{}) error {

	if c.prefix == "" {
		c.prefix = prefix
	}

	client := c.pool.Get()
	defer client.Close()

	_, err := client.Do("SET", fmt.Sprintf(c.prefix+key), value)

	if err != nil {
		// getting payload for debugging
		var payload string
		switch v := value.(type) {
		case bytes.Buffer: // return as is
			payload = v.String() // Here v is of type bytes.Buffer
		}
		log.WithFields(log.Fields{
			"error":   err.Error(),
			"key":     fmt.Sprintf(c.prefix + key),
			"payload": payload,
		}).Error("Failed to SET key...")
	} else {
		log.WithFields(log.Fields{
			"key": fmt.Sprintf(c.prefix + key),
		}).Info("Key/value SET successfuly!")
	}

	return err
}

// getAllKeys returns all keys for specified (or default) prefix
func (c *Cache) getAllKeys() ([]string, error) {

	client := c.pool.Get()
	defer client.Close()

	values, err := redis.Strings(client.Do("KEYS", fmt.Sprintf(c.prefix+"*")))

	return values, err
}

// getAllValues returns values for specified keys
func (c *Cache) getAllValues(keys []string) ([]string, error) {

	log.WithFields(log.Fields{
		"keys": keys,
	}).Debug("Getting all supplied values")

	client := c.pool.Get()
	defer client.Close()

	// preparing keys
	var args []interface{}
	for _, k := range keys {
		args = append(args, k)
	}

	jsonStr, err := redis.Strings(client.Do("MGET", args...))

	log.WithFields(log.Fields{
		"keys": keys,
	}).Debug("Returning supplied values")

	return jsonStr, err

}

// get returns key from cache
func (c *Cache) get(key string) ([]byte, error) {

	client := c.pool.Get()
	defer client.Close()

	value, err := redis.Bytes(client.Do("GET", fmt.Sprintf(c.prefix+key)))

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"key":   fmt.Sprintf(c.prefix + key),
		}).Error("Failed to GET key...")
	} else {
		log.WithFields(log.Fields{
			"key": fmt.Sprintf(c.prefix + key),
		}).Debug("Key found!")
	}

	return value, err
}

// delete removes specified entry from cache
func (c *Cache) delete(key string) error {
	if c.prefix == "" {
		c.prefix = prefix
	}

	client := c.pool.Get()
	defer client.Close()

	_, err := client.Do("DEL", key)

	return err

}

// getRedisPool returns thread safe Redis connection pool
func getRedisPool() *redis.Pool {

	// getting redis connection
	maxConnections := 10
	mc := os.Getenv("MaxConnections")
	if mc != "" {
		maxCons, err := strconv.Atoi(mc)
		if err != nil {
			maxConnections = 10
		} else {
			maxConnections = maxCons
		}
	}

	// getting redis client for state storing
	redisPool := redis.NewPool(func() (redis.Conn, error) {
		c, err := redis.Dial("tcp", AppConfig.redisAddress)

		if err != nil {
			log.WithFields(log.Fields{
				"Error": err.Error(),
			}).Warn("Failed to create Redis connection pool!")
			return nil, err
		}
		if AppConfig.redisPassword != "" {
			if _, err := c.Do("AUTH", AppConfig.redisPassword); err != nil {
				log.WithFields(log.Fields{
					"Error":        err.Error(),
					"PasswordUsed": AppConfig.redisPassword,
				}).Warn("Failed to authenticate to Redis!")
				c.Close()
				return nil, err
			} else {
				log.Debug("Authenticated to Redis successfully! ")
			}
		}

		return c, err
	}, maxConnections)

	return redisPool
}
<|endoftext|>


<commit_before>package restic

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/restic/restic/backend"
	"github.com/restic/restic/debug"
	"github.com/restic/restic/server"
)

type Cache struct {
	base string
}

func NewCache(be backend.Identifier) (c *Cache, err error) {
	// try to get explicit cache dir from environment
	dir := os.Getenv("RESTIC_CACHE")

	// otherwise try OS specific default
	if dir == "" {
		dir, err = GetCacheDir()
		if err != nil {
			return nil, err
		}
	}

	basedir := filepath.Join(dir, be.ID())
	debug.Log("Cache.New", "opened cache at %v", basedir)

	return &Cache{base: basedir}, nil
}

func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) {
	// try to open file
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return false, err
	}

	fd, err := os.Open(filename)
	defer fd.Close()
	if err != nil {
		if os.IsNotExist(err) {
			debug.Log("Cache.Has", "test for file %v: not cached", filename)
			return false, nil
		}

		debug.Log("Cache.Has", "test for file %v: error %v", filename, err)
		return false, err
	}

	debug.Log("Cache.Has", "test for file %v: is cached", filename)
	return true, nil
}

func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCloser, error) {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return nil, err
	}

	dirname := filepath.Dir(filename)
	err = os.MkdirAll(dirname, 0700)
	if err != nil {
		return nil, err
	}

	file, err := os.Create(filename)
	if err != nil {
		debug.Log("Cache.Store", "error creating file %v: %v", filename, err)
		return nil, err
	}

	debug.Log("Cache.Store", "created file %v", filename)
	return file, nil
}

func (c *Cache) Load(t backend.Type, subtype string, id backend.ID) (io.ReadCloser, error) {
	// try to open file
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return nil, err
	}

	return os.Open(filename)
}

func (c *Cache) Purge(t backend.Type, subtype string, id backend.ID) error {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return err
	}

	err = os.Remove(filename)
	debug.Log("Cache.Purge", "Remove file %v: %v", filename, err)

	if err != nil && os.IsNotExist(err) {
		return nil
	}

	return err
}

func (c *Cache) Clear(s *server.Server) error {
	list, err := c.List(backend.Snapshot)
	if err != nil {
		return err
	}

	for _, entry := range list {
		debug.Log("Cache.Clear", "found entry %v", entry)

		if ok, err := s.Test(backend.Snapshot, entry.ID.String()); !ok || err != nil {
			debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry)

			err = c.Purge(backend.Snapshot, entry.Subtype, entry.ID)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

type CacheEntry struct {
	ID      backend.ID
	Subtype string
}

func (c CacheEntry) String() string {
	if c.Subtype != "" {
		return c.ID.Str() + "." + c.Subtype
	}
	return c.ID.Str()
}

func (c *Cache) List(t backend.Type) ([]CacheEntry, error) {
	var dir string

	switch t {
	case backend.Snapshot:
		dir = filepath.Join(c.base, "snapshots")
	default:
		return nil, fmt.Errorf("cache not supported for type %v", t)
	}

	fd, err := os.Open(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return []CacheEntry{}, nil
		}
		return nil, err
	}
	defer fd.Close()

	fis, err := fd.Readdir(-1)
	if err != nil {
		return nil, err
	}

	entries := make([]CacheEntry, 0, len(fis))

	for _, fi := range fis {
		parts := strings.SplitN(fi.Name(), ".", 2)

		id, err := backend.ParseID(parts[0])
		// ignore invalid cache entries for now
		if err != nil {
			debug.Log("Cache.List", "unable to parse name %v as id: %v", parts[0], err)
			continue
		}

		e := CacheEntry{ID: id}

		if len(parts) == 2 {
			e.Subtype = parts[1]
		}

		entries = append(entries, e)
	}

	return entries, nil
}

// Construct file name for given Type.
func (c *Cache) filename(t backend.Type, subtype string, id backend.ID) (string, error) {
	filename := id.String()
	if subtype != "" {
		filename += "." + subtype
	}

	switch t {
	case backend.Snapshot:
		return filepath.Join(c.base, "snapshots", filename), nil
	}

	return "", fmt.Errorf("cache not supported for type %v", t)
}

// high-level functions

// GetCacheDir returns the cache directory according to XDG basedir spec, see
// http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
func GetCacheDir() (string, error) {
	xdgcache := os.Getenv("XDG_CACHE_HOME")
	home := os.Getenv("HOME")

	if xdgcache == "" && home == "" {
		return "", errors.New("unable to locate cache directory (XDG_CACHE_HOME and HOME unset)")
	}

	cachedir := ""
	if xdgcache != "" {
		cachedir = filepath.Join(xdgcache, "restic")
	} else if home != "" {
		cachedir = filepath.Join(home, ".cache", "restic")
	}

	fi, err := os.Stat(cachedir)
	if os.IsNotExist(err) {
		err = os.MkdirAll(cachedir, 0700)
		if err != nil {
			return "", err
		}

		fi, err = os.Stat(cachedir)
		debug.Log("getCacheDir", "create cache dir %v", cachedir)
	}

	if err != nil {
		return "", err
	}

	if !fi.IsDir() {
		return "", fmt.Errorf("cache dir %v is not a directory", cachedir)
	}

	return cachedir, nil
}

// TODO: implement RefreshIndex()
<commit_msg>cleanup<commit_after>package restic

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/restic/restic/backend"
	"github.com/restic/restic/debug"
	"github.com/restic/restic/server"
)

type Cache struct {
	base string
}

func NewCache(be backend.Identifier) (*Cache, error) {
	cacheDir, err := getCacheDir()
	if err != nil {
		return nil, err
	}

	basedir := filepath.Join(cacheDir, be.ID())
	debug.Log("Cache.New", "opened cache at %v", basedir)

	return &Cache{base: basedir}, nil
}

func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return false, err
	}
	fd, err := os.Open(filename)
	defer fd.Close()

	if err != nil {
		if os.IsNotExist(err) {
			debug.Log("Cache.Has", "test for file %v: not cached", filename)
			return false, nil
		}

		debug.Log("Cache.Has", "test for file %v: error %v", filename, err)
		return false, err
	}

	debug.Log("Cache.Has", "test for file %v: is cached", filename)
	return true, nil
}

func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCloser, error) {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return nil, err
	}

	dirname := filepath.Dir(filename)
	err = os.MkdirAll(dirname, 0700)
	if err != nil {
		return nil, err
	}

	file, err := os.Create(filename)
	if err != nil {
		debug.Log("Cache.Store", "error creating file %v: %v", filename, err)
		return nil, err
	}

	debug.Log("Cache.Store", "created file %v", filename)
	return file, nil
}

func (c *Cache) Load(t backend.Type, subtype string, id backend.ID) (io.ReadCloser, error) {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return nil, err
	}

	return os.Open(filename)
}

func (c *Cache) purge(t backend.Type, subtype string, id backend.ID) error {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return err
	}

	err = os.Remove(filename)
	debug.Log("Cache.purge", "Remove file %v: %v", filename, err)

	if err != nil && os.IsNotExist(err) {
		return nil
	}

	return err
}

func (c *Cache) Clear(s *server.Server) error {
	list, err := c.List(backend.Snapshot)
	if err != nil {
		return err
	}

	for _, entry := range list {
		debug.Log("Cache.Clear", "found entry %v", entry)

		if ok, err := s.Test(backend.Snapshot, entry.ID.String()); !ok || err != nil {
			debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry)

			err = c.purge(backend.Snapshot, entry.Subtype, entry.ID)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

type CacheEntry struct {
	ID      backend.ID
	Subtype string
}

func (c CacheEntry) String() string {
	if c.Subtype != "" {
		return c.ID.Str() + "." + c.Subtype
	}
	return c.ID.Str()
}

func (c *Cache) List(t backend.Type) ([]CacheEntry, error) {
	var dir string

	switch t {
	case backend.Snapshot:
		dir = filepath.Join(c.base, "snapshots")
	default:
		return nil, fmt.Errorf("cache not supported for type %v", t)
	}

	fd, err := os.Open(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return []CacheEntry{}, nil
		}
		return nil, err
	}
	defer fd.Close()

	fis, err := fd.Readdir(-1)
	if err != nil {
		return nil, err
	}

	entries := make([]CacheEntry, 0, len(fis))

	for _, fi := range fis {
		parts := strings.SplitN(fi.Name(), ".", 2)

		id, err := backend.ParseID(parts[0])
		// ignore invalid cache entries for now
		if err != nil {
			debug.Log("Cache.List", "unable to parse name %v as id: %v", parts[0], err)
			continue
		}

		e := CacheEntry{ID: id}

		if len(parts) == 2 {
			e.Subtype = parts[1]
		}

		entries = append(entries, e)
	}

	return entries, nil
}

func (c *Cache) filename(t backend.Type, subtype string, id backend.ID) (string, error) {
	filename := id.String()
	if subtype != "" {
		filename += "." + subtype
	}

	switch t {
	case backend.Snapshot:
		return filepath.Join(c.base, "snapshots", filename), nil
	}

	return "", fmt.Errorf("cache not supported for type %v", t)
}

func getCacheDir() (string, error) {
	if dir := os.Getenv("RESTIC_CACHE"); dir != "" {
		return dir, nil
	}

	return getXDGCacheDir()
}

// getXDGCacheDir returns the cache directory according to XDG basedir spec, see
// http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
func getXDGCacheDir() (string, error) {
	xdgcache := os.Getenv("XDG_CACHE_HOME")
	home := os.Getenv("HOME")

	if xdgcache == "" && home == "" {
		return "", errors.New("unable to locate cache directory (XDG_CACHE_HOME and HOME unset)")
	}

	cachedir := ""
	if xdgcache != "" {
		cachedir = filepath.Join(xdgcache, "restic")
	} else if home != "" {
		cachedir = filepath.Join(home, ".cache", "restic")
	}

	fi, err := os.Stat(cachedir)
	if os.IsNotExist(err) {
		err = os.MkdirAll(cachedir, 0700)
		if err != nil {
			return "", err
		}

		fi, err = os.Stat(cachedir)
		debug.Log("getCacheDir", "create cache dir %v", cachedir)
	}

	if err != nil {
		return "", err
	}

	if !fi.IsDir() {
		return "", fmt.Errorf("cache dir %v is not a directory", cachedir)
	}

	return cachedir, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	flag "github.com/ogier/pflag"
	"io/ioutil"
	"os"
	"text/template"
)

func main() {
	dataPath := flag.StringP("json", "j", "-", "JSON data file")
	tmplPath := flag.StringP("template", "t", "-", "Go template file")
	outPath := flag.StringP("output", "o", "-", "output file")

	flag.Parse()

	data, err := loadJSONData(*dataPath)
	if err != nil {
		panic(err)
	}
	tmpl, err := loadGoTemplate(*tmplPath)
	if err != nil {
		panic(err)
	}

	var outFile *os.File
	if *outPath == "-" {
		outFile = os.Stdout
	} else {
		outFile, err = os.Create(*outPath)
		if err != nil {
			panic(err)
		}
		defer outFile.Close()
	}

	err = tmpl.Execute(outFile, data)
	if err != nil {
		panic(err)
	}
}

// loadJSONData unmarshals JSON-encoded data from the file specified by path,
// and returns the result as an interface{}. If the path is "-", then data will
// be acquired from os.Stdin.
func loadJSONData(path string) (ret interface{}, err error) {
	var file *os.File
	if path == "-" {
		file = os.Stdin
	} else {
		file, err = os.Open(path)
		if err != nil {
			return
		}
		defer file.Close()
	}

	decoder := json.NewDecoder(file)
	err = decoder.Decode(&ret)
	return // whether err==nil or not, our work is done
}

// loadGoTemplate parses a Go text template from the file specified by path.
// The file contents are parsed into a top-level template with the name "root".
// If the path is "-", then the template will be parsed from os.Stdin.
func loadGoTemplate(path string) (tmpl *template.Template, err error) {
	var file *os.File
	if path == "-" {
		file = os.Stdin
	} else {
		file, err = os.Open(path)
		if err != nil {
			return
		}
		defer file.Close()
	}

	rawTmpl, err := ioutil.ReadAll(file)
	if err != nil {
		return
	}

	// explicitly parse the raw string rather than using ParseFiles
	// ParseFiles creates templates whose names are those of the files
	// and associates them with the parent template
	// this creates some confusing behavior in Template.Parse
	// see http://stackoverflow.com/questions/11805356/text-template-issue-parse-vs-parsefiles
	tmpl, err = template.New("root").Parse(string(rawTmpl))
	return // again, whether err==nil or not, this is finished
}
<commit_msg>Add help and version<commit_after>package main

import (
	"encoding/json"
	"fmt"
	flag "github.com/ogier/pflag"
	"io/ioutil"
	"os"
	"text/template"
)

const helpStr = `    jgtr - JSON Go Template Renderer

USAGE:
    jgtr [OPTIONS]

    jgtr consumes a JSON-encoded data file and a template file written in Go's
    text/template language. The values available in the data file are then used
    to render the template file and generate output.

    By default, jgtr reads the data and template from stdin, and writes output
    to stdout. Note that data and template cannot both come from stdin - at
    least one of the two must be specified via an option.

OPTIONS:
    -j FILE, --json=FILE
        Read JSON data from FILE. Specify "-" (the default) to use stdin.

    -t FILE, --template=FILE
        Read template from FILE. Specify "-" (the default) to use stdin.

    -o FILE, --output=FILE
        Write rendered template to FILE. Specify "-" (the default) to use
        stdout.

    -h, --help
        Display this help.

    -V, --version
        Display jgtr version.`

const versionStr = `0.0.0`

func main() {
	help := flag.BoolP("help", "h", false, "show help")
	version := flag.BoolP("version", "V", false, "show version")

	dataPath := flag.StringP("json", "j", "-", "JSON data file")
	tmplPath := flag.StringP("template", "t", "-", "Go template file")
	outPath := flag.StringP("output", "o", "-", "output file")

	flag.Parse()

	if *help {
		fmt.Println(helpStr)
		return
	}
	if *version {
		println(versionStr)
		return
	}

	if *dataPath == "-" && *tmplPath == "-" {
		println("Cannot use stdin for data and template simultaneously")
		os.Exit(1)
	}

	data, err := loadJSONData(*dataPath)
	if err != nil {
		panic(err)
	}
	tmpl, err := loadGoTemplate(*tmplPath)
	if err != nil {
		panic(err)
	}

	var outFile *os.File
	if *outPath == "-" {
		outFile = os.Stdout
	} else {
		outFile, err = os.Create(*outPath)
		if err != nil {
			panic(err)
		}
		defer outFile.Close()
	}

	err = tmpl.Execute(outFile, data)
	if err != nil {
		panic(err)
	}
}

// loadJSONData unmarshals JSON-encoded data from the file specified by path,
// and returns the result as an interface{}. If the path is "-", then data will
// be acquired from os.Stdin.
func loadJSONData(path string) (ret interface{}, err error) {
	var file *os.File
	if path == "-" {
		file = os.Stdin
	} else {
		file, err = os.Open(path)
		if err != nil {
			return
		}
		defer file.Close()
	}

	decoder := json.NewDecoder(file)
	err = decoder.Decode(&ret)
	return // whether err==nil or not, our work is done
}

// loadGoTemplate parses a Go text template from the file specified by path.
// The file contents are parsed into a top-level template with the name "root".
// If the path is "-", then the template will be parsed from os.Stdin.
func loadGoTemplate(path string) (tmpl *template.Template, err error) {
	var file *os.File
	if path == "-" {
		file = os.Stdin
	} else {
		file, err = os.Open(path)
		if err != nil {
			return
		}
		defer file.Close()
	}

	rawTmpl, err := ioutil.ReadAll(file)
	if err != nil {
		return
	}

	// explicitly parse the raw string rather than using ParseFiles
	// ParseFiles creates templates whose names are those of the files
	// and associates them with the parent template
	// this creates some confusing behavior in Template.Parse
	// see http://stackoverflow.com/questions/11805356/text-template-issue-parse-vs-parsefiles
	tmpl, err = template.New("root").Parse(string(rawTmpl))
	return // again, whether err==nil or not, this is finished
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io"
	"strings"
)

// JSONIn provides methods of the Input interface
type JSONIn struct {
	reader   *json.Decoder
	firstRow []string
	header   []string
	ajson    []interface{}
	count    int
}

// JSONOut provides methods of the Output interface
type JSONOut struct {
	writer  *json.Encoder
	results []map[string]string
}

func (trdsql *TRDSQL) jsonInputNew(r io.Reader) (Input, error) {
	jr := &JSONIn{}
	jr.reader = json.NewDecoder(r)
	return jr, nil
}

func (jr *JSONIn) FirstRead() ([]string, error) {
	var top interface{}
	err := jr.reader.Decode(&top)
	if err != nil {
		return nil, err
	}
	err = jr.topRow(top)
	if err != nil {
		return nil, err
	}
	debug.Printf("Column Name: [%v]", strings.Join(jr.header, ","))
	return jr.header, err
}

func (jr *JSONIn) topRow(top interface{}) error {
	switch top.(type) {
	case []interface{}:
		jr.ajson = top.([]interface{})
		val := jr.ajson[0]
		switch val.(type) {
		case map[string]interface{}:
			jr.objectFirstRow(val.(map[string]interface{}))
			return nil
		case []interface{}:
			jr.etcFirstRow(val)
		default:
			// ["a","b"]
			jr.ajson = nil
			jr.etcFirstRow(top)
		}
	case map[string]interface{}:
		// {"a":"b"} object
		jr.ajson = nil
		jr.objectFirstRow(top.(map[string]interface{}))
	}
	return nil
}

func (jr *JSONIn) objectFirstRow(obj map[string]interface{}) {
	// {"a":"b"} object
	for k, v := range obj {
		jr.header = append(jr.header, k)
		jr.firstRow = append(jr.firstRow, jsonStr(v))
	}
}

func (jr *JSONIn) etcFirstRow(val interface{}) {
	// array array
	// [["a"],
	//  ["b"]]
	jr.header = append(jr.header, "c1")
	jr.firstRow = append(jr.firstRow, jsonStr(val))
}

func jsonStr(val interface{}) string {
	switch val.(type) {
	case map[string]interface{}:
		str, _ := json.Marshal(val)
		return valString(str)
	case []interface{}:
		str, _ := json.Marshal(val)
		return valString(str)
	default:
		return valString(val)
	}
}

func (jr *JSONIn) FirstRowRead(list []interface{}) []interface{} {
	for i := range jr.header {
		list[i] = jr.firstRow[i]
	}
	return list
}

func (jr *JSONIn) RowRead(list []interface{}) ([]interface{}, error) {
	if jr.ajson != nil {
		// [] array
		jr.count++
		if jr.count >= len(jr.ajson) {
			return nil, io.EOF
		}
		list = jr.rowParse(list, jr.ajson[jr.count])
	} else {
		var data interface{}
		// {} object
		err := jr.reader.Decode(&data)
		if err != nil {
			return nil, err
		}
		list = jr.rowParse(list, data)
	}
	return list, nil
}

func (jr *JSONIn) rowParse(list []interface{}, jsonRow interface{}) []interface{} {
	switch jsonRow.(type) {
	case map[string]interface{}:
		dmap := jsonRow.(map[string]interface{})
		for i := range jr.header {
			list[i] = jsonStr(dmap[jr.header[i]])
		}
	default:
		list[0] = jsonStr(jsonRow)
	}
	return list
}

func (trdsql *TRDSQL) jsonOutNew() Output {
	js := &JSONOut{}
	js.writer = json.NewEncoder(trdsql.outStream)
	js.writer.SetIndent("", "  ")
	return js
}

func (js *JSONOut) First(columns []string) error {
	js.results = make([]map[string]string, 0)
	return nil
}

func (js *JSONOut) RowWrite(values []interface{}, columns []string) error {
	m := make(map[string]string, len(columns))
	for i, col := range values {
		m[columns[i]] = valString(col)
	}
	js.results = append(js.results, m)
	return nil
}

func (js *JSONOut) Last() error {
	return js.writer.Encode(js.results)
}
<commit_msg>Multiple array support<commit_after>package main

import (
	"encoding/json"
	"io"
	"log"
	"strings"
)

// JSONIn provides methods of the Input interface
type JSONIn struct {
	reader   *json.Decoder
	firstRow []string
	header   []string
	ajson    []interface{}
	count    int
}

// JSONOut provides methods of the Output interface
type JSONOut struct {
	writer  *json.Encoder
	results []map[string]string
}

func (trdsql *TRDSQL) jsonInputNew(r io.Reader) (Input, error) {
	jr := &JSONIn{}
	jr.reader = json.NewDecoder(r)
	return jr, nil
}

// FirstRead is read input to determine column of table
func (jr *JSONIn) FirstRead() ([]string, error) {
	var top interface{}
	err := jr.reader.Decode(&top)
	if err != nil {
		return nil, err
	}
	jr.header, jr.firstRow = jr.topLevel(top)
	debug.Printf("Column Name: [%v]", strings.Join(jr.header, ","))
	return jr.header, err
}

func (jr *JSONIn) topLevel(top interface{}) ([]string, []string) {
	switch top.(type) {
	case []interface{}:
		jr.ajson = top.([]interface{})
		val := jr.ajson[0]
		return jr.secondLevel(top, val)
	case map[string]interface{}:
		// {"a":"b"} object
		jr.ajson = nil
		return jr.objectFirstRow(top.(map[string]interface{}))
	default:
		log.Printf("Not a table format")
	}
	return nil, nil
}

// Analyze second when top is array
func (jr *JSONIn) secondLevel(top interface{}, second interface{}) ([]string, []string) {
	switch second.(type) {
	case map[string]interface{}:
		// [{}]
		return jr.objectFirstRow(second.(map[string]interface{}))
	case []interface{}:
		// [[]]
		return jr.etcFirstRow(second)
	default:
		// ["a","b"]
		jr.ajson = nil
		return jr.etcFirstRow(top)
	}
}

func (jr *JSONIn) objectFirstRow(obj map[string]interface{}) ([]string, []string) {
	// {"a":"b"} object
	var header []string
	var firstRow []string
	for k, v := range obj {
		header = append(header, k)
		firstRow = append(firstRow, jsonString(v))
	}
	return header, firstRow
}

func (jr *JSONIn) etcFirstRow(val interface{}) ([]string, []string) {
	// ex. array array
	// [["a"],
	//  ["b"]]
	var header []string
	var firstRow []string
	header = append(header, "c1")
	firstRow = append(firstRow, jsonString(val))
	return header, firstRow
}

func jsonString(val interface{}) string {
	switch val.(type) {
	case map[string]interface{}, []interface{}:
		str, err := json.Marshal(val)
		if err != nil {
			log.Println("ERROR:", err)
		}
		return valString(str)
	default:
		return valString(val)
	}
}

// FirstRowRead is read the first row
func (jr *JSONIn) FirstRowRead(list []interface{}) []interface{} {
	for i := range jr.header {
		list[i] = jr.firstRow[i]
	}
	return list
}

// RowRead is read 2row or later
func (jr *JSONIn) RowRead(list []interface{}) ([]interface{}, error) {
	if jr.ajson != nil {
		// [] array
		jr.count++
		if jr.count >= len(jr.ajson) {
			var top interface{}
			err := jr.reader.Decode(&top)
			if err != nil {
				return nil, err
			}
			_, jr.firstRow = jr.topLevel(top)
			jr.count = 0
		}
		list = jr.rowParse(list, jr.ajson[jr.count])
	} else {
		// {} object
		var data interface{}
		err := jr.reader.Decode(&data)
		if err != nil {
			return nil, err
		}
		list = jr.rowParse(list, data)
	}
	return list, nil
}

func (jr *JSONIn) rowParse(list []interface{}, jsonRow interface{}) []interface{} {
	switch jsonRow.(type) {
	case map[string]interface{}:
		dmap := jsonRow.(map[string]interface{})
		for i := range jr.header {
			list[i] = jsonString(dmap[jr.header[i]])
		}
	default:
		list[0] = jsonString(jsonRow)
	}
	return list
}

func (trdsql *TRDSQL) jsonOutNew() Output {
	js := &JSONOut{}
	js.writer = json.NewEncoder(trdsql.outStream)
	js.writer.SetIndent("", "  ")
	return js
}

// First is preparation
func (js *JSONOut) First(columns []string) error {
	js.results = make([]map[string]string, 0)
	return nil
}

// RowWrite is Addition to array
func (js *JSONOut) RowWrite(values []interface{}, columns []string) error {
	m := make(map[string]string, len(columns))
	for i, col := range values {
		m[columns[i]] = valString(col)
	}
	js.results = append(js.results, m)
	return nil
}

// Last is Actual output
func (js *JSONOut) Last() error {
	return js.writer.Encode(js.results)
}
<|endoftext|>


<commit_before>// Clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2019, 2, 1, 0, 0, 0, 0, time.Local)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
	indent         = "\t"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<commit_msg>March!<commit_after>// Clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2019, 3, 1, 0, 0, 0, 0, time.Local)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
	indent         = "\t"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<|endoftext|>