<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"

	flag "github.com/bborbe/flagenv"
	io_util "github.com/bborbe/io/util"
	"github.com/bborbe/log"
	"github.com/bborbe/server/handler/auth_basic"
	"github.com/facebookgo/grace/gracehttp"
)

const (
	PARAMETER_LOGLEVEL   = "loglevel"
	PARAMETER_AUTH_USER  = "auth-user"
	PARAMETER_AUTH_PASS  = "auth-pass"
	PARAMETER_AUTH_REALM = "auth-realm"
)

var (
	logger          = log.DefaultLogger
	portPtr         = flag.Int("port", 8080, "Port")
	documentRootPtr = flag.String("root", "", "Document root directory")
	logLevelPtr     = flag.String(PARAMETER_LOGLEVEL, log.INFO_STRING, log.FLAG_USAGE)
	authUserPtr     = flag.String(PARAMETER_AUTH_USER, "", "basic auth username")
	authPassPtr     = flag.String(PARAMETER_AUTH_PASS, "", "basic auth password")
	authRealmPtr    = flag.String(PARAMETER_AUTH_REALM, "", "basic auth realm")
)

func main() {
	defer logger.Close()
	flag.Parse()

	logger.SetLevelThreshold(log.LogStringToLevel(*logLevelPtr))
	logger.Debugf("set log level to %s", *logLevelPtr)

	server, err := createServer(*portPtr, *documentRootPtr, *authUserPtr, *authPassPtr, *authRealmPtr)
	if err != nil {
		logger.Fatal(err)
		logger.Close()
		os.Exit(1)
	}
	logger.Debugf("start server")
	gracehttp.Serve(server)
}

func createServer(port int, documentRoot string, authUser string, authPass string, authRealm string) (*http.Server, error) {
	root, err := io_util.NormalizePath(documentRoot)
	if err != nil {
		return nil, err
	}
	var handler http.Handler = http.FileServer(http.Dir(root))
	if len(authUser) > 0 && len(authPass) > 0 && len(authRealm) > 0 {
		handler = auth_basic.New(handler.ServeHTTP, func(username string, password string) (bool, error) {
			return username == authUser && password == authPass, nil
		}, authRealm)
	}
	return &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: handler}, nil
}
<commit_msg>add const for root<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"

	flag "github.com/bborbe/flagenv"
	io_util "github.com/bborbe/io/util"
	"github.com/bborbe/log"
	"github.com/bborbe/server/handler/auth_basic"
	"github.com/facebookgo/grace/gracehttp"
)

const (
	PARAMETER_ROOT       = "root"
	PARAMETER_LOGLEVEL   = "loglevel"
	PARAMETER_AUTH_USER  = "auth-user"
	PARAMETER_AUTH_PASS  = "auth-pass"
	PARAMETER_AUTH_REALM = "auth-realm"
)

var (
	logger          = log.DefaultLogger
	portPtr         = flag.Int("port", 8080, "Port")
	documentRootPtr = flag.String(PARAMETER_ROOT, "", "Document root directory")
	logLevelPtr     = flag.String(PARAMETER_LOGLEVEL, log.INFO_STRING, log.FLAG_USAGE)
	authUserPtr     = flag.String(PARAMETER_AUTH_USER, "", "basic auth username")
	authPassPtr     = flag.String(PARAMETER_AUTH_PASS, "", "basic auth password")
	authRealmPtr    = flag.String(PARAMETER_AUTH_REALM, "", "basic auth realm")
)

func main() {
	defer logger.Close()
	flag.Parse()

	logger.SetLevelThreshold(log.LogStringToLevel(*logLevelPtr))
	logger.Debugf("set log level to %s", *logLevelPtr)

	server, err := createServer(*portPtr, *documentRootPtr, *authUserPtr, *authPassPtr, *authRealmPtr)
	if err != nil {
		logger.Fatal(err)
		logger.Close()
		os.Exit(1)
	}
	logger.Debugf("start server")
	gracehttp.Serve(server)
}

func createServer(port int, documentRoot string, authUser string, authPass string, authRealm string) (*http.Server, error) {
	root, err := io_util.NormalizePath(documentRoot)
	if err != nil {
		return nil, err
	}
	var handler http.Handler = http.FileServer(http.Dir(root))
	if len(authUser) > 0 && len(authPass) > 0 && len(authRealm) > 0 {
		handler = auth_basic.New(handler.ServeHTTP, func(username string, password string) (bool, error) {
			return username == authUser && password == authPass, nil
		}, authRealm)
	}
	return &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: handler}, nil
}
<|endoftext|>


<commit_before>package websocket

import (
	"encoding/json"
	"net/http"
	"sync"
	"time"

	log "github.com/cihub/seelog"
	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
	"github.com/pborman/uuid"
)

type Message struct {
	Type string          `json:"type"`
	Data json.RawMessage `json:"data"`
	To   string          `json:"to"`
}
type Clients struct {
	sync.Mutex
	clients []*Client
	Router  *Router `inject:""`
}
type Client struct {
	out        chan<- *Message
	done       <-chan bool
	err        <-chan error
	Id         string
	disconnect chan int
}

// Add a client to a room
func (r *Clients) appendClient(client *Client) {
	r.Lock()
	r.clients = append(r.clients, client)
	r.Unlock()

	msgs := r.Router.RunOnClientConnectHandlers()
	for _, msg := range msgs {
		client.out <- msg
	}
}

// Message all the other clients
func (r *Clients) SendToAll(t string, data interface{}) {

	out, err := json.Marshal(data)
	if err != nil {
		log.Error(err)
		return
	}
	msg := &Message{Type: t, Data: out}

	r.Lock()
	clientsToRemove := make([]*Client, 0)

	for _, c := range r.clients {
		select {
		case c.out <- msg:
			// Everything went well :)
		case <-time.After(time.Second):
			log.Warn("Failed writing to websocket: timeout (", c.Id, ")")
			clientsToRemove = append(clientsToRemove, c)
		}
	}

	r.Unlock()
	go func() {
		for _, c := range clientsToRemove {
			r.removeClient(c)
		}
	}()
}

// Remove a client
func (r *Clients) removeClient(client *Client) {
	r.Lock()
	defer r.Unlock()

	for index, c := range r.clients {
		if c == client {
			c.disconnect <- websocket.CloseInternalServerErr
			r.clients = append(r.clients[:index], r.clients[(index+1):]...)
		}
	}
}

// Disconnect all clients
func (r *Clients) disconnectAll() {
	r.Lock()
	defer r.Unlock()

	for _, c := range r.clients {
		c.disconnect <- websocket.CloseGoingAway
	}
}

func newClients() *Clients {
	return &Clients{sync.Mutex{}, make([]*Client, 0), nil}
}

//func (clients *Clients) WebsocketRoute(params martini.Params, receiver <-chan *Message, sender chan<- *Message, done <-chan bool, disconnect chan<- int, err <-chan error) (int, string) {
//client := &Client{params["clientname"], receiver, sender, done, err, disconnect}
//clients.appendClient(client)

//// A single select can be used to do all the messaging
//for {
//select {
//case <-client.err:
//// Don't try to do this:
//// client.out <- &Message{"system", "system", "There has been an error with your connection"}
//// The socket connection is already long gone.
//// Use the error for statistics etc
//case msg := <-client.in:
////TODO implement command from websocket here. using same process as WebHandlerCommandToNode

//log.Info("incoming message from webui on websocket", string(msg.Data))
//clients.Router.Run(msg)
//case <-client.done:
//clients.removeClient(client)
//return 200, "OK"
//}
//}
//}
func (clients *Clients) WebsocketRoute(c *gin.Context) {
	conn, err := websocket.Upgrade(c.Writer, c.Request, nil, 1024, 1024)
	if err != nil {
		http.Error(c.Writer, "Websocket error", 400)
		log.Error(err)
		return
	}

	out := make(chan *Message)
	go senderWorker(conn, out)
	done := make(chan bool)
	wsErr := make(chan error)
	disconnect := make(chan int)
	go disconnectWorker(conn, disconnect)
	client := &Client{out, done, wsErr, uuid.New(), disconnect}
	clients.appendClient(client)

	//Listen to websocket
	for {
		msg := &Message{}
		err := conn.ReadJSON(msg)
		if err != nil {
			log.Info(conn.LocalAddr(), " Disconnected")
			close(done)
			close(out)
			close(disconnect)
			clients.removeClient(client)
			break
		}
		go clients.Router.Run(msg)
	}
}

func disconnectWorker(conn *websocket.Conn, disconnect <-chan int) {
	for code := range disconnect {
		conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(code, ""), time.Now().Add(10*time.Second))
		if err := conn.Close(); err != nil {
			log.Error("Connection could not be closed: %s", err)
			return
		}
	}
}
func senderWorker(conn *websocket.Conn, c chan *Message) {
	for msg := range c {
		err := conn.WriteJSON(msg)
		if err != nil {
			log.Error(err)
		}

	}
}
<commit_msg>ping over ws and fix panic<commit_after>package websocket

import (
	"encoding/json"
	"net/http"
	"sync"
	"time"

	log "github.com/cihub/seelog"
	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
	"github.com/pborman/uuid"
)

const (
	// Time allowed to write the file to the client.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the client.
	pongWait = 60 * time.Second

	// Send pings to client with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10
)

type Message struct {
	Type string          `json:"type"`
	Data json.RawMessage `json:"data"`
	To   string          `json:"to"`
}
type Clients struct {
	sync.Mutex
	clients []*Client
	Router  *Router `inject:""`
}
type Client struct {
	out        chan<- *Message
	done       <-chan bool
	err        <-chan error
	Id         string
	disconnect chan int
}

// Add a client to a room
func (r *Clients) appendClient(client *Client) {
	r.Lock()
	r.clients = append(r.clients, client)
	r.Unlock()

	msgs := r.Router.RunOnClientConnectHandlers()
	for _, msg := range msgs {
		client.out <- msg
	}
}

// Message all the other clients
func (r *Clients) SendToAll(t string, data interface{}) {

	out, err := json.Marshal(data)
	if err != nil {
		log.Error(err)
		return
	}
	msg := &Message{Type: t, Data: out}

	r.Lock()
	clientsToRemove := make([]*Client, 0)

	for _, c := range r.clients {
		select {
		case c.out <- msg:
			// Everything went well :)
		case <-time.After(time.Second):
			log.Warn("Failed writing to websocket: timeout (", c.Id, ")")
			clientsToRemove = append(clientsToRemove, c)
		}
	}

	r.Unlock()
	go func() {
		for _, c := range clientsToRemove {
			r.removeClient(c)
		}
	}()
}

// Remove a client
func (r *Clients) removeClient(client *Client) {
	r.Lock()
	defer r.Unlock()

	for index, c := range r.clients {
		if c == client {
			c.disconnect <- websocket.CloseInternalServerErr
			r.clients = append(r.clients[:index], r.clients[(index+1):]...)
		}
	}
}

// Disconnect all clients
func (r *Clients) disconnectAll() {
	r.Lock()
	defer r.Unlock()

	for _, c := range r.clients {
		c.disconnect <- websocket.CloseGoingAway
	}
}

func newClients() *Clients {
	return &Clients{sync.Mutex{}, make([]*Client, 0), nil}
}

func (clients *Clients) WebsocketRoute(c *gin.Context) {
	conn, err := websocket.Upgrade(c.Writer, c.Request, nil, 1024, 1024)
	if err != nil {
		http.Error(c.Writer, "Websocket error", 400)
		log.Error(err)
		return
	}

	out := make(chan *Message)
	done := make(chan bool)
	wsErr := make(chan error)
	disconnect := make(chan int)
	client := &Client{out, done, wsErr, uuid.New(), disconnect}

	go senderWorker(conn, out)
	go disconnectWorker(conn, client)

	clients.appendClient(client)

	conn.SetReadDeadline(time.Now().Add(pongWait))
	conn.SetPongHandler(func(string) error {
		//log.Debug("Got pong response from browser")
		conn.SetReadDeadline(time.Now().Add(pongWait))
		return nil
	})

	//Listen to websocket
	for {
		msg := &Message{}
		err := conn.ReadJSON(msg)
		if err != nil {
			log.Info(conn.LocalAddr(), " Disconnected")
			close(done)
			close(out)
			clients.removeClient(client)
			break
		}
		go clients.Router.Run(msg)
	}
}

func disconnectWorker(conn *websocket.Conn, c *Client) {
	for code := range c.disconnect {
		log.Debug("Closing websocket")
		conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(code, ""), time.Now().Add(writeWait))
		//We can only disconnect once so we can close this channel here
		close(c.disconnect)
		if err := conn.Close(); err != nil {
			log.Error("Connection could not be closed: %s", err)
		}
		return
	}
}
func senderWorker(conn *websocket.Conn, out chan *Message) {
	pingTicker := time.NewTicker(pingPeriod)
	for {
		select {
		case msg, opened := <-out:
			err := conn.WriteJSON(msg)
			if err != nil {
				log.Error(err)
			}
			if !opened {
				pingTicker.Stop()
				return
			}
		case <-pingTicker.C:
			if err := conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait)); err != nil {
				log.Error(err)
				return
			}

		}
	}
}
<|endoftext|>


<commit_before>package mithril

import (
	"mithril/log"
	"mithril/message"
	"sync"
	"time"

	"github.com/streadway/amqp"
)

type amqpAdaptedRequest struct {
	Publishing *amqp.Publishing
	Exchange   string
	RoutingKey string
	Mandatory  bool
	Immediate  bool
}

type AMQPPublisher struct {
	amqpUri         string
	amqpConn        *amqp.Connection
	handlingChannel *amqp.Channel
	confirmAck      chan uint64
	confirmNack     chan uint64
	notifyClose     chan *amqp.Error
	sync.Mutex
}

func NewAMQPPublisher(amqpUri string) (*AMQPPublisher, error) {
	publisher := &AMQPPublisher{
		amqpUri: amqpUri,
	}

	if err := publisher.establishConnection(); err != nil {
		return nil, err
	}
	return publisher, nil
}

func (me *AMQPPublisher) Publish(req *message.Message) error {
	var (
		amqpReq *amqpAdaptedRequest
		err     error
	)

	amqpReq = me.adaptHttpRequest(req)
	if err = me.publishAdaptedRequest(amqpReq); err != nil {
		log.Println("amqp - Failed to publish request:", err)
		return err
	}
	return nil
}

func (me *AMQPPublisher) establishConnection() (err error) {
	me.Lock()
	defer me.Unlock()

	if me.amqpConn != nil {
		return
	}

	log.Printf("amqp - no RabbitMQ connection found, establishing new connection...")
	me.amqpConn, err = amqp.Dial(me.amqpUri)
	if err != nil {
		return err
	}
	log.Printf("amqp - connected to RabbitMQ")

	log.Printf("amqp - creating channel...")
	me.handlingChannel, err = me.amqpConn.Channel()
	if err != nil {
		return err
	}
	log.Printf("amqp - channel created")

	log.Printf("amqp - setting confirm mode...")
	if err = me.handlingChannel.Confirm(false); err != nil {
		return err
	}
	log.Printf("amqp - confirm mode set")

	me.confirmAck, me.confirmNack = me.handlingChannel.NotifyConfirm(make(chan uint64, 1), make(chan uint64, 1))
	log.Printf("amqp - notify confirm channels created.")

	go func() {
		closeChan := me.handlingChannel.NotifyClose(make(chan *amqp.Error))

		select {
		case e := <-closeChan:
			log.Printf("amqp - The channel opened with RabbitMQ has been closed. %d: %s", e.Code, e.Reason)
			me.disconnect()
		}
	}()

	log.Printf("amqp - Ready to publish messages!")
	return nil
}

func (me *AMQPPublisher) disconnect() {
	me.Lock()
	defer me.Unlock()

	if me.handlingChannel != nil {
		me.handlingChannel.Close()
		me.handlingChannel = nil
	}

	if me.amqpConn != nil {
		me.amqpConn.Close()
		me.amqpConn = nil
	}
}

func (me *AMQPPublisher) adaptHttpRequest(req *message.Message) *amqpAdaptedRequest {
	return &amqpAdaptedRequest{
		Publishing: &amqp.Publishing{
			MessageId:     req.MessageId,
			CorrelationId: req.CorrelationId,
			Timestamp:     req.Timestamp,
			AppId:         req.AppId,
			ContentType:   req.ContentType,
			Body:          req.BodyBytes,
		},
		Exchange:   req.Exchange,
		RoutingKey: req.RoutingKey,
		Mandatory:  req.Mandatory,
		Immediate:  req.Immediate,
	}
}

func (me *AMQPPublisher) publishAdaptedRequest(amqpReq *amqpAdaptedRequest) (err error) {
	err = me.establishConnection()
	if err != nil {
		return err
	}

	me.Lock()
	err = me.handlingChannel.Publish(amqpReq.Exchange,
		amqpReq.RoutingKey, amqpReq.Mandatory,
		amqpReq.Immediate, *amqpReq.Publishing)
	me.Unlock()

	if err != nil {
		return err
	}

	select {
	case _ = <-me.confirmAck:
		return nil
	case _ = <-me.confirmNack:
		log.Printf("amqp - RabbitMQ nack'd a message at %s", time.Now().UTC())
		return nil
	}
}
<commit_msg>Updated the delivery mode to be persistent<commit_after>package mithril

import (
	"mithril/log"
	"mithril/message"
	"sync"
	"time"

	"github.com/streadway/amqp"
)

type amqpAdaptedRequest struct {
	Publishing *amqp.Publishing
	Exchange   string
	RoutingKey string
	Mandatory  bool
	Immediate  bool
}

type AMQPPublisher struct {
	amqpUri         string
	amqpConn        *amqp.Connection
	handlingChannel *amqp.Channel
	confirmAck      chan uint64
	confirmNack     chan uint64
	notifyClose     chan *amqp.Error
	sync.Mutex
}

func NewAMQPPublisher(amqpUri string) (*AMQPPublisher, error) {
	publisher := &AMQPPublisher{
		amqpUri: amqpUri,
	}

	if err := publisher.establishConnection(); err != nil {
		return nil, err
	}
	return publisher, nil
}

func (me *AMQPPublisher) Publish(req *message.Message) error {
	var (
		amqpReq *amqpAdaptedRequest
		err     error
	)

	amqpReq = me.adaptHttpRequest(req)
	if err = me.publishAdaptedRequest(amqpReq); err != nil {
		log.Println("amqp - Failed to publish request:", err)
		return err
	}
	return nil
}

func (me *AMQPPublisher) establishConnection() (err error) {
	me.Lock()
	defer me.Unlock()

	if me.amqpConn != nil {
		return
	}

	log.Printf("amqp - no RabbitMQ connection found, establishing new connection...")
	me.amqpConn, err = amqp.Dial(me.amqpUri)
	if err != nil {
		return err
	}
	log.Printf("amqp - connected to RabbitMQ")

	log.Printf("amqp - creating channel...")
	me.handlingChannel, err = me.amqpConn.Channel()
	if err != nil {
		return err
	}
	log.Printf("amqp - channel created")

	log.Printf("amqp - setting confirm mode...")
	if err = me.handlingChannel.Confirm(false); err != nil {
		return err
	}
	log.Printf("amqp - confirm mode set")

	me.confirmAck, me.confirmNack = me.handlingChannel.NotifyConfirm(make(chan uint64, 1), make(chan uint64, 1))
	log.Printf("amqp - notify confirm channels created.")

	go func() {
		closeChan := me.handlingChannel.NotifyClose(make(chan *amqp.Error))

		select {
		case e := <-closeChan:
			log.Printf("amqp - The channel opened with RabbitMQ has been closed. %d: %s", e.Code, e.Reason)
			me.disconnect()
		}
	}()

	log.Printf("amqp - Ready to publish messages!")
	return nil
}

func (me *AMQPPublisher) disconnect() {
	me.Lock()
	defer me.Unlock()

	if me.handlingChannel != nil {
		me.handlingChannel.Close()
		me.handlingChannel = nil
	}

	if me.amqpConn != nil {
		me.amqpConn.Close()
		me.amqpConn = nil
	}
}

func (me *AMQPPublisher) adaptHttpRequest(req *message.Message) *amqpAdaptedRequest {
	return &amqpAdaptedRequest{
		Publishing: &amqp.Publishing{
			MessageId:     req.MessageId,
			CorrelationId: req.CorrelationId,
			Timestamp:     req.Timestamp,
			AppId:         req.AppId,
			ContentType:   req.ContentType,
			Body:          req.BodyBytes,
			DeliveryMode:  amqp.Persistent,
		},
		Exchange:   req.Exchange,
		RoutingKey: req.RoutingKey,
		Mandatory:  req.Mandatory,
		Immediate:  req.Immediate,
	}
}

func (me *AMQPPublisher) publishAdaptedRequest(amqpReq *amqpAdaptedRequest) (err error) {
	err = me.establishConnection()
	if err != nil {
		return err
	}

	me.Lock()
	err = me.handlingChannel.Publish(amqpReq.Exchange,
		amqpReq.RoutingKey, amqpReq.Mandatory,
		amqpReq.Immediate, *amqpReq.Publishing)
	me.Unlock()

	if err != nil {
		return err
	}

	select {
	case _ = <-me.confirmAck:
		return nil
	case _ = <-me.confirmNack:
		log.Printf("amqp - RabbitMQ nack'd a message at %s", time.Now().UTC())
		return nil
	}
}
<|endoftext|>


<commit_before>package bktree

import (
	"github.com/kavu/go-phash"
	"log"
)

type WalkerNode struct {
	HashValue uint64
	Object    interface{}
}

type Node struct {
	HashValue uint64
	Object    interface{}
	Children  map[int]Node
}

func New(hashValue uint64, object interface{}) Node {
	node := Node{HashValue: hashValue, Object: object}
	node.Children = make(map[int]Node)
	return node
}

func (node *Node) Walk() []WalkerNode {
	var walkerNodes []WalkerNode
	for _, child := range node.Children {
		walkerNode := WalkerNode{
			HashValue: child.HashValue,
			Object:    child.Object,
		}
		walkerNodes = append(walkerNodes, walkerNode)
		walkerNodes = append(walkerNodes, child.Walk()...)
	}
	return walkerNodes
}

func (node *Node) Insert(hashValue uint64, object interface{}) {
	distance, err := phash.HammingDistanceForHashes(node.HashValue, hashValue)
	if err != nil {
		log.Fatalf("Unable to generate hamming distance")
	}

	if nextNode, ok := node.Children[distance]; ok {
		nextNode.Insert(hashValue, object)
	} else {
		node.Children[distance] = New(hashValue, object)
	}
}

func (node *Node) Find(hashValue uint64, allowedDistance int) []interface{} {
	distance, err := phash.HammingDistanceForHashes(node.HashValue, hashValue)
	if err != nil {
		log.Fatalf("Unable to generate hamming distance")
	}
	minDistance := distance - allowedDistance
	maxDistance := distance + allowedDistance

	var matchingNodes []interface{}
	if distance <= allowedDistance {
		matchingNodes = append(matchingNodes, node.Object)
	}

	for childDistance, child := range node.Children {
		if childDistance <= maxDistance || childDistance >= minDistance {
			childNodes := child.Find(hashValue, allowedDistance)
			if len(childNodes) > 0 {
				matchingNodes = append(matchingNodes, childNodes...)
			}
		}
	}

	return matchingNodes
}

func PHashValueForImage(path string) uint64 {
	hash, _ := phash.ImageHashDCT(path)
	return hash
}
<commit_msg>Format main file with go-formats<commit_after>package bktree

import (
	"log"

	"github.com/kavu/go-phash"
)

type WalkerNode struct {
	HashValue uint64
	Object    interface{}
}

type Node struct {
	HashValue uint64
	Object    interface{}
	Children  map[int]Node
}

func New(hashValue uint64, object interface{}) Node {
	node := Node{HashValue: hashValue, Object: object}
	node.Children = make(map[int]Node)
	return node
}

func (node *Node) Walk() []WalkerNode {
	var walkerNodes []WalkerNode
	for _, child := range node.Children {
		walkerNode := WalkerNode{
			HashValue: child.HashValue,
			Object:    child.Object,
		}
		walkerNodes = append(walkerNodes, walkerNode)
		walkerNodes = append(walkerNodes, child.Walk()...)
	}
	return walkerNodes
}

func (node *Node) Insert(hashValue uint64, object interface{}) {
	distance, err := phash.HammingDistanceForHashes(node.HashValue, hashValue)
	if err != nil {
		log.Fatalf("Unable to generate hamming distance")
	}

	if nextNode, ok := node.Children[distance]; ok {
		nextNode.Insert(hashValue, object)
	} else {
		node.Children[distance] = New(hashValue, object)
	}
}

func (node *Node) Find(hashValue uint64, allowedDistance int) []interface{} {
	distance, err := phash.HammingDistanceForHashes(node.HashValue, hashValue)
	if err != nil {
		log.Fatalf("Unable to generate hamming distance")
	}
	minDistance := distance - allowedDistance
	maxDistance := distance + allowedDistance

	var matchingNodes []interface{}
	if distance <= allowedDistance {
		matchingNodes = append(matchingNodes, node.Object)
	}

	for childDistance, child := range node.Children {
		if childDistance <= maxDistance || childDistance >= minDistance {
			childNodes := child.Find(hashValue, allowedDistance)
			if len(childNodes) > 0 {
				matchingNodes = append(matchingNodes, childNodes...)
			}
		}
	}

	return matchingNodes
}

func PHashValueForImage(path string) uint64 {
	hash, _ := phash.ImageHashDCT(path)
	return hash
}
<|endoftext|>


<commit_before>package httpruntime

import (
	"io"
	"net/http"
	"strings"

	"github.com/gogo/protobuf/jsonpb"
	"github.com/gogo/protobuf/proto"
)

type Marshaler interface {
	ContentType() string
	Unmarshal(io.Reader, proto.Message) error
	Marshal(io.Writer, proto.Message) error
}

var marshalDict = map[string]Marshaler{
	"application/json": MarshalerPbJSON{Marshaler: &jsonpb.Marshaler{}},
}

// OverrideMarshaler replaces marshaler for given content-type.
func OverrideMarshaler(contentType string, m Marshaler) {
	marshalDict[strings.ToLower(contentType)] = m
}

// MarshalerForRequest returns marshalers for inbound and outbound bodies.
func MarshalerForRequest(r *http.Request) (Marshaler, Marshaler) {
	inbound := marshalerOrDefault(r.Header.Get("Content-Type"))
	outbound := marshalerOrDefault(r.Header.Get("Accept"))
	return inbound, outbound
}

func marshalerOrDefault(t string) Marshaler {
	sepIdx := strings.IndexAny(t, ";,")
	// TODO we're not negotiating really. Account the q= param and additional
	// options
	if sepIdx > 0 {
		t = t[:sepIdx]
	}
	t = strings.ToLower(t)

	if m, ok := marshalDict[t]; ok {
		return m
	}
	return marshalDict[MarshalerPbJSON{}.ContentType()]
}

type MarshalerPbJSON struct {
	Marshaler *jsonpb.Marshaler
}

func (MarshalerPbJSON) ContentType() string {
	return "application/json"
}

func (MarshalerPbJSON) Unmarshal(r io.Reader, dst proto.Message) error {
	return jsonpb.Unmarshal(r, dst)
}

func (m MarshalerPbJSON) Marshal(w io.Writer, src proto.Message) error {
	return m.Marshaler.Marshal(w, src)
}
<commit_msg>gogo jsonpb -> golang jsonpb<commit_after>package httpruntime

import (
	"io"
	"net/http"
	"strings"

	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
)

type Marshaler interface {
	ContentType() string
	Unmarshal(io.Reader, proto.Message) error
	Marshal(io.Writer, proto.Message) error
}

var marshalDict = map[string]Marshaler{
	"application/json": MarshalerPbJSON{Marshaler: &jsonpb.Marshaler{}},
}

// OverrideMarshaler replaces marshaler for given content-type.
func OverrideMarshaler(contentType string, m Marshaler) {
	marshalDict[strings.ToLower(contentType)] = m
}

// MarshalerForRequest returns marshalers for inbound and outbound bodies.
func MarshalerForRequest(r *http.Request) (Marshaler, Marshaler) {
	inbound := marshalerOrDefault(r.Header.Get("Content-Type"))
	outbound := marshalerOrDefault(r.Header.Get("Accept"))
	return inbound, outbound
}

func marshalerOrDefault(t string) Marshaler {
	sepIdx := strings.IndexAny(t, ";,")
	// TODO we're not negotiating really. Account the q= param and additional
	// options
	if sepIdx > 0 {
		t = t[:sepIdx]
	}
	t = strings.ToLower(t)

	if m, ok := marshalDict[t]; ok {
		return m
	}
	return marshalDict[MarshalerPbJSON{}.ContentType()]
}

type MarshalerPbJSON struct {
	Marshaler *jsonpb.Marshaler
}

func (MarshalerPbJSON) ContentType() string {
	return "application/json"
}

func (MarshalerPbJSON) Unmarshal(r io.Reader, dst proto.Message) error {
	return jsonpb.Unmarshal(r, dst)
}

func (m MarshalerPbJSON) Marshal(w io.Writer, src proto.Message) error {
	return m.Marshaler.Marshal(w, src)
}
<|endoftext|>


<commit_before>package kcp

import (
	"io"
	"sync"
	"time"

	"github.com/v2ray/v2ray-core/common/alloc"
)

type ReceivingWindow struct {
	start uint32
	size  uint32
	list  []*DataSegment
}

func NewReceivingWindow(size uint32) *ReceivingWindow {
	return &ReceivingWindow{
		start: 0,
		size:  size,
		list:  make([]*DataSegment, size),
	}
}

func (this *ReceivingWindow) Size() uint32 {
	return this.size
}

func (this *ReceivingWindow) Position(idx uint32) uint32 {
	return (idx + this.start) % this.size
}

func (this *ReceivingWindow) Set(idx uint32, value *DataSegment) bool {
	pos := this.Position(idx)
	if this.list[pos] != nil {
		return false
	}
	this.list[pos] = value
	return true
}

func (this *ReceivingWindow) Remove(idx uint32) *DataSegment {
	pos := this.Position(idx)
	e := this.list[pos]
	this.list[pos] = nil
	return e
}

func (this *ReceivingWindow) RemoveFirst() *DataSegment {
	return this.Remove(0)
}

func (this *ReceivingWindow) Advance() {
	this.start++
	if this.start == this.size {
		this.start = 0
	}
}

type ReceivingQueue struct {
	sync.Mutex
	closed  bool
	cache   *alloc.Buffer
	queue   chan *alloc.Buffer
	timeout time.Time
}

func NewReceivingQueue() *ReceivingQueue {
	return &ReceivingQueue{
		queue: make(chan *alloc.Buffer, effectiveConfig.GetReceivingQueueSize()),
	}
}

func (this *ReceivingQueue) Read(buf []byte) (int, error) {
	if this.cache.Len() > 0 {
		nBytes, err := this.cache.Read(buf)
		if this.cache.IsEmpty() {
			this.cache.Release()
			this.cache = nil
		}
		return nBytes, err
	}

	var totalBytes int

L:
	for totalBytes < len(buf) {
		timeToSleep := time.Millisecond
		select {
		case payload, open := <-this.queue:
			if !open {
				return totalBytes, io.EOF
			}
			nBytes, err := payload.Read(buf)
			totalBytes += nBytes
			if err != nil {
				return totalBytes, err
			}
			if !payload.IsEmpty() {
				this.cache = payload
			}
			buf = buf[nBytes:]
		case <-time.After(timeToSleep):
			if totalBytes > 0 {
				break L
			}
			if !this.timeout.IsZero() && this.timeout.Before(time.Now()) {
				return totalBytes, errTimeout
			}
			timeToSleep += 500 * time.Millisecond
		}
	}

	return totalBytes, nil
}

func (this *ReceivingQueue) Put(payload *alloc.Buffer) bool {
	this.Lock()
	defer this.Unlock()

	if this.closed {
		payload.Release()
		return false
	}

	select {
	case this.queue <- payload:
		return true
	default:
		return false
	}
}

func (this *ReceivingQueue) SetReadDeadline(t time.Time) error {
	this.timeout = t
	return nil
}

func (this *ReceivingQueue) Close() {
	this.Lock()
	defer this.Unlock()

	if this.closed {
		return
	}
	this.closed = true
	close(this.queue)
}

type AckList struct {
	sync.Mutex
	writer     SegmentWriter
	timestamps []uint32
	numbers    []uint32
	nextFlush  []uint32
}

func NewACKList(writer SegmentWriter) *AckList {
	return &AckList{
		writer:     writer,
		timestamps: make([]uint32, 0, 32),
		numbers:    make([]uint32, 0, 32),
		nextFlush:  make([]uint32, 0, 32),
	}
}

func (this *AckList) Add(number uint32, timestamp uint32) {
	this.Lock()
	defer this.Unlock()

	this.timestamps = append(this.timestamps, timestamp)
	this.numbers = append(this.numbers, number)
	this.nextFlush = append(this.nextFlush, 0)
}

func (this *AckList) Clear(una uint32) {
	this.Lock()
	defer this.Unlock()

	count := 0
	for i := 0; i < len(this.numbers); i++ {
		if this.numbers[i] >= una {
			if i != count {
				this.numbers[count] = this.numbers[i]
				this.timestamps[count] = this.timestamps[i]
				this.nextFlush[count] = this.nextFlush[i]
			}
			count++
		}
	}
	if count < len(this.numbers) {
		this.numbers = this.numbers[:count]
		this.timestamps = this.timestamps[:count]
		this.nextFlush = this.nextFlush[:count]
	}
}

func (this *AckList) Flush(current uint32, rto uint32) {
	seg := new(AckSegment)
	this.Lock()
	for i := 0; i < len(this.numbers); i++ {
		if this.nextFlush[i] <= current {
			seg.Count++
			seg.NumberList = append(seg.NumberList, this.numbers[i])
			seg.TimestampList = append(seg.TimestampList, this.timestamps[i])
			this.nextFlush[i] = current + rto/2
			if seg.Count == 128 {
				break
			}
		}
	}
	this.Unlock()
	if seg.Count > 0 {
		this.writer.Write(seg)
	}
}

type ReceivingWorker struct {
	kcp         *KCP
	queue       *ReceivingQueue
	window      *ReceivingWindow
	windowMutex sync.Mutex
	acklist     *AckList
	updated     bool
	nextNumber  uint32
	windowSize  uint32
}

func NewReceivingWorker(kcp *KCP) *ReceivingWorker {
	windowSize := effectiveConfig.GetReceivingWindowSize()
	worker := &ReceivingWorker{
		kcp:        kcp,
		queue:      NewReceivingQueue(),
		window:     NewReceivingWindow(windowSize),
		windowSize: windowSize,
	}
	worker.acklist = NewACKList(worker)
	return worker
}

func (this *ReceivingWorker) ProcessSendingNext(number uint32) {
	this.acklist.Clear(number)
}

func (this *ReceivingWorker) ProcessSegment(seg *DataSegment) {
	number := seg.Number
	if _itimediff(number, this.nextNumber+this.windowSize) >= 0 || _itimediff(number, this.nextNumber) < 0 {
		return
	}

	this.ProcessSendingNext(seg.SendingNext)

	this.acklist.Add(number, seg.Timestamp)
	this.windowMutex.Lock()
	idx := number - this.nextNumber

	if !this.window.Set(idx, seg) {
		seg.Release()
	}
	this.windowMutex.Unlock()

	this.DumpWindow()
}

// @Private
func (this *ReceivingWorker) DumpWindow() {
	this.windowMutex.Lock()
	defer this.windowMutex.Unlock()

	for {
		seg := this.window.RemoveFirst()
		if seg == nil {
			break
		}

		if !this.queue.Put(seg.Data) {
			this.window.Set(0, seg)
			break
		}

		seg.Data = nil
		this.window.Advance()
		this.nextNumber++
		this.updated = true
	}
}

func (this *ReceivingWorker) Read(b []byte) (int, error) {
	return this.queue.Read(b)
}

func (this *ReceivingWorker) SetReadDeadline(t time.Time) {
	this.queue.SetReadDeadline(t)
}

func (this *ReceivingWorker) Flush() {
	this.acklist.Flush(this.kcp.current, this.kcp.rx_rto)
}

func (this *ReceivingWorker) Write(seg ISegment) {
	ackSeg := seg.(*AckSegment)
	ackSeg.Conv = this.kcp.conv
	ackSeg.ReceivingNext = this.nextNumber
	ackSeg.ReceivingWindow = this.nextNumber + this.windowSize
	if this.kcp.state == StateReadyToClose {
		ackSeg.Opt = SegmentOptionClose
	}
	this.kcp.output.Write(ackSeg)
	this.updated = false
}

func (this *ReceivingWorker) CloseRead() {
	this.queue.Close()
}

func (this *ReceivingWorker) PingNecessary() bool {
	return this.updated
}
<commit_msg>rename NewAckList<commit_after>package kcp

import (
	"io"
	"sync"
	"time"

	"github.com/v2ray/v2ray-core/common/alloc"
)

type ReceivingWindow struct {
	start uint32
	size  uint32
	list  []*DataSegment
}

func NewReceivingWindow(size uint32) *ReceivingWindow {
	return &ReceivingWindow{
		start: 0,
		size:  size,
		list:  make([]*DataSegment, size),
	}
}

func (this *ReceivingWindow) Size() uint32 {
	return this.size
}

func (this *ReceivingWindow) Position(idx uint32) uint32 {
	return (idx + this.start) % this.size
}

func (this *ReceivingWindow) Set(idx uint32, value *DataSegment) bool {
	pos := this.Position(idx)
	if this.list[pos] != nil {
		return false
	}
	this.list[pos] = value
	return true
}

func (this *ReceivingWindow) Remove(idx uint32) *DataSegment {
	pos := this.Position(idx)
	e := this.list[pos]
	this.list[pos] = nil
	return e
}

func (this *ReceivingWindow) RemoveFirst() *DataSegment {
	return this.Remove(0)
}

func (this *ReceivingWindow) Advance() {
	this.start++
	if this.start == this.size {
		this.start = 0
	}
}

type ReceivingQueue struct {
	sync.Mutex
	closed  bool
	cache   *alloc.Buffer
	queue   chan *alloc.Buffer
	timeout time.Time
}

func NewReceivingQueue() *ReceivingQueue {
	return &ReceivingQueue{
		queue: make(chan *alloc.Buffer, effectiveConfig.GetReceivingQueueSize()),
	}
}

func (this *ReceivingQueue) Read(buf []byte) (int, error) {
	if this.cache.Len() > 0 {
		nBytes, err := this.cache.Read(buf)
		if this.cache.IsEmpty() {
			this.cache.Release()
			this.cache = nil
		}
		return nBytes, err
	}

	var totalBytes int

L:
	for totalBytes < len(buf) {
		timeToSleep := time.Millisecond
		select {
		case payload, open := <-this.queue:
			if !open {
				return totalBytes, io.EOF
			}
			nBytes, err := payload.Read(buf)
			totalBytes += nBytes
			if err != nil {
				return totalBytes, err
			}
			if !payload.IsEmpty() {
				this.cache = payload
			}
			buf = buf[nBytes:]
		case <-time.After(timeToSleep):
			if totalBytes > 0 {
				break L
			}
			if !this.timeout.IsZero() && this.timeout.Before(time.Now()) {
				return totalBytes, errTimeout
			}
			timeToSleep += 500 * time.Millisecond
		}
	}

	return totalBytes, nil
}

func (this *ReceivingQueue) Put(payload *alloc.Buffer) bool {
	this.Lock()
	defer this.Unlock()

	if this.closed {
		payload.Release()
		return false
	}

	select {
	case this.queue <- payload:
		return true
	default:
		return false
	}
}

func (this *ReceivingQueue) SetReadDeadline(t time.Time) error {
	this.timeout = t
	return nil
}

func (this *ReceivingQueue) Close() {
	this.Lock()
	defer this.Unlock()

	if this.closed {
		return
	}
	this.closed = true
	close(this.queue)
}

type AckList struct {
	sync.Mutex
	writer     SegmentWriter
	timestamps []uint32
	numbers    []uint32
	nextFlush  []uint32
}

func NewAckList(writer SegmentWriter) *AckList {
	return &AckList{
		writer:     writer,
		timestamps: make([]uint32, 0, 32),
		numbers:    make([]uint32, 0, 32),
		nextFlush:  make([]uint32, 0, 32),
	}
}

func (this *AckList) Add(number uint32, timestamp uint32) {
	this.Lock()
	defer this.Unlock()

	this.timestamps = append(this.timestamps, timestamp)
	this.numbers = append(this.numbers, number)
	this.nextFlush = append(this.nextFlush, 0)
}

func (this *AckList) Clear(una uint32) {
	this.Lock()
	defer this.Unlock()

	count := 0
	for i := 0; i < len(this.numbers); i++ {
		if this.numbers[i] >= una {
			if i != count {
				this.numbers[count] = this.numbers[i]
				this.timestamps[count] = this.timestamps[i]
				this.nextFlush[count] = this.nextFlush[i]
			}
			count++
		}
	}
	if count < len(this.numbers) {
		this.numbers = this.numbers[:count]
		this.timestamps = this.timestamps[:count]
		this.nextFlush = this.nextFlush[:count]
	}
}

func (this *AckList) Flush(current uint32, rto uint32) {
	seg := new(AckSegment)
	this.Lock()
	for i := 0; i < len(this.numbers); i++ {
		if this.nextFlush[i] <= current {
			seg.Count++
			seg.NumberList = append(seg.NumberList, this.numbers[i])
			seg.TimestampList = append(seg.TimestampList, this.timestamps[i])
			this.nextFlush[i] = current + rto/2
			if seg.Count == 128 {
				break
			}
		}
	}
	this.Unlock()
	if seg.Count > 0 {
		this.writer.Write(seg)
	}
}

type ReceivingWorker struct {
	kcp         *KCP
	queue       *ReceivingQueue
	window      *ReceivingWindow
	windowMutex sync.Mutex
	acklist     *AckList
	updated     bool
	nextNumber  uint32
	windowSize  uint32
}

func NewReceivingWorker(kcp *KCP) *ReceivingWorker {
	windowSize := effectiveConfig.GetReceivingWindowSize()
	worker := &ReceivingWorker{
		kcp:        kcp,
		queue:      NewReceivingQueue(),
		window:     NewReceivingWindow(windowSize),
		windowSize: windowSize,
	}
	worker.acklist = NewAckList(worker)
	return worker
}

func (this *ReceivingWorker) ProcessSendingNext(number uint32) {
	this.acklist.Clear(number)
}

func (this *ReceivingWorker) ProcessSegment(seg *DataSegment) {
	number := seg.Number
	if _itimediff(number, this.nextNumber+this.windowSize) >= 0 || _itimediff(number, this.nextNumber) < 0 {
		return
	}

	this.ProcessSendingNext(seg.SendingNext)

	this.acklist.Add(number, seg.Timestamp)
	this.windowMutex.Lock()
	idx := number - this.nextNumber

	if !this.window.Set(idx, seg) {
		seg.Release()
	}
	this.windowMutex.Unlock()

	this.DumpWindow()
}

// @Private
func (this *ReceivingWorker) DumpWindow() {
	this.windowMutex.Lock()
	defer this.windowMutex.Unlock()

	for {
		seg := this.window.RemoveFirst()
		if seg == nil {
			break
		}

		if !this.queue.Put(seg.Data) {
			this.window.Set(0, seg)
			break
		}

		seg.Data = nil
		this.window.Advance()
		this.nextNumber++
		this.updated = true
	}
}

func (this *ReceivingWorker) Read(b []byte) (int, error) {
	return this.queue.Read(b)
}

func (this *ReceivingWorker) SetReadDeadline(t time.Time) {
	this.queue.SetReadDeadline(t)
}

func (this *ReceivingWorker) Flush() {
	this.acklist.Flush(this.kcp.current, this.kcp.rx_rto)
}

func (this *ReceivingWorker) Write(seg ISegment) {
	ackSeg := seg.(*AckSegment)
	ackSeg.Conv = this.kcp.conv
	ackSeg.ReceivingNext = this.nextNumber
	ackSeg.ReceivingWindow = this.nextNumber + this.windowSize
	if this.kcp.state == StateReadyToClose {
		ackSeg.Opt = SegmentOptionClose
	}
	this.kcp.output.Write(ackSeg)
	this.updated = false
}

func (this *ReceivingWorker) CloseRead() {
	this.queue.Close()
}

func (this *ReceivingWorker) PingNecessary() bool {
	return this.updated
}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
	"strings"
	"time"
)

var conn dbox.IConnection
var count int

var (
	t0                          time.Time
	fiscalyear, iscount, scount int
	data                        map[string]float64
	masters                     = toolkit.M{}
)

type sgaalloc struct {
	ChannelID                                    string
	TotalNow, TotalExpect, RatioNow, RatioExpect float64
	TotalSales                                   float64
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, _ := gdrj.Find(fnModel(), filter, nil)
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func getstep(count int) int {
	v := count / 100
	if v == 0 {
		return 1
	}
	return v
}

func prepmastertargetdatapromo() {
	toolkit.Println("--> Get Data rawdatapl_promotarget")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("rawdatapl_promotarget").Cursor(nil)
	defer csr.Close()

	promotarget := toolkit.M{}

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		promotarget.Set(tkm.GetString("_id"), tkm)
	}

	masters.Set("promotarget", promotarget)
}

//rawdatapl_promospg11072016_ratio
func prepmasteraggrdatapromo() {

	toolkit.Println("--> Get Data rawdatapl_promotarget")

	filter := dbox.Eq("year", fiscalyear-1)
	csr, _ := conn.NewQuery().Select().Where(filter).From("rawdatapl_promospg11072016").Cursor(nil)
	defer csr.Close()
	//promoaggrs := masters.Get("promoaggr").(toolkit.M)
	promoaggr := toolkit.M{}

	promospgaggr := toolkit.M{}
	promo2aggr := toolkit.M{}

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}
		v := promoaggr.GetFloat64(tkm.GetString("keyaccountcode")) + tkm.GetFloat64("amountinidr")
		promoaggr.Set(tkm.GetString("keyaccountcode"), v)

		if strings.Contains(tkm.GetString("grouping"), "SPG") {
			v := promospgaggr.GetFloat64(tkm.GetString("keyaccountcode")) + tkm.GetFloat64("amountinidr")
			promospgaggr.Set(tkm.GetString("keyaccountcode"), v)
		} else {
			v := promo2aggr.GetFloat64(tkm.GetString("keyaccountcode")) + tkm.GetFloat64("amountinidr")
			promo2aggr.Set(tkm.GetString("keyaccountcode"), v)
		}
	}

	masters.Set("promoaggr", promoaggr)
	masters.Set("promospgaggr", promospgaggr)
	masters.Set("promo2aggr", promo2aggr)

}

func main() {
	t0 = time.Now()
	data = make(map[string]float64)
	flag.IntVar(&fiscalyear, "year", 2015, "YYYY representation of godrej fiscal year. Default is 2015")
	flag.Parse()

	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	setinitialconnection()
	defer gdrj.CloseDb()
	prepmastertargetdatapromo()
	prepmasteraggrdatapromo()

	toolkit.Println("Start data query...")
	filter := dbox.Eq("year", fiscalyear-1)
	csr, _ := workerconn.NewQuery().Select().Where(filter).From("rawdatapl_promospg11072016").Cursor(nil)
	defer csr.Close()

	scount = csr.Count()

	jobs := make(chan toolkit.M, scount)
	result := make(chan int, scount)
	for wi := 0; wi < 10; wi++ {
		go workersave(wi, jobs, result)
	}

	iscount = 0
	step := getstep(scount) * 5

	for {
		iscount++
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- tkm

		if iscount%step == 0 {
			toolkit.Printfn("Sending %d of %d (%d) in %s", iscount, scount, iscount*100/scount,
				time.Since(t0).String())
		}

	}

	close(jobs)

	for ri := 0; ri < scount; ri++ {
		<-result

		if ri%step == 0 {
			toolkit.Printfn("Saving %d of %d (%d pct) in %s",
				ri, scount, ri*100/scount, time.Since(t0).String())
		}
	}

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func UpdateRawDataPromo(tkm toolkit.M) {
	promotargets := masters.Get("promotarget").(toolkit.M)
	promoaggrs := masters.Get("promoaggr").(toolkit.M)

	promospgaggr := masters.Get("promospgaggr").(toolkit.M)
	promo2aggr := masters.Get("promo2aggr").(toolkit.M)

	// masters.Set("promospgaggr", promospgaggr)
	// masters.Set("promo2aggr", promo2aggr)

	promotarget := toolkit.M{}
	if promotargets.Has(tkm.GetString("keyaccountcode")) {
		promotarget = promotargets.Get(tkm.GetString("keyaccountcode")).(toolkit.M)
	}

	divide := promo2aggr.GetFloat64(tkm.GetString("keyaccountcode"))
	if strings.Contains(tkm.GetString("grouping"), "SPG") {
		divide = promospgaggr.GetFloat64(tkm.GetString("keyaccountcode"))
	}

	target := promotarget.GetFloat64("target2015") * gdrj.SaveDiv(divide, promoaggrs.GetFloat64(tkm.GetString("keyaccountcode")))

	val := gdrj.SaveDiv(tkm.GetFloat64("amountinidr"), divide) * target

	tkm.Set("amountinidr_target", val)
}

func workersave(wi int, jobs <-chan toolkit.M, result chan<- int) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	qSave := workerconn.NewQuery().
		From("rawdatapl_promospg11072016_targetratio").
		SetConfig("multiexec", true).
		Save()

	trx := toolkit.M{}
	for trx = range jobs {
		UpdateRawDataPromo(trx)

		err := qSave.Exec(toolkit.M{}.Set("data", trx))
		if err != nil {
			toolkit.Println(err)
		}

		result <- 1
	}
}
<commit_msg>new ratio<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
	"strings"
	"time"
)

var conn dbox.IConnection
var count int

var (
	t0                          time.Time
	fiscalyear, iscount, scount int
	data                        map[string]float64
	masters                     = toolkit.M{}
)

type sgaalloc struct {
	ChannelID                                    string
	TotalNow, TotalExpect, RatioNow, RatioExpect float64
	TotalSales                                   float64
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, _ := gdrj.Find(fnModel(), filter, nil)
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func getstep(count int) int {
	v := count / 100
	if v == 0 {
		return 1
	}
	return v
}

func prepmastertargetdatapromo() {
	toolkit.Println("--> Get Data rawdatapl_promotarget")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("rawdatapl_promotarget").Cursor(nil)
	defer csr.Close()

	promotarget := toolkit.M{}

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		promotarget.Set(tkm.GetString("_id"), tkm)
	}

	masters.Set("promotarget", promotarget)
}

//rawdatapl_promospg11072016_ratio
func prepmasteraggrdatapromo() {

	toolkit.Println("--> Get Data rawdatapl_promo")

	filter := dbox.Eq("year", fiscalyear-1)
	csr, _ := conn.NewQuery().Select().Where(filter).From("rawdatapl_promospg11072016").Cursor(nil)
	defer csr.Close()
	//promoaggrs := masters.Get("promoaggr").(toolkit.M)
	promoaggr := toolkit.M{}

	promospgaggr := toolkit.M{}
	promo2aggr := toolkit.M{}

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}
		v := promoaggr.GetFloat64(tkm.GetString("keyaccountcode")) + tkm.GetFloat64("amountinidr")
		promoaggr.Set(tkm.GetString("keyaccountcode"), v)

		if strings.Contains(tkm.GetString("grouping"), "SPG") {
			v := promospgaggr.GetFloat64(tkm.GetString("keyaccountcode")) + tkm.GetFloat64("amountinidr")
			promospgaggr.Set(tkm.GetString("keyaccountcode"), v)
		} else {
			v := promo2aggr.GetFloat64(tkm.GetString("keyaccountcode")) + tkm.GetFloat64("amountinidr")
			promo2aggr.Set(tkm.GetString("keyaccountcode"), v)
		}
	}

	masters.Set("promoaggr", promoaggr)
	masters.Set("promospgaggr", promospgaggr)
	masters.Set("promo2aggr", promo2aggr)

	// toolkit.Println("year", promoaggr)
	// toolkit.Println("promo", promoaggr)
	// toolkit.Println("spg", promoaggr)

}

func main() {
	t0 = time.Now()
	data = make(map[string]float64)
	flag.IntVar(&fiscalyear, "year", 2015, "YYYY representation of godrej fiscal year. Default is 2015")
	flag.Parse()

	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	setinitialconnection()
	defer gdrj.CloseDb()
	prepmastertargetdatapromo()
	prepmasteraggrdatapromo()

	toolkit.Println("Start data query...")
	filter := dbox.Eq("year", fiscalyear-1)
	csr, _ := workerconn.NewQuery().Select().Where(filter).From("rawdatapl_promospg11072016").Cursor(nil)
	defer csr.Close()

	scount = csr.Count()

	jobs := make(chan toolkit.M, scount)
	result := make(chan int, scount)
	for wi := 0; wi < 10; wi++ {
		go workersave(wi, jobs, result)
	}

	iscount = 0
	step := getstep(scount) * 5

	for {
		iscount++
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- tkm

		if iscount%step == 0 {
			toolkit.Printfn("Sending %d of %d (%d) in %s", iscount, scount, iscount*100/scount,
				time.Since(t0).String())
		}

	}

	close(jobs)

	for ri := 0; ri < scount; ri++ {
		<-result

		if ri%step == 0 {
			toolkit.Printfn("Saving %d of %d (%d pct) in %s",
				ri, scount, ri*100/scount, time.Since(t0).String())
		}
	}

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func UpdateRawDataPromo(tkm toolkit.M) {
	promotargets := masters.Get("promotarget").(toolkit.M)
	promoaggrs := masters.Get("promoaggr").(toolkit.M)

	promospgaggr := masters.Get("promospgaggr").(toolkit.M)
	promo2aggr := masters.Get("promo2aggr").(toolkit.M)

	// masters.Set("promospgaggr", promospgaggr)
	// masters.Set("promo2aggr", promo2aggr)

	promotarget := toolkit.M{}
	if promotargets.Has(tkm.GetString("keyaccountcode")) {
		promotarget = promotargets.Get(tkm.GetString("keyaccountcode")).(toolkit.M)
	}

	divide := promo2aggr.GetFloat64(tkm.GetString("keyaccountcode"))
	if strings.Contains(tkm.GetString("grouping"), "SPG") {
		divide = promospgaggr.GetFloat64(tkm.GetString("keyaccountcode"))
	}

	target := promotarget.GetFloat64("target2015") * gdrj.SaveDiv(divide, promoaggrs.GetFloat64(tkm.GetString("keyaccountcode")))

	val := gdrj.SaveDiv(tkm.GetFloat64("amountinidr"), divide) * target

	tkm.Set("amountinidr_target", val)
}

func workersave(wi int, jobs <-chan toolkit.M, result chan<- int) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	qSave := workerconn.NewQuery().
		From("rawdatapl_promospg11072016_targetratio").
		SetConfig("multiexec", true).
		Save()

	trx := toolkit.M{}
	for trx = range jobs {
		UpdateRawDataPromo(trx)

		err := qSave.Exec(toolkit.M{}.Set("data", trx))
		if err != nil {
			toolkit.Println(err)
		}

		result <- 1
	}
}
<|endoftext|>


<commit_before>package hugolib

import (
	"html/template"
	"path/filepath"
	"testing"

	"github.com/spf13/hugo/source"
	"github.com/spf13/viper"
)

func TestPermalink(t *testing.T) {
	viper.Reset()
	defer viper.Reset()

	tests := []struct {
		file         string
		dir          string
		base         template.URL
		slug         string
		url          string
		uglyURLs     bool
		canonifyURLs bool
		expectedAbs  string
		expectedRel  string
	}{
		{"x/y/z/boofar.md", "x/y/z", "", "", "", false, false, "/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z/", "", "", "", false, false, "/x/y/z/boofar/", "/x/y/z/boofar/"},
		// Issue #1174
		{"x/y/z/boofar.md", "x/y/z", "http://gopher.com/", "", "", false, true, "http://gopher.com/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z/", "http://gopher.com/", "", "", true, true, "http://gopher.com/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "", "boofar", "", false, false, "/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z", "http://barnew/", "", "", false, false, "http://barnew/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/", "boofar", "", false, false, "http://barnew/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z", "", "", "", true, false, "/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "", "", "", true, false, "/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "", "boofar", "", true, false, "/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z", "http://barnew/", "", "", true, false, "http://barnew/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/", "boofar", "", true, false, "http://barnew/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/boo/", "boofar", "", true, false, "http://barnew/boo/x/y/z/boofar.html", "/boo/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/boo/", "boofar", "", true, true, "http://barnew/boo/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/boo", "boofar", "", true, true, "http://barnew/boo/x/y/z/boofar.html", "/x/y/z/boofar.html"},

		// test URL overrides
		{"x/y/z/boofar.md", "x/y/z", "", "", "/z/y/q/", false, false, "/z/y/q/", "/z/y/q/"},
	}

	viper.Set("DefaultExtension", "html")

	for i, test := range tests {
		viper.Set("uglyurls", test.uglyURLs)
		viper.Set("canonifyurls", test.canonifyURLs)
		p := &Page{
			Node: Node{
				URLPath: URLPath{
					Section: "z",
					URL:     test.url,
				},
				Site: &SiteInfo{
					BaseURL: test.base,
				},
			},
			Source: Source{File: *source.NewFile(filepath.FromSlash(test.file))},
		}

		if test.slug != "" {
			p.update(map[string]interface{}{
				"slug": test.slug,
			})
		}

		u, err := p.Permalink()
		if err != nil {
			t.Errorf("Test %d: Unable to process permalink: %s", i, err)
		}

		expected := test.expectedAbs
		if u != expected {
			t.Errorf("Test %d: Expected abs url: %s, got: %s", i, expected, u)
		}

		u, err = p.RelPermalink()
		if err != nil {
			t.Errorf("Test %d: Unable to process permalink: %s", i, err)
		}

		expected = test.expectedRel
		if u != expected {
			t.Errorf("Test %d: Expected rel url: %s, got: %s", i, expected, u)
		}
	}
}
<commit_msg>Add some missing page permalink test cases<commit_after>package hugolib

import (
	"html/template"
	"path/filepath"
	"testing"

	"github.com/spf13/hugo/source"
	"github.com/spf13/viper"
)

func TestPermalink(t *testing.T) {
	viper.Reset()
	defer viper.Reset()

	tests := []struct {
		file         string
		dir          string
		base         template.URL
		slug         string
		url          string
		uglyURLs     bool
		canonifyURLs bool
		expectedAbs  string
		expectedRel  string
	}{
		{"x/y/z/boofar.md", "x/y/z", "", "", "", false, false, "/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z/", "", "", "", false, false, "/x/y/z/boofar/", "/x/y/z/boofar/"},
		// Issue #1174
		{"x/y/z/boofar.md", "x/y/z", "http://gopher.com/", "", "", false, true, "http://gopher.com/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z/", "http://gopher.com/", "", "", true, true, "http://gopher.com/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "", "boofar", "", false, false, "/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z", "http://barnew/", "", "", false, false, "http://barnew/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/", "boofar", "", false, false, "http://barnew/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z", "", "", "", true, false, "/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "", "", "", true, false, "/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "", "boofar", "", true, false, "/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z", "http://barnew/", "", "", true, false, "http://barnew/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/", "boofar", "", true, false, "http://barnew/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/boo/", "boofar", "", true, false, "http://barnew/boo/x/y/z/boofar.html", "/boo/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/boo/", "boofar", "", false, true, "http://barnew/boo/x/y/z/boofar/", "/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/boo/", "boofar", "", false, false, "http://barnew/boo/x/y/z/boofar/", "/boo/x/y/z/boofar/"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/boo/", "boofar", "", true, true, "http://barnew/boo/x/y/z/boofar.html", "/x/y/z/boofar.html"},
		{"x/y/z/boofar.md", "x/y/z/", "http://barnew/boo", "boofar", "", true, true, "http://barnew/boo/x/y/z/boofar.html", "/x/y/z/boofar.html"},

		// test URL overrides
		{"x/y/z/boofar.md", "x/y/z", "", "", "/z/y/q/", false, false, "/z/y/q/", "/z/y/q/"},
	}

	viper.Set("DefaultExtension", "html")

	for i, test := range tests {
		viper.Set("uglyurls", test.uglyURLs)
		viper.Set("canonifyurls", test.canonifyURLs)
		p := &Page{
			Node: Node{
				URLPath: URLPath{
					Section: "z",
					URL:     test.url,
				},
				Site: &SiteInfo{
					BaseURL: test.base,
				},
			},
			Source: Source{File: *source.NewFile(filepath.FromSlash(test.file))},
		}

		if test.slug != "" {
			p.update(map[string]interface{}{
				"slug": test.slug,
			})
		}

		u, err := p.Permalink()
		if err != nil {
			t.Errorf("Test %d: Unable to process permalink: %s", i, err)
		}

		expected := test.expectedAbs
		if u != expected {
			t.Errorf("Test %d: Expected abs url: %s, got: %s", i, expected, u)
		}

		u, err = p.RelPermalink()
		if err != nil {
			t.Errorf("Test %d: Unable to process permalink: %s", i, err)
		}

		expected = test.expectedRel
		if u != expected {
			t.Errorf("Test %d: Expected rel url: %s, got: %s", i, expected, u)
		}
	}
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"os"
	"strings"
)

// DefaultUsername is the username we use if none is defined in config
var DefaultUsername string

// DefaultPassword is the password we use if none is defined in config
var DefaultPassword string

// Config encapsulates configuration loaded from Docker 'config.json' file
type Config struct {
	Auths     map[string]Auth `json:"auths"`
	usernames map[string]string
	passwords map[string]string
}

// Auth contains Docker registry username and password in base64-encoded form
type Auth struct {
	B64Auth string `json:"auth"`
}

// AreDefaultCredentialsDefined tells if default username & password are defined
func AreDefaultCredentialsDefined() bool {
	return DefaultUsername != "" || DefaultPassword != ""
}

// IsEmpty return true if structure has no relevant data inside
func (c *Config) IsEmpty() bool {
	return len(c.Auths) == 0
}

// GetCredentials gets per-registry credentials from loaded Docker config
func (c *Config) GetCredentials(registry string) (string, string, bool) {
	_, defined := c.usernames[registry]
	if !defined {
		return DefaultUsername, DefaultUsername, false
	}

	return c.usernames[registry], c.passwords[registry], true
}

// GetRegistryAuth gets per-registry base64 authentication string
func (c *Config) GetRegistryAuth(registry string) (string, bool) {
	username, password, defined := c.GetCredentials(registry)

	jsonString := fmt.Sprintf("{ \"username\": \"%s\", \"password\": \"%s\" }", username, password)

	return base64.StdEncoding.EncodeToString([]byte(jsonString)), defined
}

// Load loads a Config object from Docker JSON configuration file specified
func Load(fileName string) (*Config, error) {
	fileName = fixPath(fileName)

	f, err := os.Open(fileName)
	defer f.Close()
	if err != nil {
		return nil, err
	}

	c, err := parseConfig(f)
	if err != nil {
		return nil, err
	}

	c.usernames = make(map[string]string)
	c.passwords = make(map[string]string)
	for registry, a := range c.Auths {
		b, err := base64.StdEncoding.DecodeString(a.B64Auth)
		if err != nil {
			return nil, err
		}

		usernameAndPassword := strings.Split(string(b), ":")

		c.usernames[registry] = usernameAndPassword[0]
		c.passwords[registry] = usernameAndPassword[1]
	}

	return c, nil
}

func parseConfig(f *os.File) (*Config, error) {
	c := &Config{}

	err := json.NewDecoder(f).Decode(c)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func fixPath(path string) string {
	return strings.Replace(path, "~", os.Getenv("HOME"), 1)
}
<commit_msg>... porque tienen en su casa, unas uñas de cuchilla ...<commit_after>package config

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"os"
	"strings"
)

// DefaultUsername is the username we use if none is defined in config
var DefaultUsername string

// DefaultPassword is the password we use if none is defined in config
var DefaultPassword string

// Config encapsulates configuration loaded from Docker 'config.json' file
type Config struct {
	Auths     map[string]Auth `json:"auths"`
	usernames map[string]string
	passwords map[string]string
}

// Auth contains Docker registry username and password in base64-encoded form
type Auth struct {
	B64Auth string `json:"auth"`
}

// AreDefaultCredentialsDefined tells if default username & password are defined
func AreDefaultCredentialsDefined() bool {
	return DefaultUsername != "" || DefaultPassword != ""
}

// IsEmpty return true if structure has no relevant data inside
func (c *Config) IsEmpty() bool {
	return len(c.Auths) == 0
}

// GetCredentials gets per-registry credentials from loaded Docker config
func (c *Config) GetCredentials(registry string) (string, string, bool) {
	_, defined := c.usernames[registry]
	if !defined {
		return DefaultUsername, DefaultUsername, false
	}

	return c.usernames[registry], c.passwords[registry], true
}

// GetRegistryAuth gets per-registry base64 authentication string
func (c *Config) GetRegistryAuth(registry string) (string, bool) {
	username, password, defined := c.GetCredentials(registry)

	jsonString := fmt.Sprintf("{ \"username\": \"%s\", \"password\": \"%s\" }", username, password)

	return base64.StdEncoding.EncodeToString([]byte(jsonString)), defined
}

// Load loads a Config object from Docker JSON configuration file specified
func Load(fileName string) (*Config, error) {
	defaultFileNameUsed := fileName == "~/.docker/config.json"

	fileName = fixPath(fileName)

	f, err := os.Open(fileName)
	defer f.Close()
	if err != nil {
		if !defaultFileNameUsed {
			return nil, err
		} else {
			return &Config{}, nil
		}
	}

	c, err := parseConfig(f)
	if err != nil {
		return nil, err
	}

	c.usernames = make(map[string]string)
	c.passwords = make(map[string]string)
	for registry, a := range c.Auths {
		b, err := base64.StdEncoding.DecodeString(a.B64Auth)
		if err != nil {
			return nil, err
		}

		usernameAndPassword := strings.Split(string(b), ":")

		c.usernames[registry] = usernameAndPassword[0]
		c.passwords[registry] = usernameAndPassword[1]
	}

	return c, nil
}

func parseConfig(f *os.File) (*Config, error) {
	c := &Config{}

	err := json.NewDecoder(f).Decode(c)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func fixPath(path string) string {
	return strings.Replace(path, "~", os.Getenv("HOME"), 1)
}
<|endoftext|>


<commit_before>package view

import (
	"github.com/firba1/irq/model"
	"github.com/martini-contrib/render"
	"net/http"
	"strconv"
)

func Top(r render.Render, req *http.Request) {

	qs := req.URL.Query()

	page, err := strconv.Atoi(qs.Get("page"))
	if err != nil {
		page = 1
	}

	count, err := strconv.Atoi(qs.Get("count"))
	if err != nil || count == 0 {
		count = 20
	}

	db, err := model.NewModel("quotes.db")
	if err != nil {
		env := map[string]interface{}{
			"title": "error",
			"error": "db connection failed",
		}
		r.HTML(500, "error", env)
		return
	}

	offset := (page - 1) * count
	quotes, err := db.GetQuotes(model.Query{
        Limit: count,
        Offset: offset,
        OrderBy: []string{"score DESC"},
    })
	if err != nil {
		env := map[string]interface{}{
			"title": "error",
			"error": "failed to get quotes",
		}
		r.HTML(404, "error", env)
		return
	}

	allQuotes, err := db.GetQuotes(model.Query{})
	if err != nil {
		env := map[string]interface{}{
			"title": "error",
			"error": "failed to get quotes",
		}
		r.HTML(404, "error", env)
		return
	}

    total := len(allQuotes)
	maxPage := total / count + 1
	previousPage := page - 1
	nextPage := page + 1
	if nextPage > maxPage {
		nextPage = 0
	}

	env := map[string]interface{}{
		"title":          "Latest",
		"quotes":         quotes,
		"showPagination": true,
		"count":          count,
        "page": page,
		"previousPage":   previousPage,
		"nextPage":       nextPage,
        "total": total,
        "maxPage": maxPage,
	}
	r.HTML(200, "quote", env)
}
<commit_msg>fix title for Top<commit_after>package view

import (
	"github.com/firba1/irq/model"
	"github.com/martini-contrib/render"
	"net/http"
	"strconv"
)

func Top(r render.Render, req *http.Request) {

	qs := req.URL.Query()

	page, err := strconv.Atoi(qs.Get("page"))
	if err != nil {
		page = 1
	}

	count, err := strconv.Atoi(qs.Get("count"))
	if err != nil || count == 0 {
		count = 20
	}

	db, err := model.NewModel("quotes.db")
	if err != nil {
		env := map[string]interface{}{
			"title": "error",
			"error": "db connection failed",
		}
		r.HTML(500, "error", env)
		return
	}

	offset := (page - 1) * count
	quotes, err := db.GetQuotes(model.Query{
        Limit: count,
        Offset: offset,
        OrderBy: []string{"score DESC"},
    })
	if err != nil {
		env := map[string]interface{}{
			"title": "error",
			"error": "failed to get quotes",
		}
		r.HTML(404, "error", env)
		return
	}

	allQuotes, err := db.GetQuotes(model.Query{})
	if err != nil {
		env := map[string]interface{}{
			"title": "error",
			"error": "failed to get quotes",
		}
		r.HTML(404, "error", env)
		return
	}

    total := len(allQuotes)
	maxPage := total / count + 1
	previousPage := page - 1
	nextPage := page + 1
	if nextPage > maxPage {
		nextPage = 0
	}

	env := map[string]interface{}{
		"title":          "Top",
		"quotes":         quotes,
		"showPagination": true,
		"count":          count,
        "page": page,
		"previousPage":   previousPage,
		"nextPage":       nextPage,
        "total": total,
        "maxPage": maxPage,
	}
	r.HTML(200, "quote", env)
}
<|endoftext|>


<commit_before>package dnsstub

import (
	"fmt"
	"github.com/miekg/dns"
	"io"
	"net"
	"strings"
	"testing"
)

func TestRandUint16(t *testing.T) {
	_, err := RandUint16()
	if err != nil {
		t.Errorf("Error getting a random number: %s", err)
	}
}

type NetworkAddr struct {
	Network string
	Addr string
}

type DnsMessageRead struct {
	Message *dns.Msg
	UdpInfo *net.UDPConn
	SrcAddr net.Addr
	TcpInfo *net.TCPConn
	Error error
}

type DnsServer struct {
	Addrs []NetworkAddr
	TCPListeners []*net.TCPListener
	UDPConns []*net.UDPConn
	MsgReader chan *DnsMessageRead
}

func DnsMessageReadUDP(conn *net.UDPConn, msg_chan chan<- *DnsMessageRead) {
	for {
		var result DnsMessageRead
		buffer := make([]byte, 65536, 65536)
		_, src, err := conn.ReadFrom(buffer)
		if err != nil {
			result.Message = nil
			result.UdpInfo = nil
			result.SrcAddr = nil
			result.TcpInfo = nil
			result.Error = err
			msg_chan <- &result
			return
		}
		var dns_msg dns.Msg
		err = dns_msg.Unpack(buffer)
		if err != nil {
			result.Message = nil
			result.UdpInfo = nil
			result.SrcAddr = nil
			result.TcpInfo = nil
			result.Error = err
			msg_chan <- &result
			return
		} else {
			result.Message = &dns_msg
			result.UdpInfo = conn
			result.SrcAddr = src
			result.TcpInfo = nil
			result.Error = nil
			msg_chan <- &result
		}
	}
}

func DnsMessageReadTCP(conn *net.TCPConn, msg_chan chan<- *DnsMessageRead) {
	for {
		var result DnsMessageRead
		msglenbuf := make([]byte, 2, 2)
		_, err := io.ReadFull(conn, msglenbuf)
		if err != nil {
			result.Message = nil
			result.UdpInfo = nil
			result.SrcAddr = conn.RemoteAddr()
			result.TcpInfo = conn
			result.Error = err
			msg_chan <- &result
			return
		}
		msglen := (msglenbuf[0] << 8) | msglenbuf[1]
		buffer := make([]byte, msglen, msglen)
		_, err = io.ReadFull(conn, buffer)
		if err != nil {
			result.Message = nil
			result.UdpInfo = nil
			result.SrcAddr = conn.RemoteAddr()
			result.TcpInfo = conn
			result.Error = err
			msg_chan <- &result
			return
		}
		var dns_msg dns.Msg
		err = dns_msg.Unpack(buffer)
		if err != nil {
			result.Message = nil
			result.UdpInfo = nil
			result.SrcAddr = conn.RemoteAddr()
			result.TcpInfo = conn
			result.Error = err
			msg_chan <- &result
			return
		} else {
			result.Message = &dns_msg
			result.UdpInfo = nil
			result.SrcAddr = conn.RemoteAddr()
			result.TcpInfo = conn
			result.Error = nil
			msg_chan <- &result
		}
	}
}

func DnsMessageListenTCP(listener *net.TCPListener, msg_chan chan<- *DnsMessageRead) {
	conns := make([]*net.TCPConn, 0, 0)
	for {
		// get next TCP connection
		conn, err := listener.AcceptTCP()
		if err != nil {
			// close up shop
			for _, conn := range(conns) {
				conn.Close()
			}
			break
		}
		// Note that we don't ever collect connections until we quit.
		// This would be a problem in a real DNS server, but for our
		// test setup we don't care.
		conns = append(conns, conn)
		// start our reader goroutine
		go DnsMessageReadTCP(conn, msg_chan)
	}
}

func InitDnsServer(hostports []string) (server *DnsServer, err error) {
	addrs := make([]NetworkAddr, 0, 0)
	tcp_listeners := make([]*net.TCPListener, 0, 0)
	udp_conns := make([]*net.UDPConn, 0, 0)
	msg_chan := make(chan *DnsMessageRead, len(hostports) * 4)
	for _, hostport := range(hostports) {
		// if no port is specified, default to port 53
		if !strings.ContainsRune(hostport, ':') {
			hostport = hostport + ":53"
		}
		// set up our UDP listener
		udp_addr, err := net.ResolveUDPAddr("udp", hostport)
		if err != nil {
			goto cleanup_error
		}
		udp_conn, err := net.ListenUDP("udp", udp_addr)
		if err != nil {
			goto cleanup_error
		}
		udp_conns = append(udp_conns, udp_conn)
		new_hostport := udp_conn.LocalAddr().String()
		addrs = append(addrs, NetworkAddr{"udp", new_hostport})
		// start the UDP reader goroutine
		go DnsMessageReadUDP(udp_conn, msg_chan)
		// set up our TCP listener
		tcp_addr, err := net.ResolveTCPAddr("tcp", new_hostport)
		if err != nil {
			goto cleanup_error
		}
		tcp_listener, err := net.ListenTCP("tcp", tcp_addr)
		if err != nil {
			goto cleanup_error
		}
		tcp_listeners = append(tcp_listeners, tcp_listener)
		addrs = append(addrs, NetworkAddr{"tcp", tcp_listener.Addr().String()})
		// start the TCP reader goroutine
		go DnsMessageListenTCP(tcp_listener, msg_chan)
	}
	server = new(DnsServer)
	server.Addrs = addrs
	server.TCPListeners = tcp_listeners
	server.UDPConns = udp_conns
	server.MsgReader = msg_chan
	return server, nil

cleanup_error:
	for _, listener := range(tcp_listeners) {
		listener.Close()
	}
	for _, conn := range(udp_conns) {
		conn.Close()
	}
	return nil, err
}

func (srv *DnsServer) Answer(answers []*dns.Msg) {
	for _, answer := range(answers) {
		fmt.Printf("%s\n", answer)
		msg_read := <-srv.MsgReader
		if msg_read.Error != nil {
			fmt.Printf("Error reading DNS message: %s", msg_read.Error)
			return
		}
		query := msg_read.Message
		answer.Id = query.Id
		buffer, err := answer.Pack()
		if err != nil {
			fmt.Printf("Error packing DNS answer: %s", err)
			return
		}
		if msg_read.UdpInfo != nil {
			_, err = msg_read.UdpInfo.WriteTo(buffer, msg_read.SrcAddr)
			if err != nil {
				fmt.Printf("Error writing DNS answer via UDP: %s", err)
				return
			}
		}
	}
}

func SetupDnsServer() (server *DnsServer, err error) {
	server = nil
	for n := 0; (server == nil) && (n < 10); n++ {
		server, err = InitDnsServer([]string{"[::1]:0",})
		if server != nil {
			break
		}
		// TODO: check for "port already in use"
	}
	return server, err
}

func TestDnsQuery(t *testing.T) {
	server, err := InitDnsServer([]string{"[::1]:0",})
	if err != nil {
		t.Fatalf("Error initializing DNS server: %s", err)
	}
	fmt.Printf("%s\n", server.Addrs)
	var msg dns.Msg
	msg.SetQuestion("hostname.bind.", dns.TypeTXT)
	msg.Question[0].Qclass = dns.ClassCHAOS
	server.Answer([]*dns.Msg{&msg,})
/*
	var msg dns.Msg
	msg.SetQuestion("hostname.bind", dns.TypeTXT)
	msg.Question[0].Qclass = dns.ClassCHAOS
	var info server_info
	server := dns.Server{Addr: "[::1]:0", Net: "udp", Handler: &info}
	go server.ListenAndServe()
	// busy-loop waiting for server to start
	server.lock.Lock()
	for !server.started {
		server.lock.Unlock()
		server.lock.Lock()
	}
//	defer server.lock.Unlock()
//	port := server.Listener
//	DnsQuery("::1",
	server.Shutdown()
	*/
}

<commit_msg>Very basic test works<commit_after>package dnsstub

import (
	"fmt"
	"github.com/miekg/dns"
	"io"
	"net"
	"strings"
	"testing"
)

func TestRandUint16(t *testing.T) {
	_, err := RandUint16()
	if err != nil {
		t.Errorf("Error getting a random number: %s", err)
	}
}

type NetworkAddr struct {
	Network string
	Addr    string
}

type DnsMessageRead struct {
	Message *dns.Msg
	UdpInfo *net.UDPConn
	SrcAddr net.Addr
	TcpInfo *net.TCPConn
	Error   error
}

type DnsServer struct {
	Addrs        []NetworkAddr
	TCPListeners []*net.TCPListener
	UDPConns     []*net.UDPConn
	MsgReader    chan *DnsMessageRead
}

func DnsMessageReadUDP(conn *net.UDPConn, msg_chan chan<- *DnsMessageRead) {
	for {
		var result DnsMessageRead
		buffer := make([]byte, 65536, 65536)
		_, src, err := conn.ReadFrom(buffer)
		if err != nil {
			result.Message = nil
			result.UdpInfo = nil
			result.SrcAddr = nil
			result.TcpInfo = nil
			result.Error = err
			msg_chan <- &result
			return
		}
		var dns_msg dns.Msg
		err = dns_msg.Unpack(buffer)
		if err != nil {
			result.Message = nil
			result.UdpInfo = nil
			result.SrcAddr = nil
			result.TcpInfo = nil
			result.Error = err
			msg_chan <- &result
			return
		} else {
			result.Message = &dns_msg
			result.UdpInfo = conn
			result.SrcAddr = src
			result.TcpInfo = nil
			result.Error = nil
			msg_chan <- &result
		}
	}
}

func DnsMessageReadTCP(conn *net.TCPConn, msg_chan chan<- *DnsMessageRead) {
	for {
		var result DnsMessageRead
		msglenbuf := make([]byte, 2, 2)
		_, err := io.ReadFull(conn, msglenbuf)
		if err != nil {
			result.Message = nil
			result.UdpInfo = nil
			result.SrcAddr = conn.RemoteAddr()
			result.TcpInfo = conn
			result.Error = err
			msg_chan <- &result
			return
		}
		msglen := (msglenbuf[0] << 8) | msglenbuf[1]
		buffer := make([]byte, msglen, msglen)
		_, err = io.ReadFull(conn, buffer)
		if err != nil {
			result.Message = nil
			result.UdpInfo = nil
			result.SrcAddr = conn.RemoteAddr()
			result.TcpInfo = conn
			result.Error = err
			msg_chan <- &result
			return
		}
		var dns_msg dns.Msg
		err = dns_msg.Unpack(buffer)
		if err != nil {
			result.Message = nil
			result.UdpInfo = nil
			result.SrcAddr = conn.RemoteAddr()
			result.TcpInfo = conn
			result.Error = err
			msg_chan <- &result
			return
		} else {
			result.Message = &dns_msg
			result.UdpInfo = nil
			result.SrcAddr = conn.RemoteAddr()
			result.TcpInfo = conn
			result.Error = nil
			msg_chan <- &result
		}
	}
}

func DnsMessageListenTCP(listener *net.TCPListener, msg_chan chan<- *DnsMessageRead) {
	conns := make([]*net.TCPConn, 0, 0)
	for {
		// get next TCP connection
		conn, err := listener.AcceptTCP()
		if err != nil {
			// close up shop
			for _, conn := range conns {
				conn.Close()
			}
			break
		}
		// Note that we don't ever collect connections until we quit.
		// This would be a problem in a real DNS server, but for our
		// test setup we don't care.
		conns = append(conns, conn)
		// start our reader goroutine
		go DnsMessageReadTCP(conn, msg_chan)
	}
}

func InitDnsServer(hostports []string) (server *DnsServer, err error) {
	addrs := make([]NetworkAddr, 0, 0)
	tcp_listeners := make([]*net.TCPListener, 0, 0)
	udp_conns := make([]*net.UDPConn, 0, 0)
	msg_chan := make(chan *DnsMessageRead, len(hostports)*4)
	for _, hostport := range hostports {
		// if no port is specified, default to port 53
		if !strings.ContainsRune(hostport, ':') {
			hostport = hostport + ":53"
		}
		// set up our UDP listener
		udp_addr, err := net.ResolveUDPAddr("udp", hostport)
		if err != nil {
			goto cleanup_error
		}
		udp_conn, err := net.ListenUDP("udp", udp_addr)
		if err != nil {
			goto cleanup_error
		}
		udp_conns = append(udp_conns, udp_conn)
		new_hostport := udp_conn.LocalAddr().String()
		addrs = append(addrs, NetworkAddr{"udp", new_hostport})
		// start the UDP reader goroutine
		go DnsMessageReadUDP(udp_conn, msg_chan)
		// set up our TCP listener
		tcp_addr, err := net.ResolveTCPAddr("tcp", new_hostport)
		if err != nil {
			goto cleanup_error
		}
		tcp_listener, err := net.ListenTCP("tcp", tcp_addr)
		if err != nil {
			goto cleanup_error
		}
		tcp_listeners = append(tcp_listeners, tcp_listener)
		addrs = append(addrs, NetworkAddr{"tcp", tcp_listener.Addr().String()})
		// start the TCP reader goroutine
		go DnsMessageListenTCP(tcp_listener, msg_chan)
	}
	server = new(DnsServer)
	server.Addrs = addrs
	server.TCPListeners = tcp_listeners
	server.UDPConns = udp_conns
	server.MsgReader = msg_chan
	return server, nil

cleanup_error:
	for _, listener := range tcp_listeners {
		listener.Close()
	}
	for _, conn := range udp_conns {
		conn.Close()
	}
	return nil, err
}

func (srv *DnsServer) Answer(answers []*dns.Msg) {
	for _, answer := range answers {
		msg_read := <-srv.MsgReader
		if msg_read.Error != nil {
			fmt.Printf("Error reading DNS message: %s", msg_read.Error)
			return
		}
		query := msg_read.Message
		answer.Id = query.Id
		buffer, err := answer.Pack()
		if err != nil {
			fmt.Printf("Error packing DNS answer: %s", err)
			return
		}
		if msg_read.UdpInfo != nil {
			_, err = msg_read.UdpInfo.WriteTo(buffer, msg_read.SrcAddr)
			if err != nil {
				fmt.Printf("Error writing DNS answer via UDP: %s", err)
				return
			}
		}
	}
}

func SetupDnsServer() (server *DnsServer, err error) {
	server = nil
	for n := 0; (server == nil) && (n < 10); n++ {
		server, err = InitDnsServer([]string{"[::1]:0"})
		if server != nil {
			break
		}
		// TODO: check for "port already in use"
	}
	return server, err
}

func DnsMsgEqual(a *dns.Msg, b *dns.Msg) bool {
	result := true
	if a.MsgHdr != b.MsgHdr {
		result = false
	}
	if len(a.Question) == len(b.Question) {
		for n := range a.Question {
			if a.Question[n] != b.Question[n] {
				result = false
			}
		}
	} else {
		result = false
	}
	if len(a.Answer) == len(b.Answer) {
		for n := range a.Answer {
			if a.Answer[n] != b.Answer[n] {
				result = false
			}
		}
	} else {
		result = false
	}
	if len(a.Ns) == len(b.Ns) {
		for n := range a.Ns {
			if a.Ns[n] != b.Ns[n] {
				result = false
			}
		}
	} else {
		result = false
	}
	if len(a.Extra) == len(b.Extra) {
		for n := range a.Extra {
			if a.Extra[n] != b.Extra[n] {
				result = false
			}
		}
	} else {
		result = false
	}
	return result
}

func TestDnsQuery(t *testing.T) {
	server, err := InitDnsServer([]string{"[::1]:0"})
	if err != nil {
		t.Fatalf("Error initializing DNS server: %s", err)
	}
	var question dns.Msg
	question.SetQuestion("hostname.bind.", dns.TypeTXT)
	question.Question[0].Qclass = dns.ClassCHAOS
	expected_answer := question.Copy()
	go server.Answer([]*dns.Msg{expected_answer})
	answer, _, err := DnsQuery(server.Addrs[0].Addr, &question)
	if err != nil {
		t.Fatalf("Error querying DNS server: %s", err)
	}
	if !DnsMsgEqual(answer, expected_answer) {
		t.Fatalf("Answer not expected answer:\n%s\n%s\n", answer, expected_answer)
	}
}
<|endoftext|>


<commit_before>// Package file provides an implementation of DocumentStore using flat files.
package file

import (
	"fmt"
	"github.com/tummychow/goose/document"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"sync"
	"time"
)

func init() {
	document.RegisterStore("file", func(target *url.URL) (document.DocumentStore, error) {
		if len(target.Host) != 0 {
			return nil, fmt.Errorf("goose/document/file: unexpected URI host %q", target.Host)
		}
		return &FileDocumentStore{root: filepath.Clean(target.Path), mutex: &sync.RWMutex{}}, nil
	})
}

// similar to RFC3339Nano, but with trailing nanosecond zeroes preserved
var fileTimeFormat = "2006-01-02T15:04:05.000000000Z07:00"

// FileDocumentStore is an implementation of DocumentStore, using a standard
// UNIX filesystem. A Document corresponds to a folder on the filesystem, with
// each version corresponding to an individual file under that folder.
//
// FileDocumentStore is registered with the scheme "file". For example, you can
// initialize a new FileDocumentStore via:
//
//     import "github.com/tummychow/goose/document"
//     import _ "github.com/tummychow/goose/document/file"
//     store, err := document.NewStore("file:///var/goose/docs")
//
// This would return a FileDocumentStore using the files under /var/goose/docs.
// FileDocumentStore's URI format takes no options, hosts or user info. The
// target folder must be on the current system, readable and writable to the
// user under which Goose is running.
//
// The path must be absolute. For example, "file://goose/docs" is invalid,
// because "goose" would be interpreted as the host and "/docs" would be the
// path. To avoid this mistake, a nonempty host string in the URI will raise an
// error at instantiation time.
//
// FileDocumentStore is primarily for development. It has poor performance, and
// it can potentially block forever or behave inconsistently because it depends
// on a mutex that is shared across copies. If two separate non-copy instances
// of FileDocumentStore are initialized with the same URI, incorrect behaviors
// could occur.
//
// FileDocumentStore does not support Windows. The characters \/:*?"<>| are
// forbidden in Windows filenames, but most of these are legal in a Document's
// Name, which would create issues when trying to store such a Document on a
// Windows filesystem.
//
// The flat file schema for FileDocumentStore is an implementation detail, and
// modifications to the schema are not considered breaking. Do not rely on the
// schema.
type FileDocumentStore struct {
	// root is the root directory of the FileDocumentStore.
	root string
	// mutex is the global mutex shared between this FileDocumentStore and all
	// its copies.
	mutex *sync.RWMutex
}

func (s *FileDocumentStore) Close() {}

func (s *FileDocumentStore) Copy() (document.DocumentStore, error) {
	return &FileDocumentStore{root: s.root, mutex: s.mutex}, nil
}

func (s *FileDocumentStore) Get(name string) (document.Document, error) {
	s.mutex.RLock()
	defer s.mutex.RUnlock()

	docdir, err := s.readDirFiles(name)
	if err != nil {
		return document.Document{}, err
	}

	return s.readDocument(name, docdir[len(docdir)-1])
}

func (s *FileDocumentStore) GetAll(name string) ([]document.Document, error) {
	s.mutex.RLock()
	defer s.mutex.RUnlock()

	docdir, err := s.readDirFiles(name)
	if err != nil {
		return []document.Document{}, err
	}

	ret := make([]document.Document, 0, len(docdir))
	for i := len(docdir) - 1; i >= 0; i-- {
		doc, err := s.readDocument(name, docdir[i])
		if err != nil {
			return []document.Document{}, err
		}
		ret = append(ret, doc)
	}

	return ret, nil
}

func (s *FileDocumentStore) GetDescendants(ancestor string) ([]string, error) {
	return []string{}, nil
}

func (s *FileDocumentStore) Update(name, content string) error {
	// Update has to check the name before attempting to write the file
	if !document.ValidateName(name) {
		return document.InvalidNameError{name}
	}

	s.mutex.Lock()
	defer s.mutex.Unlock()

	err := os.MkdirAll(filepath.Join(s.root, name), 0755)
	if err != nil {
		return err
	}

	docstamp := time.Now().UTC()
	err = ioutil.WriteFile(filepath.Join(s.root, name, docstamp.Format(fileTimeFormat)), []byte(content), 0644)
	if err != nil {
		return err
	}

	return nil
}

func (s *FileDocumentStore) Clear() error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	contents, err := ioutil.ReadDir(s.root)
	if err != nil {
		return err
	}

	// delete each file or folder in the root, but not the root itself
	for _, target := range contents {
		err = os.RemoveAll(filepath.Join(s.root, target.Name()))
		if err != nil {
			return err
		}
	}

	return nil
}

// readDirFiles returns the sorted list of files for the named Document, from
// oldest to newest. Returns NotFoundError or InvalidNameError as needed.
func (s *FileDocumentStore) readDirFiles(name string) ([]os.FileInfo, error) {
	if !document.ValidateName(name) {
		return []os.FileInfo{}, document.InvalidNameError{name}
	}

	docdir, err := ioutil.ReadDir(filepath.Join(s.root, name))
	if err != nil {
		if pathErr, ok := err.(*os.PathError); ok {
			if pathErr.Err.Error() == "no such file or directory" {
				return []os.FileInfo{}, document.NotFoundError{name}
			}
		}
		return []os.FileInfo{}, err
	}

	ret := make([]os.FileInfo, 0, len(docdir))
	for _, fileinfo := range docdir {
		if fileinfo.IsDir() {
			continue
		}
		ret = append(ret, fileinfo)
	}
	if len(ret) == 0 {
		return []os.FileInfo{}, document.NotFoundError{name}
	}
	return ret, nil
}

// readDocument takes a single file and unmarshals it into a Document. It does
// not perform name validation, since the target file should be obtained from
// readDirFiles (which does the name validation for you).
func (s *FileDocumentStore) readDocument(name string, target os.FileInfo) (document.Document, error) {
	timestamp, err := time.Parse(fileTimeFormat, target.Name())
	if err != nil {
		return document.Document{}, err
	}
	content, err := ioutil.ReadFile(filepath.Join(s.root, name, target.Name()))
	if err != nil {
		if pathErr, ok := err.(*os.PathError); ok {
			if pathErr.Err.Error() == "no such file or directory" {
				return document.Document{}, document.NotFoundError{name}
			}
		}
		return document.Document{}, err
	}
	return document.Document{
		Name:      name,
		Content:   string(content),
		Timestamp: timestamp,
	}, nil
}
<commit_msg>Implement GetDescendants for FileDocumentStore<commit_after>// Package file provides an implementation of DocumentStore using flat files.
package file

import (
	"fmt"
	"github.com/tummychow/goose/document"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"sync"
	"time"
)

func init() {
	document.RegisterStore("file", func(target *url.URL) (document.DocumentStore, error) {
		if len(target.Host) != 0 {
			return nil, fmt.Errorf("goose/document/file: unexpected URI host %q", target.Host)
		}
		return &FileDocumentStore{root: filepath.Clean(target.Path), mutex: &sync.RWMutex{}}, nil
	})
}

// similar to RFC3339Nano, but with trailing nanosecond zeroes preserved
var fileTimeFormat = "2006-01-02T15:04:05.000000000Z07:00"

// FileDocumentStore is an implementation of DocumentStore, using a standard
// UNIX filesystem. A Document corresponds to a folder on the filesystem, with
// each version corresponding to an individual file under that folder.
//
// FileDocumentStore is registered with the scheme "file". For example, you can
// initialize a new FileDocumentStore via:
//
//     import "github.com/tummychow/goose/document"
//     import _ "github.com/tummychow/goose/document/file"
//     store, err := document.NewStore("file:///var/goose/docs")
//
// This would return a FileDocumentStore using the files under /var/goose/docs.
// FileDocumentStore's URI format takes no options, hosts or user info. The
// target folder must be on the current system, readable and writable to the
// user under which Goose is running.
//
// The path must be absolute. For example, "file://goose/docs" is invalid,
// because "goose" would be interpreted as the host and "/docs" would be the
// path. To avoid this mistake, a nonempty host string in the URI will raise an
// error at instantiation time.
//
// FileDocumentStore is primarily for development. It has poor performance, and
// it can potentially block forever or behave inconsistently because it depends
// on a mutex that is shared across copies. If two separate non-copy instances
// of FileDocumentStore are initialized with the same URI, incorrect behaviors
// could occur.
//
// FileDocumentStore does not support Windows. The characters \/:*?"<>| are
// forbidden in Windows filenames, but most of these are legal in a Document's
// Name, which would create issues when trying to store such a Document on a
// Windows filesystem.
//
// The flat file schema for FileDocumentStore is an implementation detail, and
// modifications to the schema are not considered breaking. Do not rely on the
// schema.
type FileDocumentStore struct {
	// root is the root directory of the FileDocumentStore.
	root string
	// mutex is the global mutex shared between this FileDocumentStore and all
	// its copies.
	mutex *sync.RWMutex
}

func (s *FileDocumentStore) Close() {}

func (s *FileDocumentStore) Copy() (document.DocumentStore, error) {
	return &FileDocumentStore{root: s.root, mutex: s.mutex}, nil
}

func (s *FileDocumentStore) Get(name string) (document.Document, error) {
	s.mutex.RLock()
	defer s.mutex.RUnlock()

	docdir, err := s.readDirFiles(name)
	if err != nil {
		return document.Document{}, err
	}

	return s.readDocument(name, docdir[len(docdir)-1])
}

func (s *FileDocumentStore) GetAll(name string) ([]document.Document, error) {
	s.mutex.RLock()
	defer s.mutex.RUnlock()

	docdir, err := s.readDirFiles(name)
	if err != nil {
		return []document.Document{}, err
	}

	ret := make([]document.Document, 0, len(docdir))
	for i := len(docdir) - 1; i >= 0; i-- {
		doc, err := s.readDocument(name, docdir[i])
		if err != nil {
			return []document.Document{}, err
		}
		ret = append(ret, doc)
	}

	return ret, nil
}

func (s *FileDocumentStore) GetDescendants(ancestor string) ([]string, error) {
	if ancestor != "" && !document.ValidateName(ancestor) {
		return []string{}, document.InvalidNameError{ancestor}
	}

	s.mutex.RLock()
	defer s.mutex.RUnlock()

	ret := []string{}

	// Walk traverses depth-first, inspecting files before directories
	// note that this algorithm depends on the traversal order
	err := filepath.Walk(filepath.Join(s.root, ancestor), func(path string, info os.FileInfo, err error) error {
		if err != nil {
			if pathErr, ok := err.(*os.PathError); ok {
				if pathErr.Err.Error() == "no such file or directory" {
					// this means the root of the walk (ie the ancestor
					// directory) did not exist, which is not an error
					// the walk will stop and we will return an empty list of
					// children
					return nil
				}
			}
			return err
		}

		if info.IsDir() {
			return nil
		}

		// this file represents some document; compute that document's name
		thisName, _ := filepath.Split(path)
		thisName = thisName[len(s.root) : len(thisName)-1]

		if ancestor == thisName {
			// this file corresponds to the ancestor document, so it is not of
			// interest to us
			return nil
		}

		// make sure not to add the same name twice (eg we may have already
		// inspected another version of this document)
		if len(ret) == 0 || thisName != ret[len(ret)-1] {
			ret = append(ret, thisName)
		}
		return nil
	})

	return ret, err
}

func (s *FileDocumentStore) Update(name, content string) error {
	// Update has to check the name before attempting to write the file
	if !document.ValidateName(name) {
		return document.InvalidNameError{name}
	}

	s.mutex.Lock()
	defer s.mutex.Unlock()

	err := os.MkdirAll(filepath.Join(s.root, name), 0755)
	if err != nil {
		return err
	}

	docstamp := time.Now().UTC()
	err = ioutil.WriteFile(filepath.Join(s.root, name, docstamp.Format(fileTimeFormat)), []byte(content), 0644)
	if err != nil {
		return err
	}

	return nil
}

func (s *FileDocumentStore) Clear() error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	contents, err := ioutil.ReadDir(s.root)
	if err != nil {
		return err
	}

	// delete each file or folder in the root, but not the root itself
	for _, target := range contents {
		err = os.RemoveAll(filepath.Join(s.root, target.Name()))
		if err != nil {
			return err
		}
	}

	return nil
}

// readDirFiles returns the sorted list of files for the named Document, from
// oldest to newest. Returns NotFoundError or InvalidNameError as needed.
func (s *FileDocumentStore) readDirFiles(name string) ([]os.FileInfo, error) {
	if !document.ValidateName(name) {
		return []os.FileInfo{}, document.InvalidNameError{name}
	}

	docdir, err := ioutil.ReadDir(filepath.Join(s.root, name))
	if err != nil {
		if pathErr, ok := err.(*os.PathError); ok {
			if pathErr.Err.Error() == "no such file or directory" {
				return []os.FileInfo{}, document.NotFoundError{name}
			}
		}
		return []os.FileInfo{}, err
	}

	ret := make([]os.FileInfo, 0, len(docdir))
	for _, fileinfo := range docdir {
		if fileinfo.IsDir() {
			continue
		}
		ret = append(ret, fileinfo)
	}
	if len(ret) == 0 {
		return []os.FileInfo{}, document.NotFoundError{name}
	}
	return ret, nil
}

// readDocument takes a single file and unmarshals it into a Document. It does
// not perform name validation, since the target file should be obtained from
// readDirFiles (which does the name validation for you).
func (s *FileDocumentStore) readDocument(name string, target os.FileInfo) (document.Document, error) {
	timestamp, err := time.Parse(fileTimeFormat, target.Name())
	if err != nil {
		return document.Document{}, err
	}
	content, err := ioutil.ReadFile(filepath.Join(s.root, name, target.Name()))
	if err != nil {
		if pathErr, ok := err.(*os.PathError); ok {
			if pathErr.Err.Error() == "no such file or directory" {
				return document.Document{}, document.NotFoundError{name}
			}
		}
		return document.Document{}, err
	}
	return document.Document{
		Name:      name,
		Content:   string(content),
		Timestamp: timestamp,
	}, nil
}
<|endoftext|>


<commit_before>package keyshare

import (
	"database/sql"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/irmago/internal/common"
)

var ErrUserNotFound = errors.New("Could not find specified user")

type DB struct {
	*sql.DB
}

func (db *DB) ExecCount(query string, args ...interface{}) (int64, error) {
	res, err := db.Exec(query, args...)
	if err != nil {
		return 0, err
	}
	return res.RowsAffected()
}

func (db *DB) ExecUser(query string, args ...interface{}) error {
	c, err := db.ExecCount(query, args...)
	if err != nil {
		return err
	}
	if c == 0 {
		return ErrUserNotFound
	}
	return nil
}

func (db *DB) QueryUser(query string, results []interface{}, args ...interface{}) error {
	res, err := db.Query(query, args...)
	if err != nil {
		return err
	}
	defer common.Close(res)
	if !res.Next() {
		return ErrUserNotFound
	}
	if results == nil {
		return nil
	}
	err = res.Scan(results...)
	if err != nil {
		return err
	}
	return nil
}

func (db *DB) QueryIterate(query string, f func(rows *sql.Rows) error, args ...interface{}) error {
	res, err := db.Query(query, args...)
	if err != nil {
		return err
	}
	defer common.Close(res)

	for res.Next() {
		if err = f(res); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>feat: be more strict in counting affected rows in keyshare database<commit_after>package keyshare

import (
	"database/sql"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/irmago/internal/common"
)

var ErrUserNotFound = errors.New("Could not find specified user")

type DB struct {
	*sql.DB
}

func (db *DB) ExecCount(query string, args ...interface{}) (int64, error) {
	res, err := db.Exec(query, args...)
	if err != nil {
		return 0, err
	}
	return res.RowsAffected()
}

func (db *DB) ExecUser(query string, args ...interface{}) error {
	c, err := db.ExecCount(query, args...)
	if err != nil {
		return err
	}
	if c != 1 {
		return ErrUserNotFound
	}
	return nil
}

func (db *DB) QueryUser(query string, results []interface{}, args ...interface{}) error {
	res, err := db.Query(query, args...)
	if err != nil {
		return err
	}
	defer common.Close(res)
	if !res.Next() {
		return ErrUserNotFound
	}
	if results == nil {
		return nil
	}
	err = res.Scan(results...)
	if err != nil {
		return err
	}
	return nil
}

func (db *DB) QueryIterate(query string, f func(rows *sql.Rows) error, args ...interface{}) error {
	res, err := db.Query(query, args...)
	if err != nil {
		return err
	}
	defer common.Close(res)

	for res.Next() {
		if err = f(res); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package blocks

import (
	"log"
	"time"
)

func Date(b *Block) {

	type dateRule struct {
		FmtString string
		Period    int
	}

	rule := &dateRule{}

	// block until we recieve a rule
	unmarshal(<-b.Routes["set_rule"], &rule)
	log.Println(rule)

	timer := time.NewTimer(time.Duration(1) * time.Second)
	outMsg := make(map[string]interface{})
	d := time.Duration(rule.Period) * time.Second

	for {
		select {
		case t := <-timer.C:
			Set(outMsg, "date", t.Format(rule.FmtString))
			broadcast(b.OutChans, outMsg)
			timer.Reset(d)
		case msg := <-b.AddChan:
			updateOutChans(msg, b)
		case <-b.QuitChan:
			quit(b)
			return
		}
	}
}
<commit_msg>no longer blocking<commit_after>package blocks

import (
    "time"
)

func Date(b *Block) {

    type dateRule struct {
        FmtString string
        Period    int
    }

    var rule *dateRule
    var d time.Duration

    timer := time.NewTimer(time.Duration(1) * time.Second)

    for {
        select {
        case t := <-timer.C:
            if rule == nil {
                break
            }

            outMsg := make(map[string]interface{})
            Set(outMsg, "date", t.Format(rule.FmtString))
            broadcast(b.OutChans, outMsg)
            timer.Reset(d)
        case msg := <- b.Routes["get_rule"]:
            if rule == nil {
                marshal(msg, &dateRule{})
            } else {
                marshal(msg, rule)
            }
        case msg := <- b.Routes["set_rule"]:
            if rule == nil {
                rule = &dateRule{}
            }
            unmarshal(msg, rule)

            d = time.Duration(rule.Period) * time.Second
            timer.Reset(d)

        case msg := <-b.AddChan:
            updateOutChans(msg, b)
        case <-b.QuitChan:
            quit(b)
            return
        }
    }
}
<|endoftext|>