<commit_before>// Copyright (c) Liam Stanley <me@liamstanley.io>. All rights reserved. Use
// of this source code is governed by the MIT license that can be found in
// the LICENSE file.

package girc

import (
	"fmt"
	"log"
	"math/rand"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	"time"
)

// RunHandlers manually runs handlers for a given event.
func (c *Client) RunHandlers(event *Event) {
	if event == nil {
		return
	}

	// Log the event.
	c.debug.Print("< " + StripRaw(event.String()))
	if c.Config.Out != nil {
		if pretty, ok := event.Pretty(); ok {
			fmt.Fprintln(c.Config.Out, StripRaw(pretty))
		}
	}

	// Regular wildcard handlers.
	c.Handlers.exec(ALLEVENTS, c, event.Copy())

	// Then regular handlers.
	c.Handlers.exec(event.Command, c, event.Copy())

	// Check if it's a CTCP.
	if ctcp := decodeCTCP(event.Copy()); ctcp != nil {
		// Execute it.
		c.CTCP.call(c, ctcp)
	}
}

// Handler is lower level implementation of a handler. See
// Caller.AddHandler()
type Handler interface {
	Execute(*Client, Event)
}

// HandlerFunc is a type that represents the function necessary to
// implement Handler.
type HandlerFunc func(client *Client, event Event)

// Execute calls the HandlerFunc with the sender and irc message.
func (f HandlerFunc) Execute(client *Client, event Event) {
	f(client, event)
}

// Caller manages internal and external (user facing) handlers.
type Caller struct {
	// mu is the mutex that should be used when accessing handlers.
	mu sync.RWMutex
	// wg is the waitgroup which is used to execute all handlers concurrently.
	wg sync.WaitGroup

	// external/internal keys are of structure:
	//   map[COMMAND][CUID]Handler
	// Also of note: "COMMAND" should always be uppercase for normalization.

	// external is a map of user facing handlers.
	external map[string]map[string]Handler
	// internal is a map of internally used handlers for the client.
	internal map[string]map[string]Handler
	// debug is the clients logger used for debugging.
	debug *log.Logger
}

// newCaller creates and initializes a new handler.
func newCaller(debugOut *log.Logger) *Caller {
	c := &Caller{
		external: map[string]map[string]Handler{},
		internal: map[string]map[string]Handler{},
		debug:    debugOut,
	}

	return c
}

// Len returns the total amount of user-entered registered handlers.
func (c *Caller) Len() int {
	var total int

	c.mu.RLock()
	for command := range c.external {
		total += len(c.external[command])
	}
	c.mu.RUnlock()

	return total
}

// Count is much like Caller.Len(), however it counts the number of
// registered handlers for a given command.
func (c *Caller) Count(cmd string) int {
	var total int

	cmd = strings.ToUpper(cmd)

	c.mu.RLock()
	for command := range c.external {
		if command == cmd {
			total += len(c.external[command])
		}
	}
	c.mu.RUnlock()

	return total
}

func (c *Caller) String() string {
	var total int

	c.mu.RLock()
	for cmd := range c.internal {
		total += len(c.internal[cmd])
	}
	c.mu.RUnlock()

	return fmt.Sprintf("<Caller external:%d internal:%d>", c.Len(), total)
}

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

// cuid generates a unique UID string for each handler for ease of removal.
func (c *Caller) cuid(cmd string, n int) (cuid, uid string) {
	b := make([]byte, n)

	for i := range b {
		b[i] = letterBytes[rand.Int63()%int64(len(letterBytes))]
	}

	return cmd + ":" + string(b), string(b)
}

// cuidToID allows easy mapping between a generated cuid and the caller
// external/internal handler maps.
func (c *Caller) cuidToID(input string) (cmd, uid string) {
	// Ignore the errors because the strings will default to empty anyway.
	_, _ = fmt.Sscanf(input, "%s:%s", &cmd, &uid)
	return cmd, uid
}

type execStack struct {
	Handler
	cuid string
}

// exec executes all handlers pertaining to specified event. Internal first,
// then external.
//
// Please note that there is no specific order/priority for which the
// handler types themselves or the handlers are executed.
func (c *Caller) exec(command string, client *Client, event *Event) {
	// Build a stack of handlers which can be executed concurrently.
	var stack []execStack

	c.mu.RLock()
	// Get internal handlers first, if Event.Source is defined. If it isn't,
	// it's a user-defined event, or one which exec() shouldn't be handling.
	if event.Source != nil {
		if _, ok := c.internal[command]; ok {
			for cuid := range c.internal[command] {
				stack = append(stack, execStack{c.internal[command][cuid], cuid})
			}
		}
	}

	// Aaand then external handlers.
	if _, ok := c.external[command]; ok {
		for cuid := range c.external[command] {
			stack = append(stack, execStack{c.external[command][cuid], cuid})
		}
	}
	c.mu.RUnlock()

	// Run all handlers concurrently across the same event. This should
	// still help prevent mis-ordered events, while speeding up the
	// execution speed.
	c.wg.Add(len(stack))
	for i := 0; i < len(stack); i++ {
		go func(index int) {
			c.debug.Printf("executing handler %s for event %s", stack[index].cuid, command)
			start := time.Now()

			// If they want to catch any panics, add to defer stack.
			if client.Config.RecoverFunc != nil {
				defer recoverHandlerPanic(client, event, stack[index].cuid, 3)
			}

			stack[index].Execute(client, *event)

			c.debug.Printf("execution of %s took %s", stack[index].cuid, time.Since(start))
			c.wg.Done()
		}(i)
	}

	// Wait for all of the handlers to complete. Not doing this may cause
	// new events from becoming ahead of older handlers.
	c.wg.Wait()
}

// ClearAll clears all external handlers currently setup within the client.
// This ignores internal handlers.
func (c *Caller) ClearAll() {
	c.mu.Lock()
	c.external = map[string]map[string]Handler{}
	c.mu.Unlock()

	c.debug.Print("cleared all external handlers")
}

// clearInternal clears all internal handlers currently setup within the
// client.
func (c *Caller) clearInternal() {
	c.mu.Lock()
	c.internal = map[string]map[string]Handler{}
	c.mu.Unlock()

	c.debug.Print("cleared all internal handlers")
}

// Clear clears all of the handlers for the given event.
// This ignores internal handlers.
func (c *Caller) Clear(cmd string) {
	cmd = strings.ToUpper(cmd)

	c.mu.Lock()
	if _, ok := c.external[cmd]; ok {
		delete(c.external, cmd)
	}
	c.mu.Unlock()

	c.debug.Printf("cleared external handlers for %q", cmd)
}

// Remove removes the handler with cuid from the handler stack. success
// indicates that it existed, and has been removed. If not success, it
// wasn't a registered handler.
func (c *Caller) Remove(cuid string) (success bool) {
	c.mu.Lock()
	success = c.remove(cuid)
	c.mu.Unlock()

	return success
}

// remove is much like Remove, however is NOT concurrency safe. Lock Caller.mu
// on your own.
func (c *Caller) remove(cuid string) (success bool) {
	cmd, uid := c.cuidToID(cuid)
	if len(cmd) == 0 || len(uid) == 0 {
		return false
	}

	// Check if the irc command/event has any handlers on it.
	if _, ok := c.external[cmd]; !ok {
		return false
	}

	// Check to see if it's actually a registered handler.
	if _, ok := c.external[cmd][cuid]; !ok {
		return false
	}

	delete(c.external[cmd], uid)

	c.debug.Printf("removed handler %q", cuid)

	// Assume success.
	return true
}

// sregister is much like Caller.register(), except that it safely locks
// the Caller mutex.
func (c *Caller) sregister(internal bool, cmd string, handler Handler) (cuid string) {
	c.mu.Lock()
	cuid = c.register(internal, cmd, handler)
	c.mu.Unlock()

	return cuid
}

// register will register a handler in the internal tracker. Unsafe (you
// must lock c.mu yourself!)
func (c *Caller) register(internal bool, cmd string, handler Handler) (cuid string) {
	var uid string

	cmd = strings.ToUpper(cmd)

	if internal {
		if _, ok := c.internal[cmd]; !ok {
			c.internal[cmd] = map[string]Handler{}
		}

		cuid, uid = c.cuid(cmd, 20)
		c.internal[cmd][uid] = handler
	} else {
		if _, ok := c.external[cmd]; !ok {
			c.external[cmd] = map[string]Handler{}
		}

		cuid, uid = c.cuid(cmd, 20)
		c.external[cmd][uid] = handler
	}

	c.debug.Printf("registering handler for %q with cuid %q (internal: %t)", cmd, cuid, internal)

	return cuid
}

// AddHandler registers a handler (matching the handler interface) for the
// given event. cuid is the handler uid which can be used to remove the
// handler with Caller.Remove().
func (c *Caller) AddHandler(cmd string, handler Handler) (cuid string) {
	return c.sregister(false, cmd, handler)
}

// Add registers the handler function for the given event. cuid is the
// handler uid which can be used to remove the handler with Caller.Remove().
func (c *Caller) Add(cmd string, handler func(client *Client, event Event)) (cuid string) {
	return c.sregister(false, cmd, HandlerFunc(handler))
}

// AddBg registers the handler function for the given event and executes it
// in a go-routine. cuid is the handler uid which can be used to remove the
// handler with Caller.Remove().
func (c *Caller) AddBg(cmd string, handler func(client *Client, event Event)) (cuid string) {
	return c.sregister(false, cmd, HandlerFunc(func(client *Client, event Event) {
		// Setting up background-based handlers this way allows us to get
		// clean call stacks for use with panic recovery.
		go func() {
			// If they want to catch any panics, add to defer stack.
			if client.Config.RecoverFunc != nil {
				defer recoverHandlerPanic(client, &event, "unknown-goroutine", 3)
			}

			handler(client, event)
		}()
	}))
}

// recoverHandlerPanic is used to catch all handler panics, and re-route
// them if necessary.
func recoverHandlerPanic(client *Client, event *Event, id string, skip int) {
	perr := recover()
	if perr == nil {
		return
	}

	var file string
	var line int
	var ok bool

	_, file, line, ok = runtime.Caller(skip)

	err := &HandlerError{
		Event:  *event,
		ID:     id,
		File:   file,
		Line:   line,
		Panic:  perr,
		Stack:  debug.Stack(),
		callOk: ok,
	}

	client.Config.RecoverFunc(client, err)
	return
}

// HandlerError is the error returned when a panic is intentionally recovered
// from. It contains useful information like the handler identifier (if
// applicable), filename, line in file where panic occurred, the call
// trace, and original event.
type HandlerError struct {
	Event  Event
	ID     string
	File   string
	Line   int
	Panic  interface{}
	Stack  []byte
	callOk bool
}

// Error returns a prettified version of HandlerError, containing ID, file,
// line, and basic error string.
func (e *HandlerError) Error() string {
	if e.callOk {
		return fmt.Sprintf("panic during handler [%s] execution in %s (line %d): %s", e.ID, e.File, e.Line, e.Panic)
	}

	return fmt.Sprintf("panic during handler [%s] execution in unknown: %s", e.ID, e.Panic)
}

// String returns the error that panic returned, as well as the entire call
// trace of where it originated.
func (e *HandlerError) String() string {
	return fmt.Sprintf("panic: %s\n\n%s", e.Panic, string(e.Stack))
}

// DefaultRecoverHandler can be used with Config.RecoverFunc as a default
// catch-all for panics. This will log the error, and the call trace to the
// debug log (see Config.Debug), or os.Stdout if Config.Debug is unset.
func DefaultRecoverHandler(client *Client, err *HandlerError) {
	if client.Config.Debug == nil {
		fmt.Println(err.Error())
		fmt.Println(err.String())
		return
	}

	client.debug.Println(err.Error())
	client.debug.Println(err.String())
}
<commit_msg>fix a bug with cuidToID which wasn't properly splitting the cuid<commit_after>// Copyright (c) Liam Stanley <me@liamstanley.io>. All rights reserved. Use
// of this source code is governed by the MIT license that can be found in
// the LICENSE file.

package girc

import (
	"fmt"
	"log"
	"math/rand"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	"time"
)

// RunHandlers manually runs handlers for a given event.
func (c *Client) RunHandlers(event *Event) {
	if event == nil {
		return
	}

	// Log the event.
	c.debug.Print("< " + StripRaw(event.String()))
	if c.Config.Out != nil {
		if pretty, ok := event.Pretty(); ok {
			fmt.Fprintln(c.Config.Out, StripRaw(pretty))
		}
	}

	// Regular wildcard handlers.
	c.Handlers.exec(ALLEVENTS, c, event.Copy())

	// Then regular handlers.
	c.Handlers.exec(event.Command, c, event.Copy())

	// Check if it's a CTCP.
	if ctcp := decodeCTCP(event.Copy()); ctcp != nil {
		// Execute it.
		c.CTCP.call(c, ctcp)
	}
}

// Handler is lower level implementation of a handler. See
// Caller.AddHandler()
type Handler interface {
	Execute(*Client, Event)
}

// HandlerFunc is a type that represents the function necessary to
// implement Handler.
type HandlerFunc func(client *Client, event Event)

// Execute calls the HandlerFunc with the sender and irc message.
func (f HandlerFunc) Execute(client *Client, event Event) {
	f(client, event)
}

// Caller manages internal and external (user facing) handlers.
type Caller struct {
	// mu is the mutex that should be used when accessing handlers.
	mu sync.RWMutex
	// wg is the waitgroup which is used to execute all handlers concurrently.
	wg sync.WaitGroup

	// external/internal keys are of structure:
	//   map[COMMAND][CUID]Handler
	// Also of note: "COMMAND" should always be uppercase for normalization.

	// external is a map of user facing handlers.
	external map[string]map[string]Handler
	// internal is a map of internally used handlers for the client.
	internal map[string]map[string]Handler
	// debug is the clients logger used for debugging.
	debug *log.Logger
}

// newCaller creates and initializes a new handler.
func newCaller(debugOut *log.Logger) *Caller {
	c := &Caller{
		external: map[string]map[string]Handler{},
		internal: map[string]map[string]Handler{},
		debug:    debugOut,
	}

	return c
}

// Len returns the total amount of user-entered registered handlers.
func (c *Caller) Len() int {
	var total int

	c.mu.RLock()
	for command := range c.external {
		total += len(c.external[command])
	}
	c.mu.RUnlock()

	return total
}

// Count is much like Caller.Len(), however it counts the number of
// registered handlers for a given command.
func (c *Caller) Count(cmd string) int {
	var total int

	cmd = strings.ToUpper(cmd)

	c.mu.RLock()
	for command := range c.external {
		if command == cmd {
			total += len(c.external[command])
		}
	}
	c.mu.RUnlock()

	return total
}

func (c *Caller) String() string {
	var total int

	c.mu.RLock()
	for cmd := range c.internal {
		total += len(c.internal[cmd])
	}
	c.mu.RUnlock()

	return fmt.Sprintf("<Caller external:%d internal:%d>", c.Len(), total)
}

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

// cuid generates a unique UID string for each handler for ease of removal.
func (c *Caller) cuid(cmd string, n int) (cuid, uid string) {
	b := make([]byte, n)

	for i := range b {
		b[i] = letterBytes[rand.Int63()%int64(len(letterBytes))]
	}

	return cmd + ":" + string(b), string(b)
}

// cuidToID allows easy mapping between a generated cuid and the caller
// external/internal handler maps.
func (c *Caller) cuidToID(input string) (cmd, uid string) {
	i := strings.IndexByte(input, 0x3A)
	if i < 0 {
		return "", ""
	}

	return input[:i], input[i+1:]
}

type execStack struct {
	Handler
	cuid string
}

// exec executes all handlers pertaining to specified event. Internal first,
// then external.
//
// Please note that there is no specific order/priority for which the
// handler types themselves or the handlers are executed.
func (c *Caller) exec(command string, client *Client, event *Event) {
	// Build a stack of handlers which can be executed concurrently.
	var stack []execStack

	c.mu.RLock()
	// Get internal handlers first, if Event.Source is defined. If it isn't,
	// it's a user-defined event, or one which exec() shouldn't be handling.
	if event.Source != nil {
		if _, ok := c.internal[command]; ok {
			for cuid := range c.internal[command] {
				stack = append(stack, execStack{c.internal[command][cuid], cuid})
			}
		}
	}

	// Aaand then external handlers.
	if _, ok := c.external[command]; ok {
		for cuid := range c.external[command] {
			stack = append(stack, execStack{c.external[command][cuid], cuid})
		}
	}
	c.mu.RUnlock()

	// Run all handlers concurrently across the same event. This should
	// still help prevent mis-ordered events, while speeding up the
	// execution speed.
	c.wg.Add(len(stack))
	for i := 0; i < len(stack); i++ {
		go func(index int) {
			c.debug.Printf("executing handler %s for event %s", stack[index].cuid, command)
			start := time.Now()

			// If they want to catch any panics, add to defer stack.
			if client.Config.RecoverFunc != nil {
				defer recoverHandlerPanic(client, event, stack[index].cuid, 3)
			}

			stack[index].Execute(client, *event)

			c.debug.Printf("execution of %s took %s", stack[index].cuid, time.Since(start))
			c.wg.Done()
		}(i)
	}

	// Wait for all of the handlers to complete. Not doing this may cause
	// new events from becoming ahead of older handlers.
	c.wg.Wait()
}

// ClearAll clears all external handlers currently setup within the client.
// This ignores internal handlers.
func (c *Caller) ClearAll() {
	c.mu.Lock()
	c.external = map[string]map[string]Handler{}
	c.mu.Unlock()

	c.debug.Print("cleared all external handlers")
}

// clearInternal clears all internal handlers currently setup within the
// client.
func (c *Caller) clearInternal() {
	c.mu.Lock()
	c.internal = map[string]map[string]Handler{}
	c.mu.Unlock()

	c.debug.Print("cleared all internal handlers")
}

// Clear clears all of the handlers for the given event.
// This ignores internal handlers.
func (c *Caller) Clear(cmd string) {
	cmd = strings.ToUpper(cmd)

	c.mu.Lock()
	if _, ok := c.external[cmd]; ok {
		delete(c.external, cmd)
	}
	c.mu.Unlock()

	c.debug.Printf("cleared external handlers for %q", cmd)
}

// Remove removes the handler with cuid from the handler stack. success
// indicates that it existed, and has been removed. If not success, it
// wasn't a registered handler.
func (c *Caller) Remove(cuid string) (success bool) {
	c.mu.Lock()
	success = c.remove(cuid)
	c.mu.Unlock()

	return success
}

// remove is much like Remove, however is NOT concurrency safe. Lock Caller.mu
// on your own.
func (c *Caller) remove(cuid string) (success bool) {
	cmd, uid := c.cuidToID(cuid)
	if len(cmd) == 0 || len(uid) == 0 {
		return false
	}

	// Check if the irc command/event has any handlers on it.
	if _, ok := c.external[cmd]; !ok {
		return false
	}

	// Check to see if it's actually a registered handler.
	if _, ok := c.external[cmd][cuid]; !ok {
		return false
	}

	delete(c.external[cmd], uid)

	c.debug.Printf("removed handler %q", cuid)

	// Assume success.
	return true
}

// sregister is much like Caller.register(), except that it safely locks
// the Caller mutex.
func (c *Caller) sregister(internal bool, cmd string, handler Handler) (cuid string) {
	c.mu.Lock()
	cuid = c.register(internal, cmd, handler)
	c.mu.Unlock()

	return cuid
}

// register will register a handler in the internal tracker. Unsafe (you
// must lock c.mu yourself!)
func (c *Caller) register(internal bool, cmd string, handler Handler) (cuid string) {
	var uid string

	cmd = strings.ToUpper(cmd)

	if internal {
		if _, ok := c.internal[cmd]; !ok {
			c.internal[cmd] = map[string]Handler{}
		}

		cuid, uid = c.cuid(cmd, 20)
		c.internal[cmd][uid] = handler
	} else {
		if _, ok := c.external[cmd]; !ok {
			c.external[cmd] = map[string]Handler{}
		}

		cuid, uid = c.cuid(cmd, 20)
		c.external[cmd][uid] = handler
	}

	c.debug.Printf("registering handler for %q with cuid %q (internal: %t)", cmd, cuid, internal)

	return cuid
}

// AddHandler registers a handler (matching the handler interface) for the
// given event. cuid is the handler uid which can be used to remove the
// handler with Caller.Remove().
func (c *Caller) AddHandler(cmd string, handler Handler) (cuid string) {
	return c.sregister(false, cmd, handler)
}

// Add registers the handler function for the given event. cuid is the
// handler uid which can be used to remove the handler with Caller.Remove().
func (c *Caller) Add(cmd string, handler func(client *Client, event Event)) (cuid string) {
	return c.sregister(false, cmd, HandlerFunc(handler))
}

// AddBg registers the handler function for the given event and executes it
// in a go-routine. cuid is the handler uid which can be used to remove the
// handler with Caller.Remove().
func (c *Caller) AddBg(cmd string, handler func(client *Client, event Event)) (cuid string) {
	return c.sregister(false, cmd, HandlerFunc(func(client *Client, event Event) {
		// Setting up background-based handlers this way allows us to get
		// clean call stacks for use with panic recovery.
		go func() {
			// If they want to catch any panics, add to defer stack.
			if client.Config.RecoverFunc != nil {
				defer recoverHandlerPanic(client, &event, "unknown-goroutine", 3)
			}

			handler(client, event)
		}()
	}))
}

// recoverHandlerPanic is used to catch all handler panics, and re-route
// them if necessary.
func recoverHandlerPanic(client *Client, event *Event, id string, skip int) {
	perr := recover()
	if perr == nil {
		return
	}

	var file string
	var line int
	var ok bool

	_, file, line, ok = runtime.Caller(skip)

	err := &HandlerError{
		Event:  *event,
		ID:     id,
		File:   file,
		Line:   line,
		Panic:  perr,
		Stack:  debug.Stack(),
		callOk: ok,
	}

	client.Config.RecoverFunc(client, err)
	return
}

// HandlerError is the error returned when a panic is intentionally recovered
// from. It contains useful information like the handler identifier (if
// applicable), filename, line in file where panic occurred, the call
// trace, and original event.
type HandlerError struct {
	Event  Event
	ID     string
	File   string
	Line   int
	Panic  interface{}
	Stack  []byte
	callOk bool
}

// Error returns a prettified version of HandlerError, containing ID, file,
// line, and basic error string.
func (e *HandlerError) Error() string {
	if e.callOk {
		return fmt.Sprintf("panic during handler [%s] execution in %s (line %d): %s", e.ID, e.File, e.Line, e.Panic)
	}

	return fmt.Sprintf("panic during handler [%s] execution in unknown: %s", e.ID, e.Panic)
}

// String returns the error that panic returned, as well as the entire call
// trace of where it originated.
func (e *HandlerError) String() string {
	return fmt.Sprintf("panic: %s\n\n%s", e.Panic, string(e.Stack))
}

// DefaultRecoverHandler can be used with Config.RecoverFunc as a default
// catch-all for panics. This will log the error, and the call trace to the
// debug log (see Config.Debug), or os.Stdout if Config.Debug is unset.
func DefaultRecoverHandler(client *Client, err *HandlerError) {
	if client.Config.Debug == nil {
		fmt.Println(err.Error())
		fmt.Println(err.String())
		return
	}

	client.debug.Println(err.Error())
	client.debug.Println(err.String())
}
<|endoftext|>


<commit_before>// Naive implementation of Hashmap data structure.
package ghost

import (
	"errors"
	"fmt"
	"hash/fnv"
)

const (
	initSize  uint32  = 2    // Default number of buckets
	threshold float32 = 0.75 // Threshold load factor to rehash table
)

type node struct {
	Next *node
	Key  string
	Val  string
}

func (n *node) String() string {
	return fmt.Sprintf("<%s, %s>", n.Key, n.Val)
}

type hashMap struct {
	Count   uint32 // Number of elements in hashmap
	Size    uint32 // Number of buckets in hashmap
	buckets []*node
}

func NewHashMap() *hashMap {
	newTable := &hashMap{}
	newTable.buckets = make([]*node, initSize)
	newTable.Size = initSize

	return newTable
}

func (h *hashMap) String() string {
	s := ""

	for i, bucket := range h.buckets {
		tmp := fmt.Sprintf("%d: ", i)
		currentNode := bucket

		for {
			if currentNode == nil {
				break
			}

			tmp += fmt.Sprintf("%s ", currentNode)

			if currentNode.Next == nil {
				break
			}

			currentNode = currentNode.Next
		}

		s += fmt.Sprintln(tmp)
	}

	return s
}

// Set or update key.
func (h *hashMap) Set(key, val string) {
	if h.loadFactor() >= threshold {
		h.rehash()
	}

	index := h.getIndex(key)
	currentNode := h.find(key, h.buckets[index])

	if currentNode == nil {
		h.buckets[index] = &node{h.buckets[index], key, val}
		h.Count++
	} else {
		currentNode.Val = val
	}
}

// Get element from the hashmap.
// Return error if value is not found.
func (h *hashMap) Get(key string) (string, error) {
	tmp := h.find(key, h.buckets[h.getIndex(key)])

	if tmp != nil {
		return tmp.Val, nil
	} else {
		return "", errors.New("No value")
	}
}

// Delete element from the hashmap.
func (h *hashMap) Del(key string) {
	index := h.getIndex(key)
	currentNode := h.buckets[index]
	var prev *node = nil

	for currentNode != nil {
		if currentNode.Key == key {
			if prev == nil {
				h.buckets[index] = currentNode.Next
			} else {
				prev.Next = currentNode.Next
			}

			h.Count--
			return
		}

		prev, currentNode = currentNode, currentNode.Next
	}
}

// Get current load factor.
func (h *hashMap) loadFactor() float32 {
	return float32(h.Count) / float32(h.Size)
}

// Allocate new bigger hashmap and rehash all keys.
func (h *hashMap) rehash() {
	h.Size <<= 1
	newBuckets := make([]*node, h.Size)

	for n := range h.nodes() {
		index := h.getIndex(n.Key)

		if newBuckets[index] == nil {
			newBuckets[index] = &node{nil, n.Key, n.Val}
		} else {
			newBuckets[index] = &node{newBuckets[index], n.Key, n.Val}
		}
	}

	h.buckets = newBuckets
}

// Navigate through all nodes
func (h *hashMap) nodes() <-chan *node {
	ch := make(chan *node)

	go func() {
		for _, bucket := range h.buckets {
			currentNode := bucket

			for {
				if currentNode == nil {
					break
				}

				ch <- currentNode

				currentNode = currentNode.Next
			}
		}
		close(ch)
	}()

	return ch
}

// Find node.
func (h *hashMap) find(key string, node *node) *node {
	for node != nil {
		if node.Key == key {
			return node
		}
		node = node.Next
	}

	return nil
}

// Get index of bucket key belongs to.
func (h *hashMap) getIndex(key string) uint32 {
	hash := fnv.New32()
	hash.Write([]byte(key))
	return hash.Sum32() % h.Size
}
<commit_msg>Remove debug hashmap string functions<commit_after>// Naive implementation of Hashmap data structure.
package ghost

import (
	"errors"
	"hash/fnv"
)

const (
	initSize  uint32  = 2    // Default number of buckets
	threshold float32 = 0.75 // Threshold load factor to rehash table
)

type node struct {
	Next *node
	Key  string
	Val  string
}

type hashMap struct {
	Count   uint32 // Number of elements in hashmap
	Size    uint32 // Number of buckets in hashmap
	buckets []*node
}

func NewHashMap() *hashMap {
	newTable := &hashMap{}
	newTable.buckets = make([]*node, initSize)
	newTable.Size = initSize

	return newTable
}

// Set or update key.
func (h *hashMap) Set(key, val string) {
	if h.loadFactor() >= threshold {
		h.rehash()
	}

	index := h.getIndex(key)
	currentNode := h.find(key, h.buckets[index])

	if currentNode == nil {
		h.buckets[index] = &node{h.buckets[index], key, val}
		h.Count++
	} else {
		currentNode.Val = val
	}
}

// Get element from the hashmap.
// Return error if value is not found.
func (h *hashMap) Get(key string) (string, error) {
	tmp := h.find(key, h.buckets[h.getIndex(key)])

	if tmp != nil {
		return tmp.Val, nil
	} else {
		return "", errors.New("No value")
	}
}

// Delete element from the hashmap.
func (h *hashMap) Del(key string) {
	index := h.getIndex(key)
	currentNode := h.buckets[index]
	var prev *node = nil

	for currentNode != nil {
		if currentNode.Key == key {
			if prev == nil {
				h.buckets[index] = currentNode.Next
			} else {
				prev.Next = currentNode.Next
			}

			h.Count--
			return
		}

		prev, currentNode = currentNode, currentNode.Next
	}
}

// Get current load factor.
func (h *hashMap) loadFactor() float32 {
	return float32(h.Count) / float32(h.Size)
}

// Allocate new bigger hashmap and rehash all keys.
func (h *hashMap) rehash() {
	h.Size <<= 1
	newBuckets := make([]*node, h.Size)

	for n := range h.nodes() {
		index := h.getIndex(n.Key)

		if newBuckets[index] == nil {
			newBuckets[index] = &node{nil, n.Key, n.Val}
		} else {
			newBuckets[index] = &node{newBuckets[index], n.Key, n.Val}
		}
	}

	h.buckets = newBuckets
}

// Navigate through all nodes
func (h *hashMap) nodes() <-chan *node {
	ch := make(chan *node)

	go func() {
		for _, bucket := range h.buckets {
			currentNode := bucket

			for {
				if currentNode == nil {
					break
				}

				ch <- currentNode

				currentNode = currentNode.Next
			}
		}
		close(ch)
	}()

	return ch
}

// Find node.
func (h *hashMap) find(key string, node *node) *node {
	for node != nil {
		if node.Key == key {
			return node
		}
		node = node.Next
	}

	return nil
}

// Get index of bucket key belongs to.
func (h *hashMap) getIndex(key string) uint32 {
	hash := fnv.New32()
	hash.Write([]byte(key))
	return hash.Sum32() % h.Size
}
<|endoftext|>


<commit_before>package hatpear

import (
	"context"
	"errors"
	"net/http"
)

type contextKey int

const (
	errorKey contextKey = iota
)

// Store stores an error into the request's context. It panics if the request
// was not configured by the middleware to store errors.
func Store(r *http.Request, err error) {
	errptr, ok := r.Context().Value(errorKey).(*error)
	if !ok {
		panic("hatpear: request not configured to store errors")
	}
	*errptr = err
}

// Get retrieves an error from the request's context.
func Get(r *http.Request) error {
	errptr, ok := r.Context().Value(errorKey).(*error)
	if !ok {
		return errors.New("hatpear: request not configured to store errors")
	}
	return *errptr
}

// Middleware adds additional functionality to an existing handler.
type Middleware func(http.Handler) http.Handler

// Catch creates middleware that processes errors stored while serving a
// request. Errors are passed to the callback, which should write them to the
// response in an appropriate format.
func Catch(h func(w http.ResponseWriter, r *http.Request, err error)) Middleware {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			var err error
			ctx := context.WithValue(r.Context(), errorKey, &err)

			next.ServeHTTP(w, r.WithContext(ctx))
			if err != nil {
				h(w, r, err)
			}
		})
	}
}

// Handler is a variant on http.Handler that can return an error.
type Handler interface {
	ServeHTTP(w http.ResponseWriter, r *http.Request) error
}

// HandlerFunc is a variant on http.HandlerFunc that can return an error.
type HandlerFunc func(w http.ResponseWriter, r *http.Request) error

func (f HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
	return f(w, r)
}

// Try converts a handler to a standard http.Handler, storing any error in the
// request's context.
func Try(h Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		Store(r, h.ServeHTTP(w, r))
	})
}
<commit_msg>Add Recover middleware function<commit_after>package hatpear

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"runtime"
)

type contextKey int

const (
	errorKey contextKey = iota
)

// Store stores an error into the request's context. It panics if the request
// was not configured by the middleware to store errors.
func Store(r *http.Request, err error) {
	errptr, ok := r.Context().Value(errorKey).(*error)
	if !ok {
		panic("hatpear: request not configured to store errors")
	}
	*errptr = err
}

// Get retrieves an error from the request's context.
func Get(r *http.Request) error {
	errptr, ok := r.Context().Value(errorKey).(*error)
	if !ok {
		return errors.New("hatpear: request not configured to store errors")
	}
	return *errptr
}

// Middleware adds additional functionality to an existing handler.
type Middleware func(http.Handler) http.Handler

// Catch creates middleware that processes errors stored while serving a
// request. Errors are passed to the callback, which should write them to the
// response in an appropriate format.
func Catch(h func(w http.ResponseWriter, r *http.Request, err error)) Middleware {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			var err error
			ctx := context.WithValue(r.Context(), errorKey, &err)

			next.ServeHTTP(w, r.WithContext(ctx))
			if err != nil {
				h(w, r, err)
			}
		})
	}
}

// Handler is a variant on http.Handler that can return an error.
type Handler interface {
	ServeHTTP(w http.ResponseWriter, r *http.Request) error
}

// HandlerFunc is a variant on http.HandlerFunc that can return an error.
type HandlerFunc func(w http.ResponseWriter, r *http.Request) error

func (f HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
	return f(w, r)
}

// Try converts a handler to a standard http.Handler, storing any error in the
// request's context.
func Try(h Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		Store(r, h.ServeHTTP(w, r))
	})
}

var (
	// RecoverStackSize is the max size of a stack for a recovered panic.
	RecoverStackSize = 4096
)

// Recover creates middleware that can recover from a panic in a handler,
// storing a *PanicError for future handling.
func Recover() Middleware {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			defer func() {
				if v := recover(); v != nil {
					stack := make([]byte, RecoverStackSize)
					length := runtime.Stack(stack, false)

					Store(r, &PanicError{
						Value: v,
						Stack: stack[:length],
					})
				}
			}()
			next.ServeHTTP(w, r)
		})
	}
}

// PanicError is an Error created from a recovered panic.
type PanicError struct {
	// Value is the value with which panic() was called
	Value interface{}
	// Stack is the stack trace of the panicing goroutine.
	Stack []byte
}

func (e *PanicError) Error() string {
	if err, ok := e.Value.(error); ok {
		return err.Error()
	}
	return fmt.Sprintf("%v", e.Value)
}
<|endoftext|>


<commit_before>package stack

import (
    "bytes"
    "net/http"
    "net/http/httptest"
)

type Stack struct {
    Handler         http.Handler
    Middleware      []Middleware
    RecoverCallback *RecoverCallback
}

type Response struct {
    buffer *bytes.Buffer
    code   int
    header http.Header
}

func NewStack(handler http.Handler) Stack {
    return Stack{
        Handler: handler,
    }
}

func (s Stack) Use(wares ...Middleware) Stack {
    s.Middleware = append(s.Middleware, wares...)
    return s
}

func (s Stack) ServeHTTP(w http.ResponseWriter, req *http.Request) {
    defer Recover(w, req, nil)
    response := Response{
        buffer: bytes.NewBufferString(""),
        code:   http.StatusOK,
        header: make(http.Header),
    }

    var rec *httptest.ResponseRecorder

    for _, ware := range s.Middleware {
        rec = httptest.NewRecorder()
        halt := !ware.ServeHTTP(rec, req)
        s.Write(&response, rec)
        if halt {
            s.finalize(&response, w)
            return
        }
    }
    rec = httptest.NewRecorder()
    s.Handler.ServeHTTP(rec, req)
    s.Write(&response, rec)
    s.finalize(&response, w)
}

func (s *Stack) Write(response *Response, rec *httptest.ResponseRecorder) {
    for key, values := range rec.Header() {
        for _, value := range values {
            response.header.Add(key, value)
        }
    }
    response.buffer.Write(rec.Body.Bytes())
    response.code = rec.Code
}

func (s *Stack) finalize(response *Response, w http.ResponseWriter) {
    for key, values := range response.header {
        for _, value := range values {
            w.Header().Add(key, value)
        }
    }
    w.WriteHeader(response.code)
    w.Write(response.buffer.Bytes())
}
<commit_msg>Recover mechanism uses the stacks assigned RecoverCallback<commit_after>package stack

import (
    "bytes"
    "net/http"
    "net/http/httptest"
)

type Stack struct {
    Handler         http.Handler
    Middleware      []Middleware
    RecoverCallback *RecoverCallback
}

type Response struct {
    buffer *bytes.Buffer
    code   int
    header http.Header
}

func NewStack(handler http.Handler) Stack {
    return Stack{
        Handler: handler,
    }
}

func (s Stack) Use(wares ...Middleware) Stack {
    s.Middleware = append(s.Middleware, wares...)
    return s
}

func (s Stack) ServeHTTP(w http.ResponseWriter, req *http.Request) {
    defer Recover(w, req, s.RecoverCallback)
    response := Response{
        buffer: bytes.NewBufferString(""),
        code:   http.StatusOK,
        header: make(http.Header),
    }

    var rec *httptest.ResponseRecorder

    for _, ware := range s.Middleware {
        rec = httptest.NewRecorder()
        halt := !ware.ServeHTTP(rec, req)
        s.Write(&response, rec)
        if halt {
            s.finalize(&response, w)
            return
        }
    }
    rec = httptest.NewRecorder()
    s.Handler.ServeHTTP(rec, req)
    s.Write(&response, rec)
    s.finalize(&response, w)
}

func (s *Stack) Write(response *Response, rec *httptest.ResponseRecorder) {
    for key, values := range rec.Header() {
        for _, value := range values {
            response.header.Add(key, value)
        }
    }
    response.buffer.Write(rec.Body.Bytes())
    response.code = rec.Code
}

func (s *Stack) finalize(response *Response, w http.ResponseWriter) {
    for key, values := range response.header {
        for _, value := range values {
            w.Header().Add(key, value)
        }
    }
    w.WriteHeader(response.code)
    w.Write(response.buffer.Bytes())
}
<|endoftext|>


<commit_before>/*
 * Copyright 2015 Xuyuan Pang
 * Author: Pang Xuyuan <xuyuanp # gmail dot com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package hador

import (
	"container/list"
	"encoding/json"
	"net/http"
	"strings"
	"sync"

	"github.com/Xuyuanp/hador/swagger"
)

// Hador struct
type Hador struct {
	Router
	*FilterChain
	Logger Logger
	root   *Node

	ctxPool  sync.Pool
	respPool sync.Pool

	Document *swagger.Document
}

// New creates new Hador instance
func New() *Hador {
	h := &Hador{
		Logger: defaultLogger,
		Document: &swagger.Document{
			Swagger:     "2.0.0",
			Definitions: swagger.Definitions{},
			Tags:        []swagger.Tag{},
			Responses:   swagger.Responses{},
			Parameters:  map[string]swagger.Parameter{},
			Consumes:    []string{},
			Produces:    []string{},
		},
	}
	h.root = NewNode(h, "", 0)
	h.Router = h.root
	h.FilterChain = NewFilterChain(h.Router)

	h.ctxPool.New = func() interface{} {
		return newContext(h.Logger)
	}
	h.respPool.New = func() interface{} {
		return NewResponseWriter(nil)
	}

	return h
}

// Default creates Hador instance with default filters(LogFilter, RecoveryFilter)
func Default() *Hador {
	h := New()
	h.Before(NewLogFilter(h.Logger))
	h.Before(NewRecoveryFilter(h.Logger))
	return h
}

// Run starts serving HTTP request
func (h *Hador) Run(addr string) error {
	h.Logger.Info("Listening on %s", addr)
	return http.ListenAndServe(addr, h)
}

func (h *Hador) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	resp := h.respPool.Get().(*responseWriter)
	resp.reset(w)
	defer h.respPool.Put(resp)

	ctx := h.ctxPool.Get().(*Context)
	ctx.reset(resp, req)
	defer h.ctxPool.Put(ctx)

	h.Serve(ctx)
}

// Serve implements Handler interface
func (h *Hador) Serve(ctx *Context) {
	h.FilterChain.Serve(ctx)
}

func (h *Hador) travel() []*Leaf {
	llist := list.New()
	h.root.travel(llist)

	leaves := make([]*Leaf, llist.Len())
	i := 0
	for e := llist.Front(); e != nil; e = e.Next() {
		leaves[i] = e.Value.(*Leaf)
		i++
	}
	return leaves
}

func (h *Hador) travelPaths() swagger.Paths {
	spaths := make(swagger.Paths)
	leaves := h.travel()
	for _, leaf := range leaves {
		if leaf.DocIgnored {
			continue
		}

		spath, ok := spaths[leaf.Path()]
		if !ok {
			spath = make(swagger.Path)
			spaths[leaf.Path()] = spath
		}

		spath[strings.ToLower(leaf.Method())] = leaf.operation
	}
	return spaths
}

// SwaggerHandler returns swagger json api handler
func (h *Hador) SwaggerHandler() Handler {
	return HandlerFunc(func(ctx *Context) {
		h.Document.Paths = h.travelPaths()
		ctx.Response.Header().Set("Content-Type", "application/json; charset-utf8")
		json.NewEncoder(ctx.Response).Encode(h.Document)
	})
}

// Swagger setups swagger config, returns json API path Leaf
func (h *Hador) Swagger(config swagger.Config) *Leaf {
	// handle API path
	leaf := h.Get(config.APIPath, h.SwaggerHandler()).DocIgnore(!config.SelfDocEnabled)

	// serve swagger-ui file
	if config.UIFilePath != "" {
		s := NewStatic(http.Dir(config.UIFilePath))
		s.Prefix = config.UIPrefix
		h.Before(s)
	}

	return leaf
}

// DocHost sets dochost of document
func (h *Hador) DocHost(host string) *Hador {
	h.Document.Host = host
	return h
}

// DocBasePath sets basepath of document
func (h *Hador) DocBasePath(path string) *Hador {
	h.Document.BasePath = path
	return h
}

// DocDefinition adds model definition
func (h *Hador) DocDefinition(model interface{}) *Hador {
	h.Document.Definitions.AddModelFrom(model)
	return h
}

// DocInfo sets info of document
func (h *Hador) DocInfo(title, description, version, termsOfServeice string) *Hador {
	h.Document.Info.Title = title
	h.Document.Info.Description = description
	h.Document.Info.Version = version
	h.Document.Info.TermsOfService = termsOfServeice
	return h
}

// DocInfoContace sets info contace of document
func (h *Hador) DocInfoContace(name, url, email string) *Hador {
	h.Document.Info.Contact = &swagger.Contact{
		Name:  name,
		URL:   url,
		Email: email,
	}
	return h
}

// DocInfoLicense sets info license of document
func (h *Hador) DocInfoLicense(name, url string) *Hador {
	h.Document.Info.License = &swagger.License{
		Name: name,
		URL:  url,
	}
	return h
}

// DocConsumes sets consumes of document
func (h *Hador) DocConsumes(mimeTypes ...string) *Hador {
	h.Document.Consumes = mimeTypes
	return h
}

// DocProduces sets produces of document
func (h *Hador) DocProduces(mimeTypes ...string) *Hador {
	h.Document.Produces = mimeTypes
	return h
}

// DocTag adds tag to document
func (h *Hador) DocTag(name, description string) *Hador {
	h.Document.Tags = append(h.Document.Tags,
		swagger.Tag{Name: name, Description: description})
	return h
}
<commit_msg>Added AddFilters method<commit_after>/*
 * Copyright 2015 Xuyuan Pang
 * Author: Pang Xuyuan <xuyuanp # gmail dot com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package hador

import (
	"container/list"
	"encoding/json"
	"net/http"
	"strings"
	"sync"

	"github.com/Xuyuanp/hador/swagger"
)

// Hador struct
type Hador struct {
	Router
	*FilterChain
	Logger Logger
	root   *Node

	ctxPool  sync.Pool
	respPool sync.Pool

	Document *swagger.Document
}

// New creates new Hador instance
func New() *Hador {
	h := &Hador{
		Logger: defaultLogger,
		Document: &swagger.Document{
			Swagger:     "2.0.0",
			Definitions: swagger.Definitions{},
			Tags:        []swagger.Tag{},
			Responses:   swagger.Responses{},
			Parameters:  map[string]swagger.Parameter{},
			Consumes:    []string{},
			Produces:    []string{},
		},
	}
	h.root = NewNode(h, "", 0)
	h.Router = h.root
	h.FilterChain = NewFilterChain(h.Router)

	h.ctxPool.New = func() interface{} {
		return newContext(h.Logger)
	}
	h.respPool.New = func() interface{} {
		return NewResponseWriter(nil)
	}

	return h
}

// Default creates Hador instance with default filters(LogFilter, RecoveryFilter)
func Default() *Hador {
	h := New()
	h.Before(NewLogFilter(h.Logger))
	h.Before(NewRecoveryFilter(h.Logger))
	return h
}

// Run starts serving HTTP request
func (h *Hador) Run(addr string) error {
	h.Logger.Info("Listening on %s", addr)
	return http.ListenAndServe(addr, h)
}

func (h *Hador) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	resp := h.respPool.Get().(*responseWriter)
	resp.reset(w)
	defer h.respPool.Put(resp)

	ctx := h.ctxPool.Get().(*Context)
	ctx.reset(resp, req)
	defer h.ctxPool.Put(ctx)

	h.Serve(ctx)
}

// Serve implements Handler interface
func (h *Hador) Serve(ctx *Context) {
	h.FilterChain.Serve(ctx)
}

// AddFilters reuses FilterChain's AddFilters method and returns self
func (h *Hador) AddFilters(filters ...Filter) *Hador {
	h.FilterChain.AddFilters(filters...)
	return h
}

func (h *Hador) travel() []*Leaf {
	llist := list.New()
	h.root.travel(llist)

	leaves := make([]*Leaf, llist.Len())
	i := 0
	for e := llist.Front(); e != nil; e = e.Next() {
		leaves[i] = e.Value.(*Leaf)
		i++
	}
	return leaves
}

func (h *Hador) travelPaths() swagger.Paths {
	spaths := make(swagger.Paths)
	leaves := h.travel()
	for _, leaf := range leaves {
		if leaf.DocIgnored {
			continue
		}

		spath, ok := spaths[leaf.Path()]
		if !ok {
			spath = make(swagger.Path)
			spaths[leaf.Path()] = spath
		}

		spath[strings.ToLower(leaf.Method())] = leaf.operation
	}
	return spaths
}

// SwaggerHandler returns swagger json api handler
func (h *Hador) SwaggerHandler() Handler {
	return HandlerFunc(func(ctx *Context) {
		h.Document.Paths = h.travelPaths()
		ctx.Response.Header().Set("Content-Type", "application/json; charset-utf8")
		json.NewEncoder(ctx.Response).Encode(h.Document)
	})
}

// Swagger setups swagger config, returns json API path Leaf
func (h *Hador) Swagger(config swagger.Config) *Leaf {
	// handle API path
	leaf := h.Get(config.APIPath, h.SwaggerHandler()).DocIgnore(!config.SelfDocEnabled)

	// serve swagger-ui file
	if config.UIFilePath != "" {
		s := NewStatic(http.Dir(config.UIFilePath))
		s.Prefix = config.UIPrefix
		h.Before(s)
	}

	return leaf
}

// DocHost sets dochost of document
func (h *Hador) DocHost(host string) *Hador {
	h.Document.Host = host
	return h
}

// DocBasePath sets basepath of document
func (h *Hador) DocBasePath(path string) *Hador {
	h.Document.BasePath = path
	return h
}

// DocDefinition adds model definition
func (h *Hador) DocDefinition(model interface{}) *Hador {
	h.Document.Definitions.AddModelFrom(model)
	return h
}

// DocInfo sets info of document
func (h *Hador) DocInfo(title, description, version, termsOfServeice string) *Hador {
	h.Document.Info.Title = title
	h.Document.Info.Description = description
	h.Document.Info.Version = version
	h.Document.Info.TermsOfService = termsOfServeice
	return h
}

// DocInfoContace sets info contace of document
func (h *Hador) DocInfoContace(name, url, email string) *Hador {
	h.Document.Info.Contact = &swagger.Contact{
		Name:  name,
		URL:   url,
		Email: email,
	}
	return h
}

// DocInfoLicense sets info license of document
func (h *Hador) DocInfoLicense(name, url string) *Hador {
	h.Document.Info.License = &swagger.License{
		Name: name,
		URL:  url,
	}
	return h
}

// DocConsumes sets consumes of document
func (h *Hador) DocConsumes(mimeTypes ...string) *Hador {
	h.Document.Consumes = mimeTypes
	return h
}

// DocProduces sets produces of document
func (h *Hador) DocProduces(mimeTypes ...string) *Hador {
	h.Document.Produces = mimeTypes
	return h
}

// DocTag adds tag to document
func (h *Hador) DocTag(name, description string) *Hador {
	h.Document.Tags = append(h.Document.Tags,
		swagger.Tag{Name: name, Description: description})
	return h
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

var opt Options

var (
	countDirs  int
	countFiles int
)

func update(f os.FileInfo) {
	if f.IsDir() {
		countDirs++
	} else {
		countFiles++
	}
}

// checkFile checks if we have to add this file.
// Returns true if we have to recurse or false if we don't
func checkFile(file File) bool {
	if opt.excludeEmptyFiles && !file.info.IsDir() && file.info.Size() == 0 {
		return false
	}
	if opt.excludeHiddenFiles && strings.HasPrefix(file.info.Name(), ".") {
		// hidden file or directory
		return false
	}
	if opt.ignoreSymLinks && file.info.Mode()&os.ModeSymlink != 0 {
		return false
	}

	update(file.info)

	// only make hash for files, skip dirs
	if !file.info.IsDir() {
		ComparePartialFile(file)
	}

	if !opt.quiet {
		fmt.Printf("[+] Analyzed: %v directories and %v files\r",
			countDirs, countFiles)
	}

	//fmt.Println(path)

	return true
}

// readDir reads the files from the dir "s" recursively and checks if there are duplicated
func readDir(s string) error {
	files, err := ioutil.ReadDir(s)
	if err != nil {
		return err
	}

	if len(files) == 0 {
		return nil
	}

	for _, f := range files {
		path := s + "/" + f.Name()
		file := File{
			path,
			f,
		}

		recurse := checkFile(file)

		if opt.enableRecursion && recurse && file.info.IsDir() {
			readDir(path)
		}
	}
	return nil
}

// reportDuplicated shows all the information regarding our duplicated files
// if showSummary is true then a summary will printed too
func reportDuplicated(showSummary bool) {
	fmt.Printf("\n\nLISTING DUPLICATED FILES\n")
	fmt.Printf("-------------------------\n")

	for k, v := range DuplicatedFiles {
		fmt.Printf("Listing duplicateds for hash : %x\n\n", k)
		for _, f := range v.list_duplicated {
			fmt.Println(f.path)
		}
		fmt.Printf("-------------------------\n")
	}

	fmt.Println("END OF LIST")
	fmt.Println()

	if showSummary {
		num_dup := 0
		sets := 0
		total_size := int64(0)
		for _, v := range DuplicatedFiles {
			dups := len(v.list_duplicated) - 1
			num_dup += dups
			sets += 1
			for _, f := range v.list_duplicated[1:] {
				total_size += f.info.Size()
			}
		}
		fmt.Printf("%d duplicated files (in %d sets), occupying %d bytes", num_dup, sets, total_size)
	}
}

// Start the program with the current options. Options param is read only
func Start(options Options) {
	opt = options

	if info, err := os.Stat(opt.currentDir); err == nil && !info.IsDir() {
		fmt.Printf("[-] %s is not a valid directory", info.Name())
		return
	}
	fmt.Println("[+] Starting in directory:", opt.currentDir)

	err := readDir(opt.currentDir)
	if err != nil {
		fmt.Println("[-]", err)
	}

	fmt.Printf("\nPartial search done. Deleting false positives\n")

	ValidateDuplicatedFiles()

	reportDuplicated(opt.showSummary)

	fmt.Println()
}
<commit_msg>moved variable to a var block<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

var (
	countDirs  int
	countFiles int
	opt        Options
)

func update(f os.FileInfo) {
	if f.IsDir() {
		countDirs++
	} else {
		countFiles++
	}
}

// checkFile checks if we have to add this file.
// Returns true if we have to recurse or false if we don't
func checkFile(file File) bool {
	if opt.excludeEmptyFiles && !file.info.IsDir() && file.info.Size() == 0 {
		return false
	}
	if opt.excludeHiddenFiles && strings.HasPrefix(file.info.Name(), ".") {
		// hidden file or directory
		return false
	}
	if opt.ignoreSymLinks && file.info.Mode()&os.ModeSymlink != 0 {
		return false
	}

	update(file.info)

	// only make hash for files, skip dirs
	if !file.info.IsDir() {
		ComparePartialFile(file)
	}
	if !opt.quiet {
		fmt.Printf("[+] Analyzed: %v directories and %v files\r",
			countDirs, countFiles)
	}
	return true
}

// readDir reads the files from the dir "s" recursively and checks if there are duplicated
func readDir(s string) error {
	files, err := ioutil.ReadDir(s)
	if err != nil {
		return err
	}
	if len(files) == 0 {
		return nil
	}

	for _, f := range files {
		path := s + "/" + f.Name()
		file := File{
			path,
			f,
		}

		recurse := checkFile(file)

		if opt.enableRecursion && recurse && file.info.IsDir() {
			readDir(path)
		}
	}
	return nil
}

// reportDuplicated shows all the information regarding our duplicated files
// if showSummary is true then a summary will printed too
func reportDuplicated(showSummary bool) {
	fmt.Printf("\n\nLISTING DUPLICATED FILES\n")
	fmt.Printf("-------------------------\n")

	for k, v := range DuplicatedFiles {
		fmt.Printf("Listing duplicateds for hash : %x\n\n", k)
		for _, f := range v.list_duplicated {
			fmt.Println(f.path)
		}
		fmt.Printf("-------------------------\n")
	}

	fmt.Println("END OF LIST")
	fmt.Println()

	if showSummary {
		num_dup := 0
		sets := 0
		total_size := int64(0)
		for _, v := range DuplicatedFiles {
			dups := len(v.list_duplicated) - 1
			num_dup += dups
			sets += 1
			for _, f := range v.list_duplicated[1:] {
				total_size += f.info.Size()
			}
		}
		fmt.Printf("%d duplicated files (in %d sets), occupying %d bytes", num_dup, sets, total_size)
	}
}

// Start the program with the current options. Options param is read only
func Start(options Options) {
	opt = options

	if info, err := os.Stat(opt.currentDir); err == nil && !info.IsDir() {
		fmt.Printf("[-] %s is not a valid directory", info.Name())
		return
	}
	fmt.Println("[+] Starting in directory:", opt.currentDir)

	err := readDir(opt.currentDir)
	if err != nil {
		fmt.Println("[-]", err)
	}

	fmt.Printf("\nPartial search done. Deleting false positives\n")

	ValidateDuplicatedFiles()

	reportDuplicated(opt.showSummary)

	fmt.Println()
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http_test

import (
	"github.com/jacobsa/aws/s3/http"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	sys_http "net/http"
	"net/http/httptest"
	"net/url"
	"testing"
)

func TestConn(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type localHandler struct {
	// Input seen.
	req *sys_http.Request

	// To be returned.
	statusCode int
	body       []byte
}

func (h *localHandler) ServeHTTP(w sys_http.ResponseWriter, r *sys_http.Request) {
	// Record the request.
	if h.req != nil {
		panic("Called twice.")
	}

	h.req = r

	// Write out the response.
	w.WriteHeader(h.statusCode)
	if _, err := w.Write(h.body); err != nil {
		panic(err)
	}
}

type ConnTest struct {
	handler  localHandler
	server   *httptest.Server
	endpoint *url.URL
}

func init() { RegisterTestSuite(&ConnTest{}) }

func (t *ConnTest) SetUp(i *TestInfo) {
	t.server = httptest.NewServer(&t.handler)

	var err error
	t.endpoint, err = url.Parse(t.server.URL)
	AssertEq(nil, err)
}

func (t *ConnTest) TearDown() {
	t.server.Close()
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ConnTest) InvalidScheme() {
	// Connection
	_, err := http.NewConn(&url.URL{Scheme: "taco", Host: "localhost"})

	ExpectThat(err, Error(HasSubstr("scheme")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *ConnTest) UnknownHost() {
	// Connection
	conn, err := http.NewConn(&url.URL{Scheme: "http", Host: "foo.sidofhdksjhf"})
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/foo",
		Headers: map[string]string{},
	}

	// Call
	_, err = conn.SendRequest(req)

	ExpectThat(err, Error(HasSubstr("foo.sidofhdksjhf")))
	ExpectThat(err, Error(HasSubstr("no such host")))
}

func (t *ConnTest) PassesOnRequestInfo() {
	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb: "PUT",
		Path: "/foo/bar",
		Headers: map[string]string{
			"taco":      "burrito",
			"enchilada": "queso",
		},
	}

	// Call
	_, err = conn.SendRequest(req)
	AssertEq(nil, err)

	AssertNe(nil, t.handler.req)
	sysReq := t.handler.req

	ExpectEq("PUT", sysReq.Method)
	ExpectEq("/foo/bar", sysReq.URL.Path)

	ExpectThat(sysReq.Header["Taco"], ElementsAre("burrito"))
	ExpectThat(sysReq.Header["Enchilada"], ElementsAre("queso"))
}

func (t *ConnTest) PathNeedsEscaping() {
	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/타코/bar",
		Headers: map[string]string{},
	}

	// Call
	_, err = conn.SendRequest(req)
	AssertEq(nil, err)

	AssertNe(nil, t.handler.req)
	sysReq := t.handler.req

	ExpectEq("/타코/bar", sysReq.URL.Path)
	ExpectEq("/%ED%83%80%EC%BD%94/bar", sysReq.RequestURI)
}

func (t *ConnTest) ReturnsStatusCode() {
	// Handler
	t.handler.statusCode = 123

	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/",
		Headers: map[string]string{},
	}

	// Call
	resp, err := conn.SendRequest(req)
	AssertEq(nil, err)

	ExpectEq(123, resp.StatusCode)
}

func (t *ConnTest) ReturnsBody() {
	// Handler
	t.handler.body = []byte{0xde, 0xad, 0x00, 0xbe, 0xef}

	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/",
		Headers: map[string]string{},
	}

	// Call
	resp, err := conn.SendRequest(req)
	AssertEq(nil, err)

	ExpectThat(resp.Body, DeepEquals(t.handler.body))
}

func (t *ConnTest) ServerReturnsEmptyBody() {
	// Handler
	t.handler.body = []byte{}

	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/",
		Headers: map[string]string{},
	}

	// Call
	resp, err := conn.SendRequest(req)
	AssertEq(nil, err)

	ExpectThat(resp.Body, ElementsAre())
}

func (t *ConnTest) HttpsAllowed() {
	t.endpoint.Scheme = "https"

	// Connection
	_, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)
}
<commit_msg>ConnTest.PathAndParametersNeedEscaping<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http_test

import (
	"github.com/jacobsa/aws/s3/http"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	sys_http "net/http"
	"net/http/httptest"
	"net/url"
	"testing"
)

func TestConn(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type localHandler struct {
	// Input seen.
	req *sys_http.Request

	// To be returned.
	statusCode int
	body       []byte
}

func (h *localHandler) ServeHTTP(w sys_http.ResponseWriter, r *sys_http.Request) {
	// Record the request.
	if h.req != nil {
		panic("Called twice.")
	}

	h.req = r

	// Write out the response.
	w.WriteHeader(h.statusCode)
	if _, err := w.Write(h.body); err != nil {
		panic(err)
	}
}

type ConnTest struct {
	handler  localHandler
	server   *httptest.Server
	endpoint *url.URL
}

func init() { RegisterTestSuite(&ConnTest{}) }

func (t *ConnTest) SetUp(i *TestInfo) {
	t.server = httptest.NewServer(&t.handler)

	var err error
	t.endpoint, err = url.Parse(t.server.URL)
	AssertEq(nil, err)
}

func (t *ConnTest) TearDown() {
	t.server.Close()
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ConnTest) InvalidScheme() {
	// Connection
	_, err := http.NewConn(&url.URL{Scheme: "taco", Host: "localhost"})

	ExpectThat(err, Error(HasSubstr("scheme")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *ConnTest) UnknownHost() {
	// Connection
	conn, err := http.NewConn(&url.URL{Scheme: "http", Host: "foo.sidofhdksjhf"})
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/foo",
		Headers: map[string]string{},
	}

	// Call
	_, err = conn.SendRequest(req)

	ExpectThat(err, Error(HasSubstr("foo.sidofhdksjhf")))
	ExpectThat(err, Error(HasSubstr("no such host")))
}

func (t *ConnTest) PassesOnRequestInfo() {
	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb: "PUT",
		Path: "/foo/bar",
		Headers: map[string]string{
			"taco":      "burrito",
			"enchilada": "queso",
		},
	}

	// Call
	_, err = conn.SendRequest(req)
	AssertEq(nil, err)

	AssertNe(nil, t.handler.req)
	sysReq := t.handler.req

	ExpectEq("PUT", sysReq.Method)
	ExpectEq("/foo/bar", sysReq.URL.Path)

	ExpectThat(sysReq.Header["Taco"], ElementsAre("burrito"))
	ExpectThat(sysReq.Header["Enchilada"], ElementsAre("queso"))
}

func (t *ConnTest) RequestContainsOneParameter() {
	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/foo/bar",
		Headers: map[string]string{},
		Parameters: map[string]string{
			"baz": "qux",
		},
	}

	// Call
	_, err = conn.SendRequest(req)
	AssertEq(nil, err)

	AssertNe(nil, t.handler.req)
	sysReq := t.handler.req

	ExpectEq("/foo/bar?baz=qux", sysReq.URL.Path)
}

func (t *ConnTest) PathAndParametersNeedEscaping() {
	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/타코/bar",
		Headers: map[string]string{},
		Parameters: map[string]string{
			"b az": "qu?x",
		},
	}

	// Call
	_, err = conn.SendRequest(req)
	AssertEq(nil, err)

	AssertNe(nil, t.handler.req)
	sysReq := t.handler.req

	ExpectEq("/타코/bar?b az=qu x", sysReq.URL.Path)
	ExpectEq("/%ED%83%80%EC%BD%94/bar?b%20az=qu%3Fx", sysReq.RequestURI)
}

func (t *ConnTest) ReturnsStatusCode() {
	// Handler
	t.handler.statusCode = 123

	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/",
		Headers: map[string]string{},
	}

	// Call
	resp, err := conn.SendRequest(req)
	AssertEq(nil, err)

	ExpectEq(123, resp.StatusCode)
}

func (t *ConnTest) ReturnsBody() {
	// Handler
	t.handler.body = []byte{0xde, 0xad, 0x00, 0xbe, 0xef}

	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/",
		Headers: map[string]string{},
	}

	// Call
	resp, err := conn.SendRequest(req)
	AssertEq(nil, err)

	ExpectThat(resp.Body, DeepEquals(t.handler.body))
}

func (t *ConnTest) ServerReturnsEmptyBody() {
	// Handler
	t.handler.body = []byte{}

	// Connection
	conn, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb:    "GET",
		Path:    "/",
		Headers: map[string]string{},
	}

	// Call
	resp, err := conn.SendRequest(req)
	AssertEq(nil, err)

	ExpectThat(resp.Body, ElementsAre())
}

func (t *ConnTest) HttpsAllowed() {
	t.endpoint.Scheme = "https"

	// Connection
	_, err := http.NewConn(t.endpoint)
	AssertEq(nil, err)
}
<|endoftext|>


<commit_before>package s3loader

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sync"
	"time"

	"github.com/motain/s3downloader/cfg"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

// func Download(bucket, prefix, localDir, regPattern string, dryRun bool) error {
func Download(inArgs *cfg.InArgs) error {
	conf, err := cfg.GetCfg()
	if err != nil {
		return err
	}

	creds := credentials.NewStaticCredentials(conf.AWSAccessKeyID, conf.AWSSecretKey, "")
	if _, err := creds.Get(); err != nil {
		return err
	}

	client := s3.New(&aws.Config{Credentials: creds, Region: aws.String(conf.Region)})

	manager := NewS3DownloadManager(client)
	d := NewDownloader(inArgs.Bucket, inArgs.LocalDir, inArgs.Regexp, manager, inArgs.DryRun)

	params := &s3.ListObjectsInput{Bucket: &inArgs.Bucket, Prefix: &inArgs.Prefix}
	if err := client.ListObjectsPages(params, d.eachPage); err != nil {
		return err
	}

	return nil
}

type (
	DownloadManager interface {
		Download(io.WriterAt, *s3.GetObjectInput) (int64, error)
	}

	S3DownloadManager struct {
		*s3manager.Downloader
	}

	Downloader struct {
		dryRun          bool
		downloadManager DownloadManager
		bucket, dir     string
		regexp          *regexp.Regexp
		w               io.Writer
		wg              sync.WaitGroup
		workers         chan int
	}
)

func NewDownloader(
	bucket,
	localDir,
	regPattern string,
	manager *S3DownloadManager,
	dryRun bool,
) *Downloader {
	return &Downloader{
		bucket:          bucket,
		dir:             localDir,
		downloadManager: manager,
		regexp:          regexp.MustCompile(regPattern),
		dryRun:          dryRun,
		workers:         make(chan int, 50),
	}
}

func NewS3DownloadManager(client *s3.S3) *S3DownloadManager {
	downloader := s3manager.NewDownloader(&s3manager.DownloadOptions{
		PartSize:    s3manager.DefaultDownloadPartSize,
		Concurrency: s3manager.DefaultDownloadConcurrency,
		S3:          client,
	})

	return &S3DownloadManager{downloader}
}

func (d *Downloader) eachPage(page *s3.ListObjectsOutput, more bool) bool {
	for _, obj := range page.Contents {
		if !d.regexp.MatchString(*obj.Key) {
			continue
		}

		if d.dryRun {
			d.logInfo(fmt.Sprintf("> Found: s3://%s/%s", d.bucket, *obj.Key))
			continue
		}

		d.workers <- 1
		go func(key string, lastModified *time.Time) {
			d.wg.Add(1)
			if err := d.downloadToFile(key, lastModified); err != nil {
				d.logErr(err)
			}
			<-d.workers
			d.wg.Done()
		}(*obj.Key, obj.LastModified)
	}

	d.wg.Wait()

	return true
}

func (d *Downloader) downloadToFile(key string, lastModified *time.Time) error {
	file := filepath.Join(d.dir, fmt.Sprintf("%s_%s", lastModified.Format(time.RFC3339), filepath.Base(key)))
	if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
		return err
	}

	fd, err := os.Create(file)
	if err != nil {
		return err
	}

	defer fd.Close()
	d.logInfo(fmt.Sprintf("> Downloading s3://%s/%s to %s...\n", d.bucket, key, file))

	params := &s3.GetObjectInput{Bucket: &d.bucket, Key: &key}
	_, err = d.downloadManager.Download(fd, params)
	if err != nil {
		return err
	}

	return nil
}

func (d *Downloader) logInfo(info string) {
	fmt.Println(info)
}

func (d *Downloader) logErr(err error) {
	fmt.Println(err)
}
<commit_msg>Refactor downloader<commit_after>package s3loader

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sync"
	"time"

	"github.com/motain/s3downloader/cfg"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

// func Download(bucket, prefix, localDir, regPattern string, dryRun bool) error {
func Download(inArgs *cfg.InArgs) error {
	conf, err := cfg.GetCfg()
	if err != nil {
		return err
	}

	creds := credentials.NewStaticCredentials(conf.AWSAccessKeyID, conf.AWSSecretKey, "")
	if _, err := creds.Get(); err != nil {
		return err
	}

	client := s3.New(&aws.Config{Credentials: creds, Region: aws.String(conf.Region)})

	manager := NewS3DownloadManager(client)
	d := NewDownloader(inArgs.Bucket, inArgs.LocalDir, inArgs.Regexp, manager, inArgs.DryRun)

	params := &s3.ListObjectsInput{Bucket: &inArgs.Bucket, Prefix: &inArgs.Prefix}
	if err := client.ListObjectsPages(params, d.eachPage()); err != nil {
		return err
	}

	return nil
}

type (
	DownloadManager interface {
		Download(io.WriterAt, *s3.GetObjectInput) (int64, error)
	}

	S3DownloadManager struct {
		*s3manager.Downloader
	}

	Downloader struct {
		dryRun          bool
		downloadManager DownloadManager
		bucket, dir     string
		regexp          *regexp.Regexp
		w               io.Writer
		wg              sync.WaitGroup
		workers         chan int
	}
)

func NewDownloader(
	bucket,
	localDir,
	regPattern string,
	manager *S3DownloadManager,
	dryRun bool,
) *Downloader {
	return &Downloader{
		bucket:          bucket,
		dir:             localDir,
		downloadManager: manager,
		regexp:          regexp.MustCompile(regPattern),
		dryRun:          dryRun,
		workers:         make(chan int, 50),
	}
}

func NewS3DownloadManager(client *s3.S3) *S3DownloadManager {
	downloader := s3manager.NewDownloader(&s3manager.DownloadOptions{
		PartSize:    s3manager.DefaultDownloadPartSize,
		Concurrency: s3manager.DefaultDownloadConcurrency,
		S3:          client,
	})

	return &S3DownloadManager{downloader}
}

type PageIteratorFunc func(*s3.ListObjectsOutput, bool) bool

func (d *Downloader) eachPage() PageIteratorFunc {
	itemHandler := d.onItemDownload

	if d.dryRun {
		itemHandler = d.onItemSearch
	}

	return d.pageIterator(itemHandler)
}

func (d *Downloader) pageIterator(f func(*s3.Object)) PageIteratorFunc {
	return func(page *s3.ListObjectsOutput, more bool) bool {
		for _, obj := range page.Contents {
			if !d.regexp.MatchString(*obj.Key) {
				continue
			}

			d.workers <- 1
			d.wg.Add(1)

			go func(obj *s3.Object) {
				f(obj)

				<-d.workers
				d.wg.Done()
			}(obj)
		}

		d.wg.Wait()

		return true
	}
}

func (d *Downloader) onItemSearch(obj *s3.Object) {
	d.logInfo(fmt.Sprintf("> Found: s3://%s/%s", d.bucket, *obj.Key))
}

func (d *Downloader) onItemDownload(obj *s3.Object) {
	if err := d.downloadToFile(*obj.Key, obj.LastModified); err != nil {
		d.logErr(err)
	}
}

func (d *Downloader) downloadToFile(key string, lastModified *time.Time) error {
	file := filepath.Join(d.dir, fmt.Sprintf("%s_%s", lastModified.Format(time.RFC3339), filepath.Base(key)))
	if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
		return err
	}

	fd, err := os.Create(file)
	if err != nil {
		return err
	}

	defer fd.Close()
	d.logInfo(fmt.Sprintf("> Downloading s3://%s/%s to %s...\n", d.bucket, key, file))

	params := &s3.GetObjectInput{Bucket: &d.bucket, Key: &key}
	_, err = d.downloadManager.Download(fd, params)
	if err != nil {
		return err
	}

	return nil
}

func (d *Downloader) logInfo(info string) {
	fmt.Println(info)
}

func (d *Downloader) logErr(err error) {
	fmt.Println(err)
}
<|endoftext|>


<commit_before>/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"

	"github.ibm.com/riethm/gopherlayer.git/datatypes"
	"github.ibm.com/riethm/gopherlayer.git/helpers/location"
	"github.ibm.com/riethm/gopherlayer.git/helpers/order"
	"github.ibm.com/riethm/gopherlayer.git/services"
	"github.ibm.com/riethm/gopherlayer.git/session"
)

func main() {
	sess := session.New() // default endpoint

	sess.Debug = true

	// Demonstrate order status helpers using a simulated product order receipt

	// First, get any valid order item ID
	items, err := services.GetAccountService(sess).
		Mask("orderItemId").
		GetNextInvoiceTopLevelBillingItems()

	// Create a receipt object to pass to the method
	receipt := datatypes.Container_Product_Order_Receipt{
		PlacedOrder: &datatypes.Billing_Order{
			Items: []datatypes.Billing_Order_Item{
				datatypes.Billing_Order_Item{
					Id: items[0].OrderItemId,
				},
			},
		},
	}

	complete, _, err := order.CheckBillingOrderStatus(sess, &receipt, []string{"COMPLETE", "PENDING"})
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Printf("Order in COMPLETE or PENDING status: %t\n", complete)
	}

	complete, _, err = order.CheckBillingOrderStatus(sess, &receipt, []string{"PENDING", "CANCELLED"})
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Printf("Order in CANCELLED or PENDING status: %t\n", complete)
	}

	complete, _, err = order.CheckBillingOrderComplete(sess, &receipt)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Printf("Order is Complete: %t\n", complete)
	}

	// Demonstrate GetDataCenterByName

	l, err := location.GetDatacenterByName(sess, "ams01")
	fmt.Printf("Found Datacenter: %d\n", *l.Id)
}
<commit_msg>Rename main in examples/examples.go as well<commit_after>/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"

	"github.ibm.com/riethm/gopherlayer.git/datatypes"
	"github.ibm.com/riethm/gopherlayer.git/helpers/location"
	"github.ibm.com/riethm/gopherlayer.git/helpers/order"
	"github.ibm.com/riethm/gopherlayer.git/services"
	"github.ibm.com/riethm/gopherlayer.git/session"
)

func testHelpers() {
	sess := session.New() // default endpoint

	sess.Debug = true

	// Demonstrate order status helpers using a simulated product order receipt

	// First, get any valid order item ID
	items, err := services.GetAccountService(sess).
		Mask("orderItemId").
		GetNextInvoiceTopLevelBillingItems()

	// Create a receipt object to pass to the method
	receipt := datatypes.Container_Product_Order_Receipt{
		PlacedOrder: &datatypes.Billing_Order{
			Items: []datatypes.Billing_Order_Item{
				datatypes.Billing_Order_Item{
					Id: items[0].OrderItemId,
				},
			},
		},
	}

	complete, _, err := order.CheckBillingOrderStatus(sess, &receipt, []string{"COMPLETE", "PENDING"})
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Printf("Order in COMPLETE or PENDING status: %t\n", complete)
	}

	complete, _, err = order.CheckBillingOrderStatus(sess, &receipt, []string{"PENDING", "CANCELLED"})
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Printf("Order in CANCELLED or PENDING status: %t\n", complete)
	}

	complete, _, err = order.CheckBillingOrderComplete(sess, &receipt)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Printf("Order is Complete: %t\n", complete)
	}

	// Demonstrate GetDataCenterByName

	l, err := location.GetDatacenterByName(sess, "ams01")
	fmt.Printf("Found Datacenter: %d\n", *l.Id)
}
<|endoftext|>


<commit_before>package aliyun

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"
)

const Version = "developing..."

const (
	L_QingDao          = "oss-cn-qingdao.aliyuncs.com"
	L_QingDaoInternal  = "oss-cn-qingdao-internal.aliyuncs.com"
	L_HangZhou         = "oss-cn-hangzhou.aliyuncs.com"
	L_HangZhouInternal = "oss-cn-hangzhou-internal.aliyuncs.com"
	L_Beijing          = "oss-cn-beijing.aliyuncs.com"
	L_BeijingInternal  = "oss-cn-beijing-internal.aliyuncs.com"
	ACL_Private        = "private"
	ACL_Public_RDONLY  = "public-read"
	ACL_Public_RDRW    = "public-read-write"
)

var (
	c = []byte(":")
	n = []byte("\n")
)

type ErrorXML struct {
	XMLName xml.Name `xml:"Error"`
	Code    string   `xml:"Code"`
	Message string   `xml:"Message"`
}

type Client struct {
	accessID     string
	accessSecret string
	*http.Client
}

func New(accessID, accessSecret string) *Client {
	return &Client{accessID, accessSecret, new(http.Client)}
}

func (client *Client) PutBucket(bucketName, location, acl string) (bucket *Bucket, err error) {
	err = checkBucketName(bucketName)
	if err != nil {
		return
	}
	header := make(http.Header)
	header.Set("x-oss-acl", acl)
	err = client.do("PUT", bucketName, location, "", header, 200, nil)
	if err != nil {
		return
	}
	bucket = &Bucket{bucketName, location, client}
	return
}

func (client *Client) DeleteBucket(bucketName, location string) (err error) {
	err = checkBucketName(bucketName)
	if err != nil {
		return
	}
	return client.do("DELETE", bucketName, location, "", nil, 204, nil)
}

func (client *Client) do(method, bucket, location, object string, header http.Header, expectedCode int, body io.Reader) (err error) {
	object = strings.Trim(object, "/")
	req, err := http.NewRequest(method, fmt.Sprintf("http://%s.%s/%s", bucket, location, object), body)
	if err != nil {
		return
	}

	if header == nil {
		header = make(http.Header)
	}
	header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
	if _, ok := header["User-Agent"]; !ok {
		header.Set("User-Agent", "oss-golang")
	}
	client.makeSignature(method, bucket, object, header)
	req.Header = header

	resp, err := client.Do(req)
	if err != nil {
		return
	}

	if resp.StatusCode == expectedCode {
		return
	}

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	// fmt.Println(string(b))

	var ex ErrorXML
	if xml.Unmarshal(b, &ex) == nil {
		err = fmt.Errorf("%d %s: %s", resp.StatusCode, ex.Code, ex.Message)
		return
	}
	return fmt.Errorf("%s", resp.Status)
}

func (client *Client) makeSignature(method, bucket, object string, header http.Header) {
	hmac := hmac.New(sha1.New, []byte(client.accessSecret))
	hmac.Write([]byte(method))
	hmac.Write(n)

	hmac.Write([]byte(header.Get("Content-Md5")))
	hmac.Write(n)

	hmac.Write([]byte(header.Get("Content-Type")))
	hmac.Write(n)

	hmac.Write([]byte(header.Get("Date")))
	hmac.Write(n)

	for key, value := range header {
		//todo sort map..
		key = strings.ToLower(key)
		if strings.HasPrefix(key, "x-oss-") {
			hmac.Write([]byte(key))
			hmac.Write(c)
			hmac.Write([]byte(strings.Join(value, ",")))
			hmac.Write(n)
		}
	}

	hmac.Write([]byte(fmt.Sprintf("/%s/%s", bucket, object)))

	header.Set("Authorization", fmt.Sprintf("OSS %s:%s", client.accessID, base64.StdEncoding.EncodeToString(hmac.Sum(nil))))
}

type Bucket struct {
	name     string
	location string
	*Client
}

func (bucket *Bucket) PutObject(objectName, contentType string, content io.ReadSeeker, headers map[string]string) (err error) {
	if content == nil {
		return
	}
	length, err := content.Seek(0, os.SEEK_END)
	if err != nil {
		return
	}
	_, err = content.Seek(0, os.SEEK_SET)
	if err != nil {
		return
	}
	header := make(http.Header)
	header.Set("Content-Type", contentType)
	header.Set("Content-Length", fmt.Sprintf("%d", length))
	for key, val := range headers {
		header.Set(key, val)
	}
	return bucket.do("PUT", bucket.name, bucket.location, objectName, header, 200, content)
}

func (bucket *Bucket) DeleteObject(objectName string) error {
	return bucket.do("DELETE", bucket.name, bucket.location, objectName, nil, 204, nil)
}

func checkBucketName(name string) error {
	nl := len(name)
	if nl < 3 || nl > 63 {
		return fmt.Errorf("400 InvalidBucketName: Invalid bucket name '%s'", name)
	}
	var c byte
	for i := 0; i < nl; i++ {
		if c = name[i]; (c < '0' && c != '-') || (c > '9' && c < 'a') || c > 'z' {
			return fmt.Errorf("400 InvalidBucketName: Invalid bucket name '%s'", name)
		}
	}
	if name[0] == '-' {
		return fmt.Errorf("400 InvalidBucketName: Invalid bucket name '%s'", name)
	}
	return nil
}
<commit_msg>fix bug<commit_after>package aliyun

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"
)

const Version = "developing..."

const (
	L_QingDao          = "oss-cn-qingdao.aliyuncs.com"
	L_QingDaoInternal  = "oss-cn-qingdao-internal.aliyuncs.com"
	L_HangZhou         = "oss-cn-hangzhou.aliyuncs.com"
	L_HangZhouInternal = "oss-cn-hangzhou-internal.aliyuncs.com"
	L_Beijing          = "oss-cn-beijing.aliyuncs.com"
	L_BeijingInternal  = "oss-cn-beijing-internal.aliyuncs.com"
	ACL_Private        = "private"
	ACL_Public_RDONLY  = "public-read"
	ACL_Public_RDRW    = "public-read-write"
)

var (
	c = []byte(":")
	n = []byte("\n")
)

type ErrorXML struct {
	XMLName xml.Name `xml:"Error"`
	Code    string   `xml:"Code"`
	Message string   `xml:"Message"`
}

type Client struct {
	accessID     string
	accessSecret string
	*http.Client
}

func New(accessID, accessSecret string) *Client {
	return &Client{accessID, accessSecret, new(http.Client)}
}

func (client *Client) PutBucket(bucketName, location, acl string) (bucket *Bucket, err error) {
	err = checkBucketName(bucketName)
	if err != nil {
		return
	}
	header := make(http.Header)
	header.Set("x-oss-acl", acl)
	err = client.do("PUT", bucketName, location, "", header, 200, nil)
	if err != nil {
		return
	}
	bucket = &Bucket{bucketName, location, client}
	return
}

func (client *Client) DeleteBucket(bucketName, location string) (err error) {
	err = checkBucketName(bucketName)
	if err != nil {
		return
	}
	return client.do("DELETE", bucketName, location, "", nil, 204, nil)
}

func (client *Client) do(method, bucket, location, object string, header http.Header, expectedCode int, body io.Reader) (err error) {
	object = strings.Trim(object, "/")
	req, err := http.NewRequest(method, fmt.Sprintf("http://%s.%s/%s", bucket, location, object), body)
	if err != nil {
		return
	}

	if header == nil {
		header = make(http.Header)
	}
	header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
	if _, ok := header["User-Agent"]; !ok {
		header.Set("User-Agent", "oss-golang")
	}
	client.makeSignature(method, bucket, object, header)
	req.Header = header

	resp, err := client.Do(req)
	if err != nil {
		return
	}

	if resp.StatusCode == expectedCode {
		return
	}

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	// fmt.Println(string(b))

	var ex ErrorXML
	if xml.Unmarshal(b, &ex) == nil {
		err = fmt.Errorf("%d %s: %s", resp.StatusCode, ex.Code, ex.Message)
		return
	}
	return fmt.Errorf("%s", resp.Status)
}

func (client *Client) makeSignature(method, bucket, object string, header http.Header) {
	hmac := hmac.New(sha1.New, []byte(client.accessSecret))
	hmac.Write([]byte(method))
	hmac.Write(n)

	hmac.Write([]byte(header.Get("Content-Md5")))
	hmac.Write(n)

	hmac.Write([]byte(header.Get("Content-Type")))
	hmac.Write(n)

	hmac.Write([]byte(header.Get("Date")))
	hmac.Write(n)

	for key, value := range header {
		//todo sort map..
		key = strings.ToLower(key)
		if strings.HasPrefix(key, "x-oss-") {
			hmac.Write([]byte(key))
			hmac.Write(c)
			hmac.Write([]byte(strings.Join(value, ",")))
			hmac.Write(n)
		}
	}

	hmac.Write([]byte(fmt.Sprintf("/%s/%s", bucket, object)))

	header.Set("Authorization", fmt.Sprintf("OSS %s:%s", client.accessID, base64.StdEncoding.EncodeToString(hmac.Sum(nil))))
}

type Bucket struct {
	Name     string
	Location string
	*Client
}

func (bucket *Bucket) PutObject(objectName, contentType string, content io.ReadSeeker, headers map[string]string) (err error) {
	if content == nil {
		return
	}
	length, err := content.Seek(0, os.SEEK_END)
	if err != nil {
		return
	}
	_, err = content.Seek(0, os.SEEK_SET)
	if err != nil {
		return
	}
	header := make(http.Header)
	header.Set("Content-Type", contentType)
	header.Set("Content-Length", fmt.Sprintf("%d", length))
	for key, val := range headers {
		header.Set(key, val)
	}
	return bucket.do("PUT", bucket.Name, bucket.Location, objectName, header, 200, content)
}

func (bucket *Bucket) DeleteObject(objectName string) error {
	return bucket.do("DELETE", bucket.Name, bucket.Location, objectName, nil, 204, nil)
}

func checkBucketName(name string) error {
	nl := len(name)
	if nl < 3 || nl > 63 {
		return fmt.Errorf("400 InvalidBucketName: Invalid bucket name '%s'", name)
	}
	var c byte
	for i := 0; i < nl; i++ {
		if c = name[i]; (c < '0' && c != '-') || (c > '9' && c < 'a') || c > 'z' {
			return fmt.Errorf("400 InvalidBucketName: Invalid bucket name '%s'", name)
		}
	}
	if name[0] == '-' {
		return fmt.Errorf("400 InvalidBucketName: Invalid bucket name '%s'", name)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Vasco Costa. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package owm implements a wrapper for the OpenWeatherMap API.
//
// Read more about the OpenWeatherMap API here: http://openweathermap.org/api.
package owm

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strconv"
)

type weatherLine struct {
	Id          int
	Main        string
	Description string
	Icon        string
}

// Weather represents the current weather at a specific location.
//
// It is returned by the WeatherBy* methods of Client.
type Weather struct {
	Coord struct {
		Lat float64
		Lon float64
	}
	Sys struct {
		Type    int
		Id      int
		Country string
		Sunrise int
		Sunset  int
	}
	Weather []weatherLine
	Base    string
	Main    struct {
		Temp      float64
		Humidity  int
		TempMin   float64 `json:"temp_min"`
		TempMax   float64 `json:"temp_max"`
		Pressure  float64
		SeaLevel  float64 `json:"sea_level"`
		GrndLevel float64 `json:"grnd_level"`
	}
	Wind struct {
		Speed float64
		Deg   float64
		Gust  float64
	}
	Clouds struct {
		All int
	}
	Dt   int
	Id   int
	Name string
	Cod  int
}

type weatherSet struct {
	Cnt     int
	Weather []Weather `json:"list"`
}

// Forecast represents the weather forecast for a specific location.
//
// It is returned by the ForecastBy* methods of Client.
type Forecast struct {
	Cod  string
	City struct {
		Id    int
		Name  string
		Coord struct {
			Lon float64
			Lat float64
		}
		Country string
	}
	Cnt     int
	Weather []Weather `json:"list"`
}

// Client represents an OpenWeatherMap API client.
type Client struct {
	key     string
	baseURL string
}

// NewClient returns a new Client given an API key.
//
// Pass an empty string as the key argument to use the client without an API key.
func NewClient(key string) *Client {
	return &Client{key, "http://api.openweathermap.org/data/2.5/"}
}

func (c *Client) data(url string) (data []byte, err error) {
	res, err := http.Get(url)
	if err != nil {
		err = errors.New("owm: error while getting weather data")
		return
	}
	data, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		err = errors.New("owm: error while getting weather data")
		return
	}
	return
}

func (c *Client) weather(url string) (w Weather, err error) {
	if c.key != "" {
		url += "&APPID=" + c.key
	}
	data, err := c.data(url)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	err = json.Unmarshal(data, &w)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	return
}

// WeatherByName decodes the current weather given the city name and the units.
func (c *Client) WeatherByName(name string, units string) (w Weather, err error) {
	w, err = c.weather(c.baseURL +
		"weather" +
		"?q=" + name +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	return
}

// WeatherById decodes the current weather given the city id and the units.
func (c *Client) WeatherById(id int, units string) (w Weather, err error) {
	w, err = c.weather(c.baseURL +
		"weather" +
		"?id=" + strconv.Itoa(id) +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	return
}

// WeatherByCoord decodes the current weather given the city coordinates and the units.
func (c *Client) WeatherByCoord(lat float64, lon float64, units string) (w Weather, err error) {
	w, err = c.weather(c.baseURL +
		"weather" +
		"?lat=" + strconv.FormatFloat(lat, 'f', 2, 64) +
		"&lon=" + strconv.FormatFloat(lon, 'f', 2, 64) +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	return
}

func (c *Client) weatherSet(url string) (ws weatherSet, err error) {
	if c.key != "" {
		url += "&APPID=" + c.key
	}
	data, err := c.data(url)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	err = json.Unmarshal(data, &ws)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	return
}

// WeatherByZone decodes a slice of current weathers given the zone coordinates, map zoom and the units.
func (c *Client) WeatherByZone(lat1, lon1, lat2, lon2 float64, zoom int, units string) (w []Weather, err error) {
	ws, err := c.weatherSet(c.baseURL +
		"box/city" +
		"?bbox=" +
		strconv.FormatFloat(lat1, 'f', 2, 64) + "," +
		strconv.FormatFloat(lon1, 'f', 2, 64) + "," +
		strconv.FormatFloat(lat2, 'f', 2, 64) + "," +
		strconv.FormatFloat(lon2, 'f', 2, 64) + "," +
		strconv.Itoa(zoom) +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	w = ws.Weather
	return
}

// WeatherByRadius decodes a slice of current weathers given the center coordinates, radius and the units.
func (c *Client) WeatherByRadius(lat, lon, radius float64, units string) (w []Weather, err error) {
	ws, err := c.weatherSet(c.baseURL +
		"find" +
		"?lat=" + strconv.FormatFloat(lat, 'f', 2, 64) + "," +
		"&lon=" + strconv.FormatFloat(lon, 'f', 2, 64) + "," +
		"&cnt=" + strconv.FormatFloat(radius, 'f', 2, 64) +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	w = ws.Weather
	return
}

// WeatherByIds decodes a slice of current weathers given a slice of city ids and the units.
func (c *Client) WeatherByIds(id []int, units string) (w []Weather, err error) {
	var ids string
	for i := range id {
		ids += strconv.Itoa(id[i]) + ","
	}
	ws, err := c.weatherSet(c.baseURL +
		"group" +
		"?id=" + ids[:len(ids)-1] +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	w = ws.Weather
	return
}

func (c *Client) forecast(url string) (f Forecast, err error) {
	if c.key != "" {
		url += "&APPID=" + c.key
	}
	data, err := c.data(url)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	err = json.Unmarshal(data, &f)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	return
}

// ForecastByName decodes the current forecast given the city name and the units.
func (c *Client) ForecastByName(name string, units string) (f Forecast, err error) {
	f, err = c.forecast(c.baseURL +
		"forecast" +
		"?q=" + name +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	return
}
<commit_msg>added method ForecastById<commit_after>// Copyright 2014 Vasco Costa. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package owm implements a wrapper for the OpenWeatherMap API.
//
// Read more about the OpenWeatherMap API here: http://openweathermap.org/api.
package owm

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strconv"
)

type weatherLine struct {
	Id          int
	Main        string
	Description string
	Icon        string
}

// Weather represents the current weather at a specific location.
//
// It is returned by the WeatherBy* methods of Client.
type Weather struct {
	Coord struct {
		Lat float64
		Lon float64
	}
	Sys struct {
		Type    int
		Id      int
		Country string
		Sunrise int
		Sunset  int
	}
	Weather []weatherLine
	Base    string
	Main    struct {
		Temp      float64
		Humidity  int
		TempMin   float64 `json:"temp_min"`
		TempMax   float64 `json:"temp_max"`
		Pressure  float64
		SeaLevel  float64 `json:"sea_level"`
		GrndLevel float64 `json:"grnd_level"`
	}
	Wind struct {
		Speed float64
		Deg   float64
		Gust  float64
	}
	Clouds struct {
		All int
	}
	Dt   int
	Id   int
	Name string
	Cod  int
}

type weatherSet struct {
	Cnt     int
	Weather []Weather `json:"list"`
}

// Forecast represents the weather forecast for a specific location.
//
// It is returned by the ForecastBy* methods of Client.
type Forecast struct {
	Cod  string
	City struct {
		Id    int
		Name  string
		Coord struct {
			Lon float64
			Lat float64
		}
		Country string
	}
	Cnt     int
	Weather []Weather `json:"list"`
}

// Client represents an OpenWeatherMap API client.
type Client struct {
	key     string
	baseURL string
}

// NewClient returns a new Client given an API key.
//
// Pass an empty string as the key argument to use the client without an API key.
func NewClient(key string) *Client {
	return &Client{key, "http://api.openweathermap.org/data/2.5/"}
}

func (c *Client) data(url string) (data []byte, err error) {
	res, err := http.Get(url)
	if err != nil {
		err = errors.New("owm: error while getting weather data")
		return
	}
	data, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		err = errors.New("owm: error while getting weather data")
		return
	}
	return
}

func (c *Client) weather(url string) (w Weather, err error) {
	if c.key != "" {
		url += "&APPID=" + c.key
	}
	data, err := c.data(url)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	err = json.Unmarshal(data, &w)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	return
}

// WeatherByName decodes the current weather given the city name and the units.
func (c *Client) WeatherByName(name string, units string) (w Weather, err error) {
	w, err = c.weather(c.baseURL +
		"weather" +
		"?q=" + name +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	return
}

// WeatherById decodes the current weather given the city id and the units.
func (c *Client) WeatherById(id int, units string) (w Weather, err error) {
	w, err = c.weather(c.baseURL +
		"weather" +
		"?id=" + strconv.Itoa(id) +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	return
}

// WeatherByCoord decodes the current weather given the city coordinates and the units.
func (c *Client) WeatherByCoord(lat float64, lon float64, units string) (w Weather, err error) {
	w, err = c.weather(c.baseURL +
		"weather" +
		"?lat=" + strconv.FormatFloat(lat, 'f', 2, 64) +
		"&lon=" + strconv.FormatFloat(lon, 'f', 2, 64) +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	return
}

func (c *Client) weatherSet(url string) (ws weatherSet, err error) {
	if c.key != "" {
		url += "&APPID=" + c.key
	}
	data, err := c.data(url)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	err = json.Unmarshal(data, &ws)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	return
}

// WeatherByZone decodes a slice of current weathers given the zone coordinates, map zoom and the units.
func (c *Client) WeatherByZone(lat1, lon1, lat2, lon2 float64, zoom int, units string) (w []Weather, err error) {
	ws, err := c.weatherSet(c.baseURL +
		"box/city" +
		"?bbox=" +
		strconv.FormatFloat(lat1, 'f', 2, 64) + "," +
		strconv.FormatFloat(lon1, 'f', 2, 64) + "," +
		strconv.FormatFloat(lat2, 'f', 2, 64) + "," +
		strconv.FormatFloat(lon2, 'f', 2, 64) + "," +
		strconv.Itoa(zoom) +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	w = ws.Weather
	return
}

// WeatherByRadius decodes a slice of current weathers given the center coordinates, radius and the units.
func (c *Client) WeatherByRadius(lat, lon, radius float64, units string) (w []Weather, err error) {
	ws, err := c.weatherSet(c.baseURL +
		"find" +
		"?lat=" + strconv.FormatFloat(lat, 'f', 2, 64) + "," +
		"&lon=" + strconv.FormatFloat(lon, 'f', 2, 64) + "," +
		"&cnt=" + strconv.FormatFloat(radius, 'f', 2, 64) +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	w = ws.Weather
	return
}

// WeatherByIds decodes a slice of current weathers given a slice of city ids and the units.
func (c *Client) WeatherByIds(id []int, units string) (w []Weather, err error) {
	var ids string
	for i := range id {
		ids += strconv.Itoa(id[i]) + ","
	}
	ws, err := c.weatherSet(c.baseURL +
		"group" +
		"?id=" + ids[:len(ids)-1] +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	w = ws.Weather
	return
}

func (c *Client) forecast(url string) (f Forecast, err error) {
	if c.key != "" {
		url += "&APPID=" + c.key
	}
	data, err := c.data(url)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	err = json.Unmarshal(data, &f)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
		return
	}
	return
}

// ForecastByName decodes the current forecast given the city name and the units.
func (c *Client) ForecastByName(name string, units string) (f Forecast, err error) {
	f, err = c.forecast(c.baseURL +
		"forecast" +
		"?q=" + name +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	return
}

// ForecastById decodes the current forecast given the city id and the units.
func (c *Client) ForecastById(id int, units string) (f Forecast, err error) {
	f, err = c.forecast(c.baseURL +
		"forecast" +
		"?id=" + strconv.Itoa(id) +
		"&units=" + units)
	if err != nil {
		err = errors.New("owm: error while decoding weather")
	}
	return
}
<|endoftext|>