<commit_before>package xchango

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"text/template"
	"time"
)

type ExchangeConfig interface {
	ExchangeURL() string
	MaxFetchSize() int
	UserDomain() string
	ExchangeVersion() string
	LookAheadDays() int
}

type exchangeVersion interface {
	FolderRequest() string
	CalendarRequest() string
	CalendarDetailRequest() string
}

type ExchangeUser struct {
	Username string
	Password string
}

type ExchangeCalendar struct {
	Folderid  string
	Changekey string
}

var exchangeConfig ExchangeConfig
var version exchangeVersion
var host string

func SetExchangeConfig(config ExchangeConfig) {
	exchangeConfig = config

	switch config.ExchangeVersion() {
	case "2006":
		version = exchange2006{}
	default:
		panic(errors.New(fmt.Sprintf("Unsupported exchange version %s.  Current supported versions are: 2006", config.ExchangeVersion())))
	}

	u, err := url.Parse(config.ExchangeURL())
	if err != nil {
		panic(err)
	}
	host = u.Host
}

func GetExchangeCalendar(user *ExchangeUser) (*ExchangeCalendar, error) {
	soapReq := version.FolderRequest()
	results, err := postContents([]byte(soapReq), user)

	if err != nil {
		return &ExchangeCalendar{}, err
	}

	item := parseCalendarFolder(string(results))

	cal := ExchangeCalendar{
		Folderid:  item.Id,
		Changekey: item.ChangeKey,
	}
	return &cal, nil
}

func GetExchangeAppointments(user *ExchangeUser, cal *ExchangeCalendar) ([]Appointment, error) {
	// This first call will just get ids for each appt
	calRequest := buildCalendarItemRequest(cal.Folderid, cal.Changekey)
	calResults, err := postContents(calRequest, user)
	if err != nil {
		log.Println("Error while getting soap response:", err)
		return nil, err
	}
	itemIds := parseAppointments(calResults)

	// This call will get all the fields given the ids
	appRequest := buildCalendarDetailRequest(itemIds)
	appResults, err := postContents(appRequest, user)
	if err != nil {
		return nil, err
	}

	appointments := parseAppointments(appResults)
	if err != nil {
		return nil, err
	}

	return appointments, nil
}

func buildCalendarItemRequest(folderid string, changekey string) []byte {

	days := exchangeConfig.LookAheadDays()
	if days < 1 {
		days = 14
	}
	startDate := time.Now().UTC().Format(time.RFC3339)
	endDate := time.Now().UTC().AddDate(0, 0, days).Format(time.RFC3339)

	data := struct {
		StartDate    string
		EndDate      string
		FolderId     string
		ChangeKey    string
		MaxFetchSize int
	}{
		startDate,
		endDate,
		folderid,
		changekey,
		exchangeConfig.MaxFetchSize(),
	}

	t, err := template.New("cal").Parse(version.CalendarRequest())
	if err != nil {
		log.Printf("Error is", err)
	}
	var doc bytes.Buffer
	t.Execute(&doc, data)
	if err != nil {
		log.Printf("Error while building contents ", err)
	}

	return doc.Bytes()
}

func buildCalendarDetailRequest(itemIds []Appointment) []byte {

	data := struct {
		Appointments []Appointment
	}{
		itemIds,
	}

	t, err := template.New("detail").Parse(version.CalendarDetailRequest())
	if err != nil {
		log.Printf("Error is", err)
	}
	var doc bytes.Buffer
	t.Execute(&doc, data)
	return doc.Bytes()
}

func postContents(contents []byte, user *ExchangeUser) (string, error) {
	req2, err := http.NewRequest("POST", exchangeConfig.ExchangeURL(), bytes.NewBuffer(contents))

	req2.Header.Set("Host", user.Username+"@"+host)
	req2.Header.Set("Content-Type", "text/xml")
	req2.SetBasicAuth(exchangeConfig.UserDomain()+"/"+user.Username, user.Password)

	// TODO, allow client to be injected by tests!
	client := &http.Client{}
	response, err := client.Do(req2)
	if err != nil {
		return "", err
	}
	defer response.Body.Close()

	content, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Println(err)
		return "", err
	}
	return string(content), nil
}
<commit_msg>Changing appointments to use pointer<commit_after>package xchango

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"text/template"
	"time"
)

type ExchangeConfig interface {
	ExchangeURL() string
	MaxFetchSize() int
	UserDomain() string
	ExchangeVersion() string
	LookAheadDays() int
}

type exchangeVersion interface {
	FolderRequest() string
	CalendarRequest() string
	CalendarDetailRequest() string
}

type ExchangeUser struct {
	Username string
	Password string
}

type ExchangeCalendar struct {
	Folderid  string
	Changekey string
}

var exchangeConfig ExchangeConfig
var version exchangeVersion
var host string

func SetExchangeConfig(config ExchangeConfig) {
	exchangeConfig = config

	switch config.ExchangeVersion() {
	case "2006":
		version = exchange2006{}
	default:
		panic(errors.New(fmt.Sprintf("Unsupported exchange version %s.  Current supported versions are: 2006", config.ExchangeVersion())))
	}

	u, err := url.Parse(config.ExchangeURL())
	if err != nil {
		panic(err)
	}
	host = u.Host
}

func GetExchangeCalendar(user *ExchangeUser) (*ExchangeCalendar, error) {
	soapReq := version.FolderRequest()
	results, err := postContents([]byte(soapReq), user)

	if err != nil {
		return &ExchangeCalendar{}, err
	}

	item := parseCalendarFolder(string(results))

	cal := ExchangeCalendar{
		Folderid:  item.Id,
		Changekey: item.ChangeKey,
	}
	return &cal, nil
}

func GetExchangeAppointments(user *ExchangeUser, cal *ExchangeCalendar) (*[]Appointment, error) {
	// This first call will just get ids for each appt
	calRequest := buildCalendarItemRequest(cal.Folderid, cal.Changekey)
	calResults, err := postContents(calRequest, user)
	if err != nil {
		log.Println("Error while getting soap response:", err)
		return nil, err
	}
	itemIds := parseAppointments(calResults)

	// This call will get all the fields given the ids
	appRequest := buildCalendarDetailRequest(itemIds)
	appResults, err := postContents(appRequest, user)
	if err != nil {
		return nil, err
	}

	appointments := parseAppointments(appResults)
	if err != nil {
		return nil, err
	}

	return &appointments, nil
}

func buildCalendarItemRequest(folderid string, changekey string) []byte {

	days := exchangeConfig.LookAheadDays()
	if days < 1 {
		days = 14
	}
	startDate := time.Now().UTC().Format(time.RFC3339)
	endDate := time.Now().UTC().AddDate(0, 0, days).Format(time.RFC3339)

	data := struct {
		StartDate    string
		EndDate      string
		FolderId     string
		ChangeKey    string
		MaxFetchSize int
	}{
		startDate,
		endDate,
		folderid,
		changekey,
		exchangeConfig.MaxFetchSize(),
	}

	t, err := template.New("cal").Parse(version.CalendarRequest())
	if err != nil {
		log.Printf("Error is", err)
	}
	var doc bytes.Buffer
	t.Execute(&doc, data)
	if err != nil {
		log.Printf("Error while building contents ", err)
	}

	return doc.Bytes()
}

func buildCalendarDetailRequest(itemIds []Appointment) []byte {

	data := struct {
		Appointments []Appointment
	}{
		itemIds,
	}

	t, err := template.New("detail").Parse(version.CalendarDetailRequest())
	if err != nil {
		log.Printf("Error is", err)
	}
	var doc bytes.Buffer
	t.Execute(&doc, data)
	return doc.Bytes()
}

func postContents(contents []byte, user *ExchangeUser) (string, error) {
	req2, err := http.NewRequest("POST", exchangeConfig.ExchangeURL(), bytes.NewBuffer(contents))

	req2.Header.Set("Host", user.Username+"@"+host)
	req2.Header.Set("Content-Type", "text/xml")
	req2.SetBasicAuth(exchangeConfig.UserDomain()+"/"+user.Username, user.Password)

	// TODO, allow client to be injected by tests!
	client := &http.Client{}
	response, err := client.Do(req2)
	if err != nil {
		return "", err
	}
	defer response.Body.Close()

	content, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Println(err)
		return "", err
	}
	return string(content), nil
}
<|endoftext|>


<commit_before>package core

import "sync"

const (
	NONE = iota
	KEY_VALUE
	PRIORITY
	ARRAY
	VALUE
)

type RouteID int
type SharedType int
type Message interface{}
type Connection chan Message
type MessageMap map[RouteID]Message

// Interrupt is a function that interrupts a running block in order to change its state.
// If the interrupt returns false, the block will quit.
type Interrupt func() bool

// Kernel is the core function that operates on an inbound message. It works by populating
// the outbound MessageMap, and can be interrupted on its Interrupt channel.
type Kernel func(MessageMap, MessageMap, MessageMap, Store, chan Interrupt) Interrupt

// A Pin is an inbound or outbound route to a block used in the Spec.
type Pin struct {
	Name string
}

// A Spec defines a block's input and output Pins, and the block's Kernel.
type Spec struct {
	Inputs  []Pin
	Outputs []Pin
	Shared  SharedType
	Kernel  Kernel
}

// A Route is an inbound route to a block. A Route holds the channel that allows Messages
// to be passed into the block. A Route's Path is applied to the inbound Message before populating the
// MessageMap and calling the Kernel. A Route can be set to a Value, instead of waiting for an inbound message.
type Route struct {
	Name  string
	Value interface{}
	C     chan Message
}

type RouteExport struct {
	Name  string
	Path  string
	Value Message
	C     chan Message
}

// An Output holds a set of Connections. Each Connection refers to a Route.C. Every outbound
// mesage is sent on every Connection in the Connections set.
type Output struct {
	Name        string
	Connections map[Connection]struct{}
}

// A ManifestPair is a unique reference to an Output/Connection pair
type ManifestPair struct {
	int
	Connection
}

// A block's Manifest is the set of Connections
type Manifest map[ManifestPair]struct{}

// A block's BlockState is the pair of input/output MessageMaps, and the Manifest
type BlockState struct {
	inputValues    MessageMap
	outputValues   MessageMap
	internalValues MessageMap
	manifest       Manifest
	Processed      bool
}

type Store interface {
	Lock()
	Unlock()
}

type SharedStore struct {
	Type  SharedType
	Store Store
}

// A block's BlockRouting is the set of Input and Output routes, and the Interrupt channel
type BlockRouting struct {
	Inputs        []Route
	Outputs       []Output
	Shared        SharedStore
	InterruptChan chan Interrupt
	sync.RWMutex
}

// A Block is comprised of a state, a set of routes, and a kernel
type Block struct {
	state   BlockState
	routing BlockRouting
	kernel  Kernel
}
<commit_msg>removing route export<commit_after>package core

import "sync"

const (
	NONE = iota
	KEY_VALUE
	PRIORITY
	ARRAY
	VALUE
)

type RouteID int
type SharedType int
type Message interface{}
type Connection chan Message
type MessageMap map[RouteID]Message

// Interrupt is a function that interrupts a running block in order to change its state.
// If the interrupt returns false, the block will quit.
type Interrupt func() bool

// Kernel is the core function that operates on an inbound message. It works by populating
// the outbound MessageMap, and can be interrupted on its Interrupt channel.
type Kernel func(MessageMap, MessageMap, MessageMap, Store, chan Interrupt) Interrupt

// A Pin is an inbound or outbound route to a block used in the Spec.
type Pin struct {
	Name string
}

// A Spec defines a block's input and output Pins, and the block's Kernel.
type Spec struct {
	Inputs  []Pin
	Outputs []Pin
	Shared  SharedType
	Kernel  Kernel
}

// A Route is an inbound route to a block. A Route holds the channel that allows Messages
// to be passed into the block. A Route's Path is applied to the inbound Message before populating the
// MessageMap and calling the Kernel. A Route can be set to a Value, instead of waiting for an inbound message.
type Route struct {
	Name  string
	Value interface{}
	C     chan Message
}

// An Output holds a set of Connections. Each Connection refers to a Route.C. Every outbound
// mesage is sent on every Connection in the Connections set.
type Output struct {
	Name        string
	Connections map[Connection]struct{}
}

// A ManifestPair is a unique reference to an Output/Connection pair
type ManifestPair struct {
	int
	Connection
}

// A block's Manifest is the set of Connections
type Manifest map[ManifestPair]struct{}

// A block's BlockState is the pair of input/output MessageMaps, and the Manifest
type BlockState struct {
	inputValues    MessageMap
	outputValues   MessageMap
	internalValues MessageMap
	manifest       Manifest
	Processed      bool
}

type Store interface {
	Lock()
	Unlock()
}

type SharedStore struct {
	Type  SharedType
	Store Store
}

// A block's BlockRouting is the set of Input and Output routes, and the Interrupt channel
type BlockRouting struct {
	Inputs        []Route
	Outputs       []Output
	Shared        SharedStore
	InterruptChan chan Interrupt
	sync.RWMutex
}

// A Block is comprised of a state, a set of routes, and a kernel
type Block struct {
	state   BlockState
	routing BlockRouting
	kernel  Kernel
}
<|endoftext|>


<commit_before>package gocrawl

import (
	"net/url"
	"strings"

	"github.com/PuerkitoBio/purell"
)

const (
	robotsTxtPath = "/robots.txt"
)

// U is a convenience type definition, it is a map[*url.URL]interface{}
// that can be used to enqueue URLs with state information.
type U map[*url.URL]interface{}

// S is a convenience type definition, it is a map[string]interface{}
// that can be used to enqueue URLs (the string) with state information.
type S map[string]interface{}

// URLContext contains all information related to an URL to process.
type URLContext struct {
	HeadBeforeGet bool
	State         interface{}

	// Internal fields, available through getters
	url                 *url.URL
	normalizedURL       *url.URL
	sourceURL           *url.URL
	normalizedSourceURL *url.URL
}

// URL returns the URL.
func (uc *URLContext) URL() *url.URL {
	return uc.url
}

// NormalizedURL returns the normalized URL (using Options.URLNormalizationFlags)
// of the URL.
func (uc *URLContext) NormalizedURL() *url.URL {
	return uc.normalizedURL
}

// SourceURL returns the source URL, if any (the URL that enqueued this
// URL).
func (uc *URLContext) SourceURL() *url.URL {
	return uc.sourceURL
}

// NormalizedSourceURL returns the normalized form of the source URL,
// if any (using Options.URLNormalizationFlags).
func (uc *URLContext) NormalizedSourceURL() *url.URL {
	return uc.normalizedSourceURL
}

// IsRobotsURL indicates if the URL is a robots.txt URL.
func (uc *URLContext) IsRobotsURL() bool {
	return isRobotsURL(uc.normalizedURL)
}

// cloneForRedirect returns a new URLContext with the given
// destination URL with the same sourceURL and normalizedSourceURL.
func (uc *URLContext) cloneForRedirect(dst *url.URL, normFlags purell.NormalizationFlags) *URLContext {
	var src, normalizedSrc *url.URL
	if uc.sourceURL != nil {
		src = &url.URL{}
		*src = *uc.sourceURL
	}
	if src == nil && uc.url != nil {
		// if the current context doesn't have a source URL, use its URL as
		// source (e.g. for a seed URL that triggers a redirect)
		src = &url.URL{}
		*src = *uc.url
	}

	if uc.normalizedSourceURL != nil {
		normalizedSrc = &url.URL{}
		*normalizedSrc = *uc.normalizedSourceURL
	}
	if normalizedSrc == nil {
		normalizedSrc = &url.URL{}
		*normalizedSrc = *uc.normalizedURL
	}

	rawDst := &url.URL{}
	*rawDst = *dst
	purell.NormalizeURL(dst, normFlags)
	return &URLContext{
		HeadBeforeGet:       uc.HeadBeforeGet,
		State:               uc.State,
		url:                 rawDst,
		normalizedURL:       dst,
		sourceURL:           src,
		normalizedSourceURL: normalizedSrc,
	}
}

// Implement in a private func, because called from HttpClient also (without
// an URLContext).
func isRobotsURL(u *url.URL) bool {
	if u == nil {
		return false
	}
	return strings.ToLower(u.Path) == robotsTxtPath
}

	func toStringArrayContextURL(list []*URLContext) string {
	var str string
	for _, item := range list {
		str += item.NormalizedURL().String()
	}
	return str
}

func (uc *URLContext) getRobotsURLCtx() (*URLContext, error) {
	robURL, err := uc.normalizedURL.Parse(robotsTxtPath)
	if err != nil {
		return nil, err
	}
	return &URLContext{
		false, // Never request HEAD before GET for robots.txt
		nil,   // Always nil state
		robURL,
		robURL,       // Normalized is same as raw
		uc.sourceURL, // Source and normalized source is same as for current context
		uc.normalizedSourceURL,
	}, nil
}

func (c *Crawler) toURLContexts(raw interface{}, src *url.URL) []*URLContext {
	var res []*URLContext

	mapString := func(v S) {
		res = make([]*URLContext, 0, len(v))
		for s, st := range v {
			ctx, err := c.stringToURLContext(s, src)
			if err != nil {
				c.Options.Extender.Error(newCrawlError(nil, err, CekParseURL))
				c.logFunc(LogError, "ERROR parsing URL %s", s)
			} else {
				ctx.State = st
				res = append(res, ctx)
			}
		}
	}

	mapURL := func(v U) {
		res = make([]*URLContext, 0, len(v))
		for u, st := range v {
			ctx := c.urlToURLContext(u, src)
			ctx.State = st
			res = append(res, ctx)
		}
	}

	switch v := raw.(type) {
	case *URLContext:
		res = []*URLContext{v}

	case string:
		// Convert a single string URL to an URLContext
		ctx, err := c.stringToURLContext(v, src)
		if err != nil {
			c.Options.Extender.Error(newCrawlError(nil, err, CekParseURL))
			c.logFunc(LogError, "ERROR parsing URL %s", v)
		} else {
			res = []*URLContext{ctx}
		}

	case []string:
		// Convert all strings to URLContexts
		res = make([]*URLContext, 0, len(v))
		for _, s := range v {
			ctx, err := c.stringToURLContext(s, src)
			if err != nil {
				c.Options.Extender.Error(newCrawlError(nil, err, CekParseURL))
				c.logFunc(LogError, "ERROR parsing URL %s", s)
			} else {
				res = append(res, ctx)
			}
		}

	case *url.URL:
		res = []*URLContext{c.urlToURLContext(v, src)}

	case []*url.URL:
		res = make([]*URLContext, 0, len(v))
		for _, u := range v {
			res = append(res, c.urlToURLContext(u, src))
		}

	case map[string]interface{}:
		mapString(S(v))

	case S:
		mapString(v)

	case map[*url.URL]interface{}:
		mapURL(U(v))

	case U:
		mapURL(v)

	default:
		if raw != nil {
			panic("unsupported URL type passed as empty interface")
		}
	}
	return res
}

func (c *Crawler) stringToURLContext(str string, src *url.URL) (*URLContext, error) {
	u, err := url.Parse(str)
	if err != nil {
		return nil, err
	}
	return c.urlToURLContext(u, src), nil
}

func (c *Crawler) urlToURLContext(u, src *url.URL) *URLContext {
	var rawSrc *url.URL

	rawU := *u
	purell.NormalizeURL(u, c.Options.URLNormalizationFlags)
	if src != nil {
		rawSrc = &url.URL{}
		*rawSrc = *src
		purell.NormalizeURL(src, c.Options.URLNormalizationFlags)
	}

	return &URLContext{
		c.Options.HeadBeforeGet,
		nil,
		&rawU,
		u,
		rawSrc,
		src,
	}
}
<commit_msg>added comma<commit_after>package gocrawl

import (
	"net/url"
	"strings"

	"github.com/PuerkitoBio/purell"
)

const (
	robotsTxtPath = "/robots.txt"
)

// U is a convenience type definition, it is a map[*url.URL]interface{}
// that can be used to enqueue URLs with state information.
type U map[*url.URL]interface{}

// S is a convenience type definition, it is a map[string]interface{}
// that can be used to enqueue URLs (the string) with state information.
type S map[string]interface{}

// URLContext contains all information related to an URL to process.
type URLContext struct {
	HeadBeforeGet bool
	State         interface{}

	// Internal fields, available through getters
	url                 *url.URL
	normalizedURL       *url.URL
	sourceURL           *url.URL
	normalizedSourceURL *url.URL
}

// URL returns the URL.
func (uc *URLContext) URL() *url.URL {
	return uc.url
}

// NormalizedURL returns the normalized URL (using Options.URLNormalizationFlags)
// of the URL.
func (uc *URLContext) NormalizedURL() *url.URL {
	return uc.normalizedURL
}

// SourceURL returns the source URL, if any (the URL that enqueued this
// URL).
func (uc *URLContext) SourceURL() *url.URL {
	return uc.sourceURL
}

// NormalizedSourceURL returns the normalized form of the source URL,
// if any (using Options.URLNormalizationFlags).
func (uc *URLContext) NormalizedSourceURL() *url.URL {
	return uc.normalizedSourceURL
}

// IsRobotsURL indicates if the URL is a robots.txt URL.
func (uc *URLContext) IsRobotsURL() bool {
	return isRobotsURL(uc.normalizedURL)
}

// cloneForRedirect returns a new URLContext with the given
// destination URL with the same sourceURL and normalizedSourceURL.
func (uc *URLContext) cloneForRedirect(dst *url.URL, normFlags purell.NormalizationFlags) *URLContext {
	var src, normalizedSrc *url.URL
	if uc.sourceURL != nil {
		src = &url.URL{}
		*src = *uc.sourceURL
	}
	if src == nil && uc.url != nil {
		// if the current context doesn't have a source URL, use its URL as
		// source (e.g. for a seed URL that triggers a redirect)
		src = &url.URL{}
		*src = *uc.url
	}

	if uc.normalizedSourceURL != nil {
		normalizedSrc = &url.URL{}
		*normalizedSrc = *uc.normalizedSourceURL
	}
	if normalizedSrc == nil {
		normalizedSrc = &url.URL{}
		*normalizedSrc = *uc.normalizedURL
	}

	rawDst := &url.URL{}
	*rawDst = *dst
	purell.NormalizeURL(dst, normFlags)
	return &URLContext{
		HeadBeforeGet:       uc.HeadBeforeGet,
		State:               uc.State,
		url:                 rawDst,
		normalizedURL:       dst,
		sourceURL:           src,
		normalizedSourceURL: normalizedSrc,
	}
}

// Implement in a private func, because called from HttpClient also (without
// an URLContext).
func isRobotsURL(u *url.URL) bool {
	if u == nil {
		return false
	}
	return strings.ToLower(u.Path) == robotsTxtPath
}

	func toStringArrayContextURL(list []*URLContext) string {
	var str string
	for _, item := range list {
		if len(str) != 0 {
			str += ", "
		}
		str += item.NormalizedURL().String()
	}
	return str
}

func (uc *URLContext) getRobotsURLCtx() (*URLContext, error) {
	robURL, err := uc.normalizedURL.Parse(robotsTxtPath)
	if err != nil {
		return nil, err
	}
	return &URLContext{
		false, // Never request HEAD before GET for robots.txt
		nil,   // Always nil state
		robURL,
		robURL,       // Normalized is same as raw
		uc.sourceURL, // Source and normalized source is same as for current context
		uc.normalizedSourceURL,
	}, nil
}

func (c *Crawler) toURLContexts(raw interface{}, src *url.URL) []*URLContext {
	var res []*URLContext

	mapString := func(v S) {
		res = make([]*URLContext, 0, len(v))
		for s, st := range v {
			ctx, err := c.stringToURLContext(s, src)
			if err != nil {
				c.Options.Extender.Error(newCrawlError(nil, err, CekParseURL))
				c.logFunc(LogError, "ERROR parsing URL %s", s)
			} else {
				ctx.State = st
				res = append(res, ctx)
			}
		}
	}

	mapURL := func(v U) {
		res = make([]*URLContext, 0, len(v))
		for u, st := range v {
			ctx := c.urlToURLContext(u, src)
			ctx.State = st
			res = append(res, ctx)
		}
	}

	switch v := raw.(type) {
	case *URLContext:
		res = []*URLContext{v}

	case string:
		// Convert a single string URL to an URLContext
		ctx, err := c.stringToURLContext(v, src)
		if err != nil {
			c.Options.Extender.Error(newCrawlError(nil, err, CekParseURL))
			c.logFunc(LogError, "ERROR parsing URL %s", v)
		} else {
			res = []*URLContext{ctx}
		}

	case []string:
		// Convert all strings to URLContexts
		res = make([]*URLContext, 0, len(v))
		for _, s := range v {
			ctx, err := c.stringToURLContext(s, src)
			if err != nil {
				c.Options.Extender.Error(newCrawlError(nil, err, CekParseURL))
				c.logFunc(LogError, "ERROR parsing URL %s", s)
			} else {
				res = append(res, ctx)
			}
		}

	case *url.URL:
		res = []*URLContext{c.urlToURLContext(v, src)}

	case []*url.URL:
		res = make([]*URLContext, 0, len(v))
		for _, u := range v {
			res = append(res, c.urlToURLContext(u, src))
		}

	case map[string]interface{}:
		mapString(S(v))

	case S:
		mapString(v)

	case map[*url.URL]interface{}:
		mapURL(U(v))

	case U:
		mapURL(v)

	default:
		if raw != nil {
			panic("unsupported URL type passed as empty interface")
		}
	}
	return res
}

func (c *Crawler) stringToURLContext(str string, src *url.URL) (*URLContext, error) {
	u, err := url.Parse(str)
	if err != nil {
		return nil, err
	}
	return c.urlToURLContext(u, src), nil
}

func (c *Crawler) urlToURLContext(u, src *url.URL) *URLContext {
	var rawSrc *url.URL

	rawU := *u
	purell.NormalizeURL(u, c.Options.URLNormalizationFlags)
	if src != nil {
		rawSrc = &url.URL{}
		*rawSrc = *src
		purell.NormalizeURL(src, c.Options.URLNormalizationFlags)
	}

	return &URLContext{
		c.Options.HeadBeforeGet,
		nil,
		&rawU,
		u,
		rawSrc,
		src,
	}
}
<|endoftext|>


<commit_before>package couchcandy

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

// GetDatabaseInfo returns basic information about the database in session.
func (c *CouchCandy) GetDatabaseInfo() (*DatabaseInfo, error) {

	url := createDatabaseURL(c.LclSession)
	page, err := readFrom(url, c.GetHandler)
	if err != nil {
		return nil, err
	}

	dbInfo := &DatabaseInfo{}
	unmarshallError := json.Unmarshal(page, dbInfo)
	return dbInfo, unmarshallError

}

// GetDocument Returns the specified document.
func (c *CouchCandy) GetDocument(id string, v interface{}, options Options) error {

	url := createDocumentURLWithOptions(c.LclSession, id, options)
	page, err := readFrom(url, c.GetHandler)
	if err != nil {
		return err
	}

	unmarshallError := json.Unmarshal(page, v)
	return unmarshallError

}

// PostDocument Adds a document in the database but the system will generate
// an id. Look at PutDocumentWithID for setting an id for the document explicitly.
func (c *CouchCandy) PostDocument(document interface{}) (*OperationResponse, error) {

	url := createDatabaseURL(c.LclSession)
	bodyStr, marshallError := safeMarshall(document)
	if marshallError != nil {
		return nil, marshallError
	}

	page, err := readFromWithBody(url, bodyStr, c.PostHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// PutDocument Updates a document in the database. Note that _id and _rev
// fields are required in the passed document.
func (c *CouchCandy) PutDocument(document interface{}) (*OperationResponse, error) {

	bodyStr, marshallError := safeMarshall(document)
	if marshallError != nil {
		return nil, marshallError
	}

	url := createPutDocumentURL(c.LclSession, bodyStr)

	page, err := readFromWithBody(url, bodyStr, c.PutHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// PutDocumentWithID Inserts a document in the database with the specified id
func (c *CouchCandy) PutDocumentWithID(id string, document interface{}) (*OperationResponse, error) {

	url := fmt.Sprintf("%s/%s", createDatabaseURL(c.LclSession), id)

	bodyStr, marshallError := safeMarshall(document)
	if marshallError != nil {
		return nil, marshallError
	}

	page, err := readFromWithBody(url, bodyStr, c.PutHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// GetAllDocuments : Returns all documents in the database based on the passed parameters.
func (c *CouchCandy) GetAllDocuments(options Options) (*AllDocuments, error) {

	checkOptionsForAllDocuments(&options)
	url := fmt.Sprintf("%s/_all_docs?descending=%v&limit=%v&include_docs=%v", createDatabaseURL(c.LclSession), options.Descending, options.Limit, options.IncludeDocs)
	page, err := readFrom(url, c.GetHandler)
	if err != nil {
		return nil, err
	}

	return toAllDocuments(page)

}

// GetDocumentsByKeys Fetches all the documents corresponding to the passed keys array.
func (c *CouchCandy) GetDocumentsByKeys(keys []string, options Options) (*AllDocuments, error) {

	checkOptionsForAllDocuments(&options)
	url := fmt.Sprintf("%s/_all_docs?descending=%v&limit=%v&include_docs=%v", createDatabaseURL(c.LclSession), options.Descending, options.Limit, options.IncludeDocs)

	body, marhsallError := json.Marshal(&AllDocumentsKeys{
		keys: keys,
	})
	if marhsallError != nil {
		return nil, marhsallError
	}

	page, err := readFromWithBody(url, string(body), c.PostHandler)
	if err != nil {
		return nil, err
	}

	return toAllDocuments(page)

}

// PutDatabase : Creates a database in CouchDB
func (c *CouchCandy) PutDatabase(name string) (*OperationResponse, error) {

	c.LclSession.Database = name
	url := createDatabaseURL(c.LclSession)

	page, err := readFromWithBody(url, "", c.PutHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// DeleteDatabase : Deletes the passed database from the system.
func (c *CouchCandy) DeleteDatabase(name string) (*OperationResponse, error) {

	c.LclSession.Database = name
	url := createDatabaseURL(c.LclSession)
	page, err := readFrom(url, c.DeleteHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// DeleteDocument Deletes the passed document with revision from the database
func (c *CouchCandy) DeleteDocument(id string, revision string) (*OperationResponse, error) {

	url := fmt.Sprintf("%s?rev=%s", createDocumentURL(c.LclSession, id), revision)
	page, err := readFrom(url, c.DeleteHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// GetAllDatabases : Returns all the database names in the system.
func (c *CouchCandy) GetAllDatabases() ([]string, error) {

	url := createAllDatabasesURL(c.LclSession)
	page, err := readFrom(url, c.GetHandler)
	if err != nil {
		return nil, err
	}

	var dbs []string
	unmarshallError := json.Unmarshal(page, &dbs)
	return dbs, unmarshallError

}

// GetChangeNotifications : Return the current change notifications.
func (c *CouchCandy) GetChangeNotifications(options Options) (*Changes, error) {

	url := fmt.Sprintf("%s/_changes?style=%s", createDatabaseURL(c.LclSession), options.Style)
	page, err := readFrom(url, c.GetHandler)
	if err != nil {
		return nil, err
	}

	changes := &Changes{}
	unmarshallError := json.Unmarshal(page, changes)
	return changes, unmarshallError

}

func readFromWithBody(url string, body string, handler func(str string, bd string) (*http.Response, error)) ([]byte, error) {

	res, err := handler(url, body)
	if err != nil {
		return nil, err
	}

	page, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		return nil, err
	}

	return page, nil

}

func readFrom(url string, handler func(str string) (*http.Response, error)) ([]byte, error) {

	res, err := handler(url)
	if err != nil {
		return nil, err
	}

	page, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		return nil, err
	}

	return page, nil

}

func defaultPostHandler(url string, body string) (*http.Response, error) {
	return defaultHandlerWithBody(http.MethodPost, url, body, &http.Client{})
}

func defaultPutHandler(url string, body string) (*http.Response, error) {
	return defaultHandlerWithBody(http.MethodPut, url, body, &http.Client{})
}

func defaultHandlerWithBody(method, url, body string, client CandyHTTPClient) (*http.Response, error) {

	bodyJSON := strings.NewReader(body)
	request, requestError := http.NewRequest(method, url, bodyJSON)
	if requestError != nil {
		return nil, requestError
	}

	request.Header.Add("Content-Type", "application/json")
	response, err := client.Do(request)
	if err != nil {
		return nil, err
	}
	return response, nil
}

func defaultGetHandler(url string) (*http.Response, error) {
	return defaultHandler(http.MethodGet, url, &http.Client{})
}

func defaultDeleteHandler(url string) (*http.Response, error) {
	return defaultHandler(http.MethodDelete, url, &http.Client{})
}

func defaultHandler(method, url string, client CandyHTTPClient) (*http.Response, error) {

	request, requestError := http.NewRequest(method, url, nil)
	if requestError != nil {
		return nil, requestError
	}

	response, err := client.Do(request)
	if err != nil {
		return nil, err
	}
	return response, nil

}
<commit_msg>simpler method signatures when multiple strings are declared as parameters<commit_after>package couchcandy

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

// GetDatabaseInfo returns basic information about the database in session.
func (c *CouchCandy) GetDatabaseInfo() (*DatabaseInfo, error) {

	url := createDatabaseURL(c.LclSession)
	page, err := readFrom(url, c.GetHandler)
	if err != nil {
		return nil, err
	}

	dbInfo := &DatabaseInfo{}
	unmarshallError := json.Unmarshal(page, dbInfo)
	return dbInfo, unmarshallError

}

// GetDocument Returns the specified document.
func (c *CouchCandy) GetDocument(id string, v interface{}, options Options) error {

	url := createDocumentURLWithOptions(c.LclSession, id, options)
	page, err := readFrom(url, c.GetHandler)
	if err != nil {
		return err
	}

	unmarshallError := json.Unmarshal(page, v)
	return unmarshallError

}

// PostDocument Adds a document in the database but the system will generate
// an id. Look at PutDocumentWithID for setting an id for the document explicitly.
func (c *CouchCandy) PostDocument(document interface{}) (*OperationResponse, error) {

	url := createDatabaseURL(c.LclSession)
	bodyStr, marshallError := safeMarshall(document)
	if marshallError != nil {
		return nil, marshallError
	}

	page, err := readFromWithBody(url, bodyStr, c.PostHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// PutDocument Updates a document in the database. Note that _id and _rev
// fields are required in the passed document.
func (c *CouchCandy) PutDocument(document interface{}) (*OperationResponse, error) {

	bodyStr, marshallError := safeMarshall(document)
	if marshallError != nil {
		return nil, marshallError
	}

	url := createPutDocumentURL(c.LclSession, bodyStr)

	page, err := readFromWithBody(url, bodyStr, c.PutHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// PutDocumentWithID Inserts a document in the database with the specified id
func (c *CouchCandy) PutDocumentWithID(id string, document interface{}) (*OperationResponse, error) {

	url := fmt.Sprintf("%s/%s", createDatabaseURL(c.LclSession), id)

	bodyStr, marshallError := safeMarshall(document)
	if marshallError != nil {
		return nil, marshallError
	}

	page, err := readFromWithBody(url, bodyStr, c.PutHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// GetAllDocuments : Returns all documents in the database based on the passed parameters.
func (c *CouchCandy) GetAllDocuments(options Options) (*AllDocuments, error) {

	checkOptionsForAllDocuments(&options)
	url := fmt.Sprintf("%s/_all_docs?descending=%v&limit=%v&include_docs=%v", createDatabaseURL(c.LclSession), options.Descending, options.Limit, options.IncludeDocs)
	page, err := readFrom(url, c.GetHandler)
	if err != nil {
		return nil, err
	}

	return toAllDocuments(page)

}

// GetDocumentsByKeys Fetches all the documents corresponding to the passed keys array.
func (c *CouchCandy) GetDocumentsByKeys(keys []string, options Options) (*AllDocuments, error) {

	checkOptionsForAllDocuments(&options)
	url := fmt.Sprintf("%s/_all_docs?descending=%v&limit=%v&include_docs=%v", createDatabaseURL(c.LclSession), options.Descending, options.Limit, options.IncludeDocs)

	body, marhsallError := json.Marshal(&AllDocumentsKeys{
		keys: keys,
	})
	if marhsallError != nil {
		return nil, marhsallError
	}

	page, err := readFromWithBody(url, string(body), c.PostHandler)
	if err != nil {
		return nil, err
	}

	return toAllDocuments(page)

}

// PutDatabase : Creates a database in CouchDB
func (c *CouchCandy) PutDatabase(name string) (*OperationResponse, error) {

	c.LclSession.Database = name
	url := createDatabaseURL(c.LclSession)

	page, err := readFromWithBody(url, "", c.PutHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// DeleteDatabase : Deletes the passed database from the system.
func (c *CouchCandy) DeleteDatabase(name string) (*OperationResponse, error) {

	c.LclSession.Database = name
	url := createDatabaseURL(c.LclSession)
	page, err := readFrom(url, c.DeleteHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// DeleteDocument Deletes the passed document with revision from the database
func (c *CouchCandy) DeleteDocument(id string, revision string) (*OperationResponse, error) {

	url := fmt.Sprintf("%s?rev=%s", createDocumentURL(c.LclSession, id), revision)
	page, err := readFrom(url, c.DeleteHandler)
	if err != nil {
		return nil, err
	}

	return toOperationResponse(page)

}

// GetAllDatabases : Returns all the database names in the system.
func (c *CouchCandy) GetAllDatabases() ([]string, error) {

	url := createAllDatabasesURL(c.LclSession)
	page, err := readFrom(url, c.GetHandler)
	if err != nil {
		return nil, err
	}

	var dbs []string
	unmarshallError := json.Unmarshal(page, &dbs)
	return dbs, unmarshallError

}

// GetChangeNotifications : Return the current change notifications.
func (c *CouchCandy) GetChangeNotifications(options Options) (*Changes, error) {

	url := fmt.Sprintf("%s/_changes?style=%s", createDatabaseURL(c.LclSession), options.Style)
	page, err := readFrom(url, c.GetHandler)
	if err != nil {
		return nil, err
	}

	changes := &Changes{}
	unmarshallError := json.Unmarshal(page, changes)
	return changes, unmarshallError

}

func readFromWithBody(url, body string, handler func(str string, bd string) (*http.Response, error)) ([]byte, error) {

	res, err := handler(url, body)
	if err != nil {
		return nil, err
	}

	page, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		return nil, err
	}

	return page, nil

}

func readFrom(url string, handler func(str string) (*http.Response, error)) ([]byte, error) {

	res, err := handler(url)
	if err != nil {
		return nil, err
	}

	page, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		return nil, err
	}

	return page, nil

}

func defaultPostHandler(url, body string) (*http.Response, error) {
	return defaultHandlerWithBody(http.MethodPost, url, body, &http.Client{})
}

func defaultPutHandler(url, body string) (*http.Response, error) {
	return defaultHandlerWithBody(http.MethodPut, url, body, &http.Client{})
}

func defaultHandlerWithBody(method, url, body string, client CandyHTTPClient) (*http.Response, error) {

	bodyJSON := strings.NewReader(body)
	request, requestError := http.NewRequest(method, url, bodyJSON)
	if requestError != nil {
		return nil, requestError
	}

	request.Header.Add("Content-Type", "application/json")
	response, err := client.Do(request)
	if err != nil {
		return nil, err
	}
	return response, nil
}

func defaultGetHandler(url string) (*http.Response, error) {
	return defaultHandler(http.MethodGet, url, &http.Client{})
}

func defaultDeleteHandler(url string) (*http.Response, error) {
	return defaultHandler(http.MethodDelete, url, &http.Client{})
}

func defaultHandler(method, url string, client CandyHTTPClient) (*http.Response, error) {

	request, requestError := http.NewRequest(method, url, nil)
	if requestError != nil {
		return nil, requestError
	}

	response, err := client.Do(request)
	if err != nil {
		return nil, err
	}
	return response, nil

}
<|endoftext|>


<commit_before>package util

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io"
)

const _key = "2131231231231234"

//Encrypt encripta texto baseado na documentação do GO
func Encrypt(s string) string {
	key := []byte(_key)
	plaintext := []byte(s)
	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}
	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}
	stream := cipher.NewCFBEncrypter(block, iv)
	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)
	// convert to base64
	return base64.URLEncoding.EncodeToString(ciphertext)
}

//Decrypt decripta string encriptada
func Decrypt(s string) string {
	ciphertext, _ := base64.URLEncoding.DecodeString(s)

	block, err := aes.NewCipher([]byte(_key))
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	if len(ciphertext) < aes.BlockSize {
		return ""
	}
	iv := ciphertext[:aes.BlockSize]
	ciphertext = ciphertext[aes.BlockSize:]

	stream := cipher.NewCFBDecrypter(block, iv)

	// XORKeyStream can work in-place if the two arguments are the same.
	stream.XORKeyStream(ciphertext, ciphertext)

	return fmt.Sprintf("%s", ciphertext)
}

//Base64 converte um string para base64
func Base64(s string) string {
	sEnc := base64.StdEncoding.EncodeToString([]byte(s))
	return sEnc
}

//Base64Decode converte uma string base64 para uma string normal
func Base64Decode(s string) string {
	sDec, _ := base64.StdEncoding.DecodeString(s)
	return string(sDec)
}

//Sha256 converts string to hash sha256. Encode default: Base64
func Sha256(s string, e string) string {
	var sEnc string

	h := sha256.New()
	h.Write([]byte(s))

	switch e {
	case "hex":
		sEnc = hex.EncodeToString(h.Sum(nil))
	default:
		sEnc = base64.StdEncoding.EncodeToString(h.Sum(nil))
	}
	return sEnc
}
<commit_msg>:recycle: Refatora geração da chave<commit_after>package util

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io"
)

const _key = "2131231231231234"

//Encrypt encripta texto baseado na documentação do GO
func Encrypt(s string) string {
	key := []byte(_key)
	plaintext := []byte(s)
	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}
	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}
	stream := cipher.NewCFBEncrypter(block, iv)
	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)
	// convert to base64
	return base64.URLEncoding.EncodeToString(ciphertext)
}

//Decrypt decripta string encriptada
func Decrypt(s string) string {
	ciphertext, _ := base64.URLEncoding.DecodeString(s)

	block, err := aes.NewCipher([]byte(_key))
	if err != nil {
		panic(err)
	}

	// The IV needs to be unique, but not secure. Therefore it's common to
	// include it at the beginning of the ciphertext.
	if len(ciphertext) < aes.BlockSize {
		return ""
	}
	iv := ciphertext[:aes.BlockSize]
	ciphertext = ciphertext[aes.BlockSize:]

	stream := cipher.NewCFBDecrypter(block, iv)

	// XORKeyStream can work in-place if the two arguments are the same.
	stream.XORKeyStream(ciphertext, ciphertext)

	return fmt.Sprintf("%s", ciphertext)
}

//Base64 converte um string para base64
func Base64(s string) string {
	sEnc := base64.StdEncoding.EncodeToString([]byte(s))
	return sEnc
}

//Base64Decode converte uma string base64 para uma string normal
func Base64Decode(s string) string {
	sDec, _ := base64.StdEncoding.DecodeString(s)
	return string(sDec)
}

//Sha256 converts string to hash sha256. Encode default: Base64
func Sha256(s, e string) string {
	var sEnc string

	h := sha256.New()
	h.Write([]byte(s))

	switch e {
	case "hex":
		sEnc = hex.EncodeToString(h.Sum(nil))
	default:
		sEnc = base64.StdEncoding.EncodeToString(h.Sum(nil))
	}
	return sEnc
}
<|endoftext|>


<commit_before>package utils

import (
	"errors"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

/*
	Return the file content in a string.
*/
func FileReadAll(filepath string) (string, error) {
	contents, err := ioutil.ReadFile(filepath)
	return string(contents), err
}

/*
	Return the file content in a []byte.
*/
func FileReadAllBytes(filepath string) ([]byte, error) {
	contents, err := ioutil.ReadFile(filepath)
	return contents, err
}

/*
	Write a string in a file.
*/
func FileWriteAll(filepath, content string) error {
	return ioutil.WriteFile(filepath, []byte(content), 0600)
}

/*
	Write a []byte in a file.
*/
func FileWriteAllBytes(filepath string, content []byte) error {
	return ioutil.WriteFile(filepath, content, 0600)
}

/*
	Test if a file exists.
	(if the target is a dir, the function returns false)
*/
func FileExists(path string) bool {
	if info, err := os.Stat(path); err == nil {
		if !info.IsDir() {
			return true
		}
	}
	return false
}

/*
	Test if a dir exists.
	(if the target is a file, the function returns false)
*/
func DirExists(path string) bool {
	if info, err := os.Stat(path); err == nil {
		if info.IsDir() {
			return true
		}
	}
	return false
}

/*
	Use unix "du" command to get directory disk usage
*/
func DirectorySize(dir string) (int64, error) {
	var size int64 // Directory size
	var err error  // Error handling
	var out []byte // Command output

	out, err = exec.Command("du", "-s", dir).Output()
	if err != nil {
		return 0, err
	}

	s := strings.Split(string(out), "\t")
	if len(s) < 2 {
		return 0, errors.New("Can't get disk usage")
	}
	size = int64(S2I(s[0]))

	return size * 1024, nil
}
<commit_msg>Add ionice idle for the 'du' process<commit_after>package utils

import (
	"errors"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

/*
	Return the file content in a string.
*/
func FileReadAll(filepath string) (string, error) {
	contents, err := ioutil.ReadFile(filepath)
	return string(contents), err
}

/*
	Return the file content in a []byte.
*/
func FileReadAllBytes(filepath string) ([]byte, error) {
	contents, err := ioutil.ReadFile(filepath)
	return contents, err
}

/*
	Write a string in a file.
*/
func FileWriteAll(filepath, content string) error {
	return ioutil.WriteFile(filepath, []byte(content), 0600)
}

/*
	Write a []byte in a file.
*/
func FileWriteAllBytes(filepath string, content []byte) error {
	return ioutil.WriteFile(filepath, content, 0600)
}

/*
	Test if a file exists.
	(if the target is a dir, the function returns false)
*/
func FileExists(path string) bool {
	if info, err := os.Stat(path); err == nil {
		if !info.IsDir() {
			return true
		}
	}
	return false
}

/*
	Test if a dir exists.
	(if the target is a file, the function returns false)
*/
func DirExists(path string) bool {
	if info, err := os.Stat(path); err == nil {
		if info.IsDir() {
			return true
		}
	}
	return false
}

/*
	Use unix "du" command to get directory disk usage
*/
func DirectorySize(dir string) (int64, error) {
	var size int64 // Directory size
	var err error  // Error handling
	var out []byte // Command output

	out, err = exec.Command("ionice", "-c", "3", "du", "-s", dir).Output()
	if err != nil {
		return 0, err
	}

	s := strings.Split(string(out), "\t")
	if len(s) < 2 {
		return 0, errors.New("Can't get disk usage")
	}
	size = int64(S2I(s[0]))

	return size * 1024, nil
}
<|endoftext|>


<commit_before>package utils

import (
	"net/http"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/registry/auth"
	"github.com/endophage/gotuf/signed"
	"github.com/gorilla/mux"
	"golang.org/x/net/context"

	"github.com/docker/notary/errors"
)

// contextHandler defines an alterate HTTP handler interface which takes in
// a context for authorization and returns an HTTP application error.
type contextHandler func(ctx context.Context, w http.ResponseWriter, r *http.Request) *errors.HTTPError

// rootHandler is an implementation of an HTTP request handler which handles
// authorization and calling out to the defined alternate http handler.
type rootHandler struct {
	handler contextHandler
	auth    auth.AccessController
	actions []string
	context context.Context
	trust   signed.CryptoService
	//cachePool redis.Pool
}

// RootHandlerFactory creates a new rootHandler factory  using the given
// Context creator and authorizer.  The returned factory allows creating
// new rootHandlers from the alternate http handler contextHandler and
// a scope.
func RootHandlerFactory(auth auth.AccessController, ctx context.Context, trust signed.CryptoService) func(contextHandler, ...string) *rootHandler {
	return func(handler contextHandler, actions ...string) *rootHandler {
		return &rootHandler{
			handler: handler,
			auth:    auth,
			actions: actions,
			context: ctx,
			trust:   trust,
		}
	}
}

// ServeHTTP serves an HTTP request and implements the http.Handler interface.
func (root *rootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	ctx := context.WithValue(root.context, "repo", vars["imageName"])

	// endophage: I don't guarantee that a signer will always be stateless but a CryptoService
	// is expected to be. Create a new Signer for each request.
	ctx = context.WithValue(ctx, "signer", signed.NewSigner(root.trust))
	ctx = context.WithValue(ctx, "cryptoService", root.trust)

	ctx = context.WithValue(ctx, "http.request", r)

	if root.auth != nil {
		var err error
		access := buildAccessRecords(vars["imageName"], root.actions...)
		if ctx, err = root.auth.Authorized(ctx, access...); err != nil {
			if err, ok := err.(auth.Challenge); ok {
				err.ServeHTTP(w, r)
				return
			}
			http.Error(w, err.Error(), http.StatusUnauthorized)
			return
		}
	}
	if err := root.handler(ctx, w, r); err != nil {
		logrus.Error("[Notary Server] ", err.Error())
		http.Error(w, err.Error(), err.HTTPStatus)
		return
	}
	return
}

func buildAccessRecords(repo string, actions ...string) []auth.Access {
	requiredAccess := make([]auth.Access, 0, len(actions))
	for _, action := range actions {
		requiredAccess = append(requiredAccess, auth.Access{
			Resource: auth.Resource{
				Type: "repo",
				Name: repo,
			},
			Action: action,
		})
	}
	return requiredAccess
}
<commit_msg>challenge.ServeHTTP doesn't set Unauthorized status code Signed-off-by: David Lawrence <david.lawrence@docker.com> (github: endophage)<commit_after>package utils

import (
	"net/http"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/registry/auth"
	"github.com/endophage/gotuf/signed"
	"github.com/gorilla/mux"
	"golang.org/x/net/context"

	"github.com/docker/notary/errors"
)

// contextHandler defines an alterate HTTP handler interface which takes in
// a context for authorization and returns an HTTP application error.
type contextHandler func(ctx context.Context, w http.ResponseWriter, r *http.Request) *errors.HTTPError

// rootHandler is an implementation of an HTTP request handler which handles
// authorization and calling out to the defined alternate http handler.
type rootHandler struct {
	handler contextHandler
	auth    auth.AccessController
	actions []string
	context context.Context
	trust   signed.CryptoService
	//cachePool redis.Pool
}

// RootHandlerFactory creates a new rootHandler factory  using the given
// Context creator and authorizer.  The returned factory allows creating
// new rootHandlers from the alternate http handler contextHandler and
// a scope.
func RootHandlerFactory(auth auth.AccessController, ctx context.Context, trust signed.CryptoService) func(contextHandler, ...string) *rootHandler {
	return func(handler contextHandler, actions ...string) *rootHandler {
		return &rootHandler{
			handler: handler,
			auth:    auth,
			actions: actions,
			context: ctx,
			trust:   trust,
		}
	}
}

// ServeHTTP serves an HTTP request and implements the http.Handler interface.
func (root *rootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	ctx := context.WithValue(root.context, "repo", vars["imageName"])

	// endophage: I don't guarantee that a signer will always be stateless but a CryptoService
	// is expected to be. Create a new Signer for each request.
	ctx = context.WithValue(ctx, "signer", signed.NewSigner(root.trust))
	ctx = context.WithValue(ctx, "cryptoService", root.trust)

	ctx = context.WithValue(ctx, "http.request", r)

	if root.auth != nil {
		var err error
		access := buildAccessRecords(vars["imageName"], root.actions...)
		if ctx, err = root.auth.Authorized(ctx, access...); err != nil {
			if err, ok := err.(auth.Challenge); ok {
				err.ServeHTTP(w, r)
				w.WriteHeader(http.StatusUnauthorized)
				return
			}
			http.Error(w, err.Error(), http.StatusUnauthorized)
			return
		}
	}
	if err := root.handler(ctx, w, r); err != nil {
		logrus.Error("[Notary Server] ", err.Error())
		http.Error(w, err.Error(), err.HTTPStatus)
		return
	}
	return
}

func buildAccessRecords(repo string, actions ...string) []auth.Access {
	requiredAccess := make([]auth.Access, 0, len(actions))
	for _, action := range actions {
		requiredAccess = append(requiredAccess, auth.Access{
			Resource: auth.Resource{
				Type: "repo",
				Name: repo,
			},
			Action: action,
		})
	}
	return requiredAccess
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

// TestInt8SliceToString test the returned type and value of `int8SliceToString()`
func TestInt8SliceToString(t *testing.T) {
	expected := "abc"
	actual := int8SliceToString([]int8{97, 98, 99}) // See the ASCII table: 97 = "a" ; 98 ="b" ; 99 = "c"

	assert.IsType(t, expected, actual, "`getNetinfo()` should return a `string`")
	assert.Equal(t, expected, actual, "`int8SliceToString([]int8{97, 98, 99})` should be equal to \"abc\"")
}

// TestReadAndTrimFile test the returned type and value of `readAndTrimFile()`
// func TestReadAndTrimFile(t *testing.T) {
// }

// TODO maybe stick to the testing stdlib intead of testify
// func TestInt8SliceToString(t *testing.T) {
// 	char := []int8{49, 50, 51}
// 	expected := "123"
// 	actual := int8SliceToString(char)
// 	if actual != expected {
// 		t.Error("expected:", expected)
// 		t.Error("actual  :", actual)
// 	}
// }
<commit_msg>Update testing<commit_after>package main

import (
	"regexp"
	"testing"

	"github.com/stretchr/testify/assert"
)

// TestInt8SliceToString test the returned type and value of `int8SliceToString()`
func TestInt8SliceToString(t *testing.T) {
	expected := "abc"
	actual := int8SliceToString([]int8{97, 98, 99}) // See the ASCII table: 97 = "a" ; 98 ="b" ; 99 = "c"

	assert.IsType(t, expected, actual, "`int8SliceToString()` should return a `string`")
	assert.Equal(t, expected, actual, "`int8SliceToString([]int8{97, 98, 99})` should be equal to \"abc\"")
}

// TestReadAndTrimFile test the returned type and value of `readAndTrimFile()`
func TestReadAndTrimFile(t *testing.T) {
	expected := "package main"
	actual := readAndTrimFile("utils_test.go")

	assert.IsType(t, expected, actual, "`readAndTrimFile()` should return a `string`")
	assert.Regexp(t, regexp.MustCompile(expected), actual, "`readAndTrimFile(\"utils_test.go\")` should contain \"package main\"")
	//TODO improve tests with Equal instead of Regexp by using a temp file ?
}

// TODO maybe stick to the testing stdlib intead of testify
// func TestInt8SliceToString(t *testing.T) {
// 	char := []int8{49, 50, 51}
// 	expected := "123"
// 	actual := int8SliceToString(char)
// 	if actual != expected {
// 		t.Error("expected:", expected)
// 		t.Error("actual  :", actual)
// 	}
// }
<|endoftext|>


<commit_before>package expleto

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	// "path/filepath"
	"strings"
	"testing"
)

func func_name(string) ([]byte, error) {

	return nil, errors.New("Can't read ")

}
func TestGetDataFromFile(t *testing.T) {
	// non exists files
	nonexist_cfgFiles := []string{
		"fixtures/nonexist/config/app.json",
		"fixtures/nonexist/config/app.yml",
		"fixtures/nonexist/config/app.toml",
	}
	for _, f := range nonexist_cfgFiles {
		_, err := GetDataFromFile(f)
		if err == nil {
			t.Fatal("There wasn't raise an error")
		}
		str := fmt.Sprintf("%v", err)
		if !strings.Contains(str, "no such file or directory") {
			t.Fatal("The '" + str + "' is not contain the err 'no such file or directory'")
		}
	}

	t_file, err := ioutil.TempFile(os.TempDir(), "temp.json")
	defer os.Remove(t_file.Name())
	if err != nil {
		t.Fatal(err)
	}
	// ioutil.ReadFile = func_name
	_, err = GetDataFromFile(t_file.Name())
	if err != nil {
		t.Fatal("There shouldn't raise an error")
	}

}

func TestMockGetDataFromFile(t *testing.T) {
	stat := func(filename string) (os.FileInfo, error) {
		return nil, errors.New("err msg")
	}

	readfile := func(filename string) ([]byte, error) {
		t.Error("should not call this function")
		return nil, nil
	}

	getDataFromFile := getDataFromFileFactory(stat, readfile)

	if _, err := getDataFromFile("foo"); err.Error() != "err msg" {
		t.Error("expected an error to be thrown")
	}
}
func TestFormatError(t *testing.T) {
	err := errors.New("test 1")
	err1 := FormatError(err)
	err_str := fmt.Sprintf("%v", err)

	err1_str := fmt.Sprintf("%v", err1)
	if err_str == err1_str {
		t.Fatal("err1_str: " + err1_str + " == err_str " + err_str)
	}
}
<commit_msg>adding mock tests<commit_after>package expleto

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	// "path/filepath"
	"strings"
	"testing"
	"time"
)

type FileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modtime time.Time
	isdir   bool
	sys     interface{}
}

func New(name string, size int64, mode os.FileMode, modtime time.Time, isdir bool,
	sys interface{}) (fi os.FileInfo) {

	fi = &FileInfo{
		name:    name,
		size:    size,
		mode:    mode,
		modtime: modtime,
		isdir:   isdir,
		sys:     sys,
	}
	return
}

func (fi FileInfo) Name() string {
	return fi.name
}

func (fi FileInfo) Size() int64 {
	return fi.size
}

func (fi FileInfo) Mode() os.FileMode {
	return fi.mode
}

func (fi FileInfo) ModTime() time.Time {
	return fi.modtime
}

func (fi FileInfo) IsDir() bool {
	return fi.isdir
}

func (fi FileInfo) Sys() interface{} {
	return fi.sys
}

func TestGetDataFromFile(t *testing.T) {
	// non exists files
	nonexist_cfgFiles := []string{
		"fixtures/nonexist/config/app.json",
		"fixtures/nonexist/config/app.yml",
		"fixtures/nonexist/config/app.toml",
	}
	for _, f := range nonexist_cfgFiles {
		_, err := GetDataFromFile(f)
		if err == nil {
			t.Fatal("There wasn't raise an error")
		}
		str := fmt.Sprintf("%v", err)
		if !strings.Contains(str, "no such file or directory") {
			t.Fatal("The '" + str + "' is not contain the err 'no such file or directory'")
		}
	}

	t_file, err := ioutil.TempFile(os.TempDir(), "temp.json")
	defer os.Remove(t_file.Name())
	if err != nil {
		t.Fatal(err)
	}
	// ioutil.ReadFile = func_name
	_, err = GetDataFromFile(t_file.Name())
	if err != nil {
		t.Fatal("There shouldn't raise an error")
	}

}

func TestMockGetDataFromFile(t *testing.T) {
	// bad
	stat := func(filename string) (os.FileInfo, error) {
		return nil, errors.New("err msg")
	}

	readfile := func(filename string) ([]byte, error) {
		t.Error("should not call this function")
		return nil, nil
	}

	getDataFromFile := getDataFromFileFactory(stat, readfile)

	if _, err := getDataFromFile("foo"); err.Error() != "err msg" {
		t.Error("expected an error to be thrown")
	}

}

func TestMockGetDataFromFileG(t *testing.T) {

	//good
	stat_ok := func(filename string) (os.FileInfo, error) {
		fi := New("file.txt", int64(123), os.ModeType, time.Now(), true, nil)
		return fi, nil
	}
	readfile := func(filename string) ([]byte, error) {

		return nil, errors.New("err msg")
	}
	getDataFromFile := getDataFromFileFactory(stat_ok, readfile)

	if _, err := getDataFromFile("foo"); err.Error() != "err msg" {
		t.Error("expected an error to be thrown")
	}

}
func TestFormatError(t *testing.T) {
	err := errors.New("test 1")
	err1 := FormatError(err)
	err_str := fmt.Sprintf("%v", err)

	err1_str := fmt.Sprintf("%v", err1)
	if err_str == err1_str {
		t.Fatal("err1_str: " + err1_str + " == err_str " + err_str)
	}
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
)

var (
	errStrEmtpy = fmt.Errorf("the string is empty")
	errStrType  = fmt.Errorf("the type of the value is not string")
)

type strLenValidator struct {
	min int
	max int
}

// NewStrLenValidator returns a validator to validate that the length of the
// string must be between min and max.
func NewStrLenValidator(min, max int) Validator {
	return strLenValidator{min: min, max: max}
}

func (sv strLenValidator) Validate(v interface{}) error {
	if v == nil {
		return errStrEmtpy
	}

	s, ok := v.(string)
	if !ok {
		return errStrType
	}

	_len := len(s)
	if _len > sv.max || _len < sv.min {
		return fmt.Errorf("the length of %s is %d, not between %d and %d",
			s, _len, sv.min, sv.max)
	}
	return nil
}

type notEmptyStrValidator struct{}

func (e notEmptyStrValidator) Validate(v interface{}) error {
	if v != nil {
		if s, ok := v.(string); ok {
			if len(s) > 0 {
				return nil
			}
			return errStrEmtpy
		}
		return errStrType
	}
	return errStrEmtpy
}

// NewStrNotEmptyValidator returns a validator to validate that the value must
// not be an empty string.
func NewStrNotEmptyValidator() Validator {
	return notEmptyStrValidator{}
}
<commit_msg>Add the url validator.<commit_after>package config

import (
	"fmt"
	"net/url"
)

var (
	errStrEmtpy = fmt.Errorf("the string is empty")
	errStrType  = fmt.Errorf("the type of the value is not string")
)

type strLenValidator struct {
	min int
	max int
}

// NewStrLenValidator returns a validator to validate that the length of the
// string must be between min and max.
func NewStrLenValidator(min, max int) Validator {
	return strLenValidator{min: min, max: max}
}

func (sv strLenValidator) Validate(v interface{}) error {
	if v == nil {
		return errStrEmtpy
	}

	s, ok := v.(string)
	if !ok {
		return errStrType
	}

	_len := len(s)
	if _len > sv.max || _len < sv.min {
		return fmt.Errorf("the length of %s is %d, not between %d and %d",
			s, _len, sv.min, sv.max)
	}
	return nil
}

type notEmptyStrValidator struct{}

func (e notEmptyStrValidator) Validate(v interface{}) error {
	if v != nil {
		if s, ok := v.(string); ok {
			if len(s) > 0 {
				return nil
			}
			return errStrEmtpy
		}
		return errStrType
	}
	return errStrEmtpy
}

// NewStrNotEmptyValidator returns a validator to validate that the value must
// not be an empty string.
func NewStrNotEmptyValidator() Validator {
	return notEmptyStrValidator{}
}

type urlValidator struct{}

func (u urlValidator) Validate(v interface{}) error {
	if v == nil {
		return fmt.Errorf("the url is nil")
	}

	s, ok := v.(string)
	if !ok {
		return fmt.Errorf("the url is not string")
	}

	_, err := url.Parse(s)
	return err
}

// NewURLValidator returns a validator to validate whether a url is valid.
func NewURLValidator() Validator {
	return urlValidator{}
}
<|endoftext|>


<commit_before>package tsm1

// Timestamp encoding is adaptive and based on structure of the timestamps that are encoded.  It
// uses a combination of delta encoding, scaling and compression using simple8b, run length encoding
// as well as falling back to no compression if needed.
//
// Timestamp values to be encoded should be sorted before encoding.  When encoded, the values are
// first delta-encoded.  The first value is the starting timestamp, subsequent values are the difference.
// from the prior value.
//
// Timestamp resolution can also be in the nanosecond.  Many timestamps are monotonically increasing
// and fall on even boundaries of time such as every 10s.  When the timestamps have this structure,
// they are scaled by the largest common divisor that is also a factor of 10.  This has the effect
// of converting very large integer deltas into very small one that can be reversed by multiplying them
// by the scaling factor.
//
// Using these adjusted values, if all the deltas are the same, the time range is stored using run
// length encoding.  If run length encoding is not possible and all values are less than 1 << 60 - 1
//  (~36.5 yrs in nanosecond resolution), then the timestamps are encoded using simple8b encoding.  If
// any value exceeds the maximum values, the deltas are stored uncompressed using 8b each.
//
// Each compressed byte slice has a 1 byte header indicating the compression type.  The 4 high bits
// indicated the encoding type.  The 4 low bits are used by the encoding type.
//
// For run-length encoding, the 4 low bits store the log10 of the scaling factor.  The next 8 bytes are
// the starting timestamp, next 1-10 bytes is the delta value using variable-length encoding, finally the
// next 1-10 bytes is the count of values.
//
// For simple8b encoding, the 4 low bits store the log10 of the scaling factor.  The next 8 bytes is the
// first delta value stored uncompressed, the remaining bytes are 64bit words containg compressed delta
// values.
//
// For uncompressed encoding, the delta values are stored using 8 bytes each.

import (
	"encoding/binary"
	"fmt"
	"math"

	"github.com/jwilder/encoding/simple8b"
)

const (
	// timeUncompressed is a an uncompressed format using 8 bytes per timestamp
	timeUncompressed = 0
	// timeCompressedPackedSimple is a bit-packed format using simple8b encoding
	timeCompressedPackedSimple = 1
	// timeCompressedRLE is a run-length encoding format
	timeCompressedRLE = 2
)

// TimeEncoder encodes time.Time to byte slices.
type TimeEncoder interface {
	Write(t int64)
	Bytes() ([]byte, error)
}

type encoder struct {
	ts []uint64
}

// NewTimeEncoder returns a TimeEncoder
func NewTimeEncoder() TimeEncoder {
	return &encoder{}
}

// Write adds a time.Time to the compressed stream.
func (e *encoder) Write(t int64) {
	e.ts = append(e.ts, uint64(t))
}

func (e *encoder) reduce() (max, divisor uint64, rle bool, deltas []uint64) {
	// Compute the deltas in place to avoid allocating another slice
	deltas = e.ts
	// Starting values for a max and divisor
	max, divisor = 0, 1e12

	// Indicates whether the the deltas can be run-length encoded
	rle = true

	// Iterate in reverse so we can apply deltas in place
	for i := len(deltas) - 1; i > 0; i-- {

		// First differential encode the values
		deltas[i] = deltas[i] - deltas[i-1]

		// We also need to keep track of the max value and largest common divisor
		v := deltas[i]

		if v > max {
			max = v
		}

		for {
			// If our value is divisible by 10, break.  Otherwise, try the next smallest divisor.
			if v%divisor == 0 {
				break
			}
			divisor /= 10
		}

		// Skip the first value || see if prev = curr.  The deltas can be RLE if the are all equal.
		rle = i == len(deltas)-1 || rle && (deltas[i+1] == deltas[i])
	}
	return
}

// Bytes returns the encoded bytes of all written times.
func (e *encoder) Bytes() ([]byte, error) {
	if len(e.ts) == 0 {
		return []byte{}, nil
	}

	// Maximum and largest common divisor.  rle is true if dts (the delta timestamps),
	// are all the same.
	max, div, rle, dts := e.reduce()

	// The deltas are all the same, so we can run-length encode them
	if rle && len(e.ts) > 1 {
		return e.encodeRLE(e.ts[0], e.ts[1], div, len(e.ts))
	}

	// We can't compress this time-range, the deltas exceed 1 << 60
	if max > simple8b.MaxValue {
		return e.encodeRaw()
	}

	return e.encodePacked(div, dts)
}

func (e *encoder) encodePacked(div uint64, dts []uint64) ([]byte, error) {
	enc := simple8b.NewEncoder()
	for _, v := range dts[1:] {
		enc.Write(uint64(v) / div)
	}

	b := make([]byte, 8+1)

	// 4 high bits used for the encoding type
	b[0] = byte(timeCompressedPackedSimple) << 4
	// 4 low bits are the log10 divisor
	b[0] |= byte(math.Log10(float64(div)))

	// The first delta value
	binary.BigEndian.PutUint64(b[1:9], uint64(dts[0]))

	// The compressed deltas
	deltas, err := enc.Bytes()
	if err != nil {
		return nil, err
	}

	return append(b, deltas...), nil
}

func (e *encoder) encodeRaw() ([]byte, error) {
	b := make([]byte, 1+len(e.ts)*8)
	b[0] = byte(timeUncompressed) << 4
	for i, v := range e.ts {
		binary.BigEndian.PutUint64(b[1+i*8:1+i*8+8], uint64(v))
	}
	return b, nil
}

func (e *encoder) encodeRLE(first, delta, div uint64, n int) ([]byte, error) {
	// Large varints can take up to 10 bytes
	b := make([]byte, 1+10*3)

	// 4 high bits used for the encoding type
	b[0] = byte(timeCompressedRLE) << 4
	// 4 low bits are the log10 divisor
	b[0] |= byte(math.Log10(float64(div)))

	i := 1
	// The first timestamp
	binary.BigEndian.PutUint64(b[i:], uint64(first))
	i += 8
	// The first delta
	i += binary.PutUvarint(b[i:], uint64(delta/div))
	// The number of times the delta is repeated
	i += binary.PutUvarint(b[i:], uint64(n))

	return b[:i], nil
}

type TimeDecoder struct {
	v   int64
	i   int
	ts  []uint64
	dec simple8b.Decoder
	err error
}

func (d *TimeDecoder) Init(b []byte) {
	d.v = 0
	d.i = 0
	d.ts = d.ts[:0]
	d.err = nil
	d.decode(b)
}

func (d *TimeDecoder) Next() bool {
	if d.i >= len(d.ts) {
		return false
	}
	d.v = int64(d.ts[d.i])
	d.i++
	return true
}

func (d *TimeDecoder) Read() int64 {
	return d.v
}

func (d *TimeDecoder) Error() error {
	return d.err
}

func (d *TimeDecoder) decode(b []byte) {
	if len(b) == 0 {
		return
	}

	// Encoding type is stored in the 4 high bits of the first byte
	encoding := b[0] >> 4
	switch encoding {
	case timeUncompressed:
		d.decodeRaw(b[1:])
	case timeCompressedRLE:
		d.decodeRLE(b)
	case timeCompressedPackedSimple:
		d.decodePacked(b)
	default:
		d.err = fmt.Errorf("unknown encoding: %v", encoding)
	}
}

func (d *TimeDecoder) decodePacked(b []byte) {
	div := uint64(math.Pow10(int(b[0] & 0xF)))
	first := uint64(binary.BigEndian.Uint64(b[1:9]))

	d.dec.SetBytes(b[9:])

	d.i = 0
	deltas := d.ts[:0]
	deltas = append(deltas, first)
	for d.dec.Next() {
		deltas = append(deltas, d.dec.Read())
	}

	// Compute the prefix sum and scale the deltas back up
	for i := 1; i < len(deltas); i++ {
		dgap := deltas[i] * div
		deltas[i] = deltas[i-1] + dgap
	}

	d.i = 0
	d.ts = deltas
}

func (d *TimeDecoder) decodeRLE(b []byte) {
	var i, n int

	// Lower 4 bits hold the 10 based exponent so we can scale the values back up
	mod := int64(math.Pow10(int(b[i] & 0xF)))
	i++

	// Next 8 bytes is the starting timestamp
	first := binary.BigEndian.Uint64(b[i : i+8])
	i += 8

	// Next 1-10 bytes is our (scaled down by factor of 10) run length values
	value, n := binary.Uvarint(b[i:])

	// Scale the value back up
	value *= uint64(mod)
	i += n

	// Last 1-10 bytes is how many times the value repeats
	count, _ := binary.Uvarint(b[i:])

	// Rebuild construct the original values now
	deltas := d.ts[:0]
	for i := 0; i < int(count); i++ {
		deltas = append(deltas, value)
	}

	// Reverse the delta-encoding
	deltas[0] = first
	for i := 1; i < len(deltas); i++ {
		deltas[i] = deltas[i-1] + deltas[i]
	}

	d.i = 0
	d.ts = deltas
}

func (d *TimeDecoder) decodeRaw(b []byte) {
	d.i = 0
	d.ts = make([]uint64, len(b)/8)
	for i := range d.ts {
		d.ts[i] = binary.BigEndian.Uint64(b[i*8 : i*8+8])

		delta := d.ts[i]
		// Compute the prefix sum and scale the deltas back up
		if i > 0 {
			d.ts[i] = d.ts[i-1] + delta
		}
	}
}

func CountTimestamps(b []byte) int {
	if len(b) == 0 {
		return 0
	}

	// Encoding type is stored in the 4 high bits of the first byte
	encoding := b[0] >> 4
	switch encoding {
	case timeUncompressed:
		// Uncompressed timestamps are just 8 bytes each
		return len(b[1:]) / 8
	case timeCompressedRLE:
		// First 9 bytes are the starting timestamp and scaling factor, skip over them
		i := 9
		// Next 1-10 bytes is our (scaled down by factor of 10) run length values
		_, n := binary.Uvarint(b[9:])
		i += n
		// Last 1-10 bytes is how many times the value repeats
		count, _ := binary.Uvarint(b[i:])
		return int(count)
	case timeCompressedPackedSimple:
		// First 9 bytes are the starting timestamp and scaling factor, skip over them
		dec := simple8b.NewDecoder(b[9:])
		count := 1
		// Count the deltas
		for dec.Next() {
			count++
		}
		return count
	default:
		return 0
	}
}
<commit_msg>Optimized timestamp run-length decoding<commit_after>package tsm1

// Timestamp encoding is adaptive and based on structure of the timestamps that are encoded.  It
// uses a combination of delta encoding, scaling and compression using simple8b, run length encoding
// as well as falling back to no compression if needed.
//
// Timestamp values to be encoded should be sorted before encoding.  When encoded, the values are
// first delta-encoded.  The first value is the starting timestamp, subsequent values are the difference.
// from the prior value.
//
// Timestamp resolution can also be in the nanosecond.  Many timestamps are monotonically increasing
// and fall on even boundaries of time such as every 10s.  When the timestamps have this structure,
// they are scaled by the largest common divisor that is also a factor of 10.  This has the effect
// of converting very large integer deltas into very small one that can be reversed by multiplying them
// by the scaling factor.
//
// Using these adjusted values, if all the deltas are the same, the time range is stored using run
// length encoding.  If run length encoding is not possible and all values are less than 1 << 60 - 1
//  (~36.5 yrs in nanosecond resolution), then the timestamps are encoded using simple8b encoding.  If
// any value exceeds the maximum values, the deltas are stored uncompressed using 8b each.
//
// Each compressed byte slice has a 1 byte header indicating the compression type.  The 4 high bits
// indicated the encoding type.  The 4 low bits are used by the encoding type.
//
// For run-length encoding, the 4 low bits store the log10 of the scaling factor.  The next 8 bytes are
// the starting timestamp, next 1-10 bytes is the delta value using variable-length encoding, finally the
// next 1-10 bytes is the count of values.
//
// For simple8b encoding, the 4 low bits store the log10 of the scaling factor.  The next 8 bytes is the
// first delta value stored uncompressed, the remaining bytes are 64bit words containg compressed delta
// values.
//
// For uncompressed encoding, the delta values are stored using 8 bytes each.

import (
	"encoding/binary"
	"fmt"
	"math"

	"github.com/jwilder/encoding/simple8b"
)

const (
	// timeUncompressed is a an uncompressed format using 8 bytes per timestamp
	timeUncompressed = 0
	// timeCompressedPackedSimple is a bit-packed format using simple8b encoding
	timeCompressedPackedSimple = 1
	// timeCompressedRLE is a run-length encoding format
	timeCompressedRLE = 2
)

// TimeEncoder encodes time.Time to byte slices.
type TimeEncoder interface {
	Write(t int64)
	Bytes() ([]byte, error)
}

type encoder struct {
	ts []uint64
}

// NewTimeEncoder returns a TimeEncoder
func NewTimeEncoder() TimeEncoder {
	return &encoder{}
}

// Write adds a time.Time to the compressed stream.
func (e *encoder) Write(t int64) {
	e.ts = append(e.ts, uint64(t))
}

func (e *encoder) reduce() (max, divisor uint64, rle bool, deltas []uint64) {
	// Compute the deltas in place to avoid allocating another slice
	deltas = e.ts
	// Starting values for a max and divisor
	max, divisor = 0, 1e12

	// Indicates whether the the deltas can be run-length encoded
	rle = true

	// Iterate in reverse so we can apply deltas in place
	for i := len(deltas) - 1; i > 0; i-- {

		// First differential encode the values
		deltas[i] = deltas[i] - deltas[i-1]

		// We also need to keep track of the max value and largest common divisor
		v := deltas[i]

		if v > max {
			max = v
		}

		for {
			// If our value is divisible by 10, break.  Otherwise, try the next smallest divisor.
			if v%divisor == 0 {
				break
			}
			divisor /= 10
		}

		// Skip the first value || see if prev = curr.  The deltas can be RLE if the are all equal.
		rle = i == len(deltas)-1 || rle && (deltas[i+1] == deltas[i])
	}
	return
}

// Bytes returns the encoded bytes of all written times.
func (e *encoder) Bytes() ([]byte, error) {
	if len(e.ts) == 0 {
		return []byte{}, nil
	}

	// Maximum and largest common divisor.  rle is true if dts (the delta timestamps),
	// are all the same.
	max, div, rle, dts := e.reduce()

	// The deltas are all the same, so we can run-length encode them
	if rle && len(e.ts) > 1 {
		return e.encodeRLE(e.ts[0], e.ts[1], div, len(e.ts))
	}

	// We can't compress this time-range, the deltas exceed 1 << 60
	if max > simple8b.MaxValue {
		return e.encodeRaw()
	}

	return e.encodePacked(div, dts)
}

func (e *encoder) encodePacked(div uint64, dts []uint64) ([]byte, error) {
	enc := simple8b.NewEncoder()
	for _, v := range dts[1:] {
		enc.Write(uint64(v) / div)
	}

	b := make([]byte, 8+1)

	// 4 high bits used for the encoding type
	b[0] = byte(timeCompressedPackedSimple) << 4
	// 4 low bits are the log10 divisor
	b[0] |= byte(math.Log10(float64(div)))

	// The first delta value
	binary.BigEndian.PutUint64(b[1:9], uint64(dts[0]))

	// The compressed deltas
	deltas, err := enc.Bytes()
	if err != nil {
		return nil, err
	}

	return append(b, deltas...), nil
}

func (e *encoder) encodeRaw() ([]byte, error) {
	b := make([]byte, 1+len(e.ts)*8)
	b[0] = byte(timeUncompressed) << 4
	for i, v := range e.ts {
		binary.BigEndian.PutUint64(b[1+i*8:1+i*8+8], uint64(v))
	}
	return b, nil
}

func (e *encoder) encodeRLE(first, delta, div uint64, n int) ([]byte, error) {
	// Large varints can take up to 10 bytes
	b := make([]byte, 1+10*3)

	// 4 high bits used for the encoding type
	b[0] = byte(timeCompressedRLE) << 4
	// 4 low bits are the log10 divisor
	b[0] |= byte(math.Log10(float64(div)))

	i := 1
	// The first timestamp
	binary.BigEndian.PutUint64(b[i:], uint64(first))
	i += 8
	// The first delta
	i += binary.PutUvarint(b[i:], uint64(delta/div))
	// The number of times the delta is repeated
	i += binary.PutUvarint(b[i:], uint64(n))

	return b[:i], nil
}

type TimeDecoder struct {
	v    int64
	i, n int
	ts   []uint64
	dec  simple8b.Decoder
	err  error

	// The delta value for a run-length encoded byte slice
	rleDelta int64

	encoding byte
}

func (d *TimeDecoder) Init(b []byte) {
	d.v = 0
	d.i = 0
	d.ts = d.ts[:0]
	d.err = nil
	if len(b) > 0 {
		// Encoding type is stored in the 4 high bits of the first byte
		d.encoding = b[0] >> 4
	}
	d.decode(b)
}

func (d *TimeDecoder) Next() bool {
	if d.encoding == timeCompressedRLE {
		if d.i >= d.n {
			return false
		}
		d.i++
		d.v += d.rleDelta
		return d.i < d.n
	}

	if d.i >= len(d.ts) {
		return false
	}
	d.v = int64(d.ts[d.i])
	d.i++
	return true
}

func (d *TimeDecoder) Read() int64 {
	return d.v
}

func (d *TimeDecoder) Error() error {
	return d.err
}

func (d *TimeDecoder) decode(b []byte) {
	if len(b) == 0 {
		return
	}

	switch d.encoding {
	case timeUncompressed:
		d.decodeRaw(b[1:])
	case timeCompressedRLE:
		d.decodeRLE(b)
	case timeCompressedPackedSimple:
		d.decodePacked(b)
	default:
		d.err = fmt.Errorf("unknown encoding: %v", d.encoding)
	}
}

func (d *TimeDecoder) decodePacked(b []byte) {
	div := uint64(math.Pow10(int(b[0] & 0xF)))
	first := uint64(binary.BigEndian.Uint64(b[1:9]))

	d.dec.SetBytes(b[9:])

	d.i = 0
	deltas := d.ts[:0]
	deltas = append(deltas, first)
	for d.dec.Next() {
		deltas = append(deltas, d.dec.Read())
	}

	// Compute the prefix sum and scale the deltas back up
	for i := 1; i < len(deltas); i++ {
		dgap := deltas[i] * div
		deltas[i] = deltas[i-1] + dgap
	}

	d.i = 0
	d.ts = deltas
}

func (d *TimeDecoder) decodeRLE(b []byte) {
	var i, n int

	// Lower 4 bits hold the 10 based exponent so we can scale the values back up
	mod := int64(math.Pow10(int(b[i] & 0xF)))
	i++

	// Next 8 bytes is the starting timestamp
	first := binary.BigEndian.Uint64(b[i : i+8])
	i += 8

	// Next 1-10 bytes is our (scaled down by factor of 10) run length values
	value, n := binary.Uvarint(b[i:])

	// Scale the value back up
	value *= uint64(mod)
	i += n

	// Last 1-10 bytes is how many times the value repeats
	count, _ := binary.Uvarint(b[i:])

	d.v = int64(first - value)
	d.rleDelta = int64(value)

	d.i = -1
	d.n = int(count)
}

func (d *TimeDecoder) decodeRaw(b []byte) {
	d.i = 0
	d.ts = make([]uint64, len(b)/8)
	for i := range d.ts {
		d.ts[i] = binary.BigEndian.Uint64(b[i*8 : i*8+8])

		delta := d.ts[i]
		// Compute the prefix sum and scale the deltas back up
		if i > 0 {
			d.ts[i] = d.ts[i-1] + delta
		}
	}
}

func CountTimestamps(b []byte) int {
	if len(b) == 0 {
		return 0
	}

	// Encoding type is stored in the 4 high bits of the first byte
	encoding := b[0] >> 4
	switch encoding {
	case timeUncompressed:
		// Uncompressed timestamps are just 8 bytes each
		return len(b[1:]) / 8
	case timeCompressedRLE:
		// First 9 bytes are the starting timestamp and scaling factor, skip over them
		i := 9
		// Next 1-10 bytes is our (scaled down by factor of 10) run length values
		_, n := binary.Uvarint(b[9:])
		i += n
		// Last 1-10 bytes is how many times the value repeats
		count, _ := binary.Uvarint(b[i:])
		return int(count)
	case timeCompressedPackedSimple:
		// First 9 bytes are the starting timestamp and scaling factor, skip over them
		dec := simple8b.NewDecoder(b[9:])
		count := 1
		// Count the deltas
		for dec.Next() {
			count++
		}
		return count
	default:
		return 0
	}
}
<|endoftext|>


<commit_before>// A simple logging module that mimics the behavior of Python's logging module.
//
// All it does basically is wrap Go's logger with nice multi-level logging calls, and
// allows you to set the logging level of your app in runtime.
//
// Logging is done just like calling fmt.Sprintf:
// 		logging.Info("This object is %s and that is %s", obj, that)
//
// example output:
//	2013/05/07 01:20:26 INFO @ db.go:528: Registering plugin REPLICATION
//	2013/05/07 01:20:26 INFO @ db.go:562: Registered 6 plugins and 22 commands
//	2013/05/07 01:20:26 INFO @ slave.go:277: Running replication watchdog loop!
//	2013/05/07 01:20:26 INFO @ redis.go:49: Redis adapter listening on 0.0.0.0:2000
//	2013/05/07 01:20:26 WARN @ main.go:69: Starting adapter...
//	2013/05/07 01:20:26 INFO @ db.go:966: Finished dump load. Loaded 2 objects from dump
//	2013/05/07 01:22:26 INFO @ db.go:329: Checking persistence... 0 changes since 2m0.000297531s
//	2013/05/07 01:22:26 INFO @ db.go:337: No need to save the db. no changes...
//	2013/05/07 01:22:26 DEBUG @ db.go:341: Sleeping for 2m0s
//
package logging

import (
	"fmt"
	"io"
	"log"
	"path"
	"runtime"
	"runtime/debug"
	"strings"
	"time"
)

const (
	DEBUG    = 1
	INFO     = 2
	WARNING  = 4
	WARN     = 4
	ERROR    = 8
	NOTICE   = 16 //notice is like info but for really important stuff ;)
	CRITICAL = 32
	QUIET    = ERROR | NOTICE | CRITICAL               //setting for errors only
	NORMAL   = INFO | WARN | ERROR | NOTICE | CRITICAL // default setting - all besides debug
	ALL      = 255
	NOTHING  = 0
)

var levels_ascending = []int{DEBUG, INFO, WARNING, ERROR, NOTICE, CRITICAL}

var LevlelsByName = map[string]int{
	"DEBUG":    DEBUG,
	"INFO":     INFO,
	"WARNING":  WARN,
	"WARN":     WARN,
	"ERROR":    ERROR,
	"NOTICE":   NOTICE,
	"CRITICAL": CRITICAL,
	"QUIET":    QUIET,
	"NORMAL":   NORMAL,
	"ALL":      ALL,
	"NOTHING":  NOTHING,
}

//default logging level is ALL
var level int = ALL

// Set the logging level.
//
// Contrary to Python that specifies a minimal level, this logger is set with a bit mask
// of active levels.
//
// e.g. for INFO and ERROR use:
// 		SetLevel(logging.INFO | logging.ERROR)
//
// For everything but debug and info use:
// 		SetLevel(logging.ALL &^ (logging.INFO | logging.DEBUG))
//
func SetLevel(l int) {
	level = l
}

// Set a minimal level for loggin, setting all levels higher than this level as well.
//
// the severity order is DEBUG, INFO, WARNING, ERROR, CRITICAL
func SetMinimalLevel(l int) {

	newLevel := 0
	for _, level := range levels_ascending {
		if level >= l {
			newLevel |= level
		}
	}
	SetLevel(newLevel)

}

// Set minimal level by string, useful for config files and command line arguments. Case insensitive.
//
// Possible level names are DEBUG, INFO, WARNING, ERROR, NOTICE, CRITICAL
func SetMinimalLevelByName(l string) error {
	l = strings.ToUpper(strings.Trim(l, " "))
	level, found := LevlelsByName[l]
	if !found {
		Error("Could not set level - not found level %s", l)
		return fmt.Errorf("Invalid level %s", l)
	}

	SetMinimalLevel(level)
	return nil
}

// Set the output writer. for now it just wraps log.SetOutput()
func SetOutput(w io.Writer) {
	log.SetOutput(w)
}

//a pluggable logger interface
type LoggingHandler interface {
	SetFormatter(Formatter)
	Emit(ctx *MessageContext, message string, args ...interface{}) error
}

type strandardHandler struct {
	formatter Formatter
}

func (l *strandardHandler) SetFormatter(f Formatter) {
	l.formatter = f
}

// default handling interface - just
func (l *strandardHandler) Emit(ctx *MessageContext, message string, args ...interface{}) error {
	log.Print(l.formatter.Format(ctx, message, args...))
	return nil
}

var currentHandler LoggingHandler = &strandardHandler{
	DefaultFormatter,
}

// Set the current handler of the library. We currently support one handler, but it might be nice to have more
func SetHandler(h LoggingHandler) {
	currentHandler = h
}

func CurrentHandler() LoggingHandler {
	return currentHandler
}

type MessageContext struct {
	Level     string
	File      string
	Line      int
	TimeStamp time.Time
}

//get the stack (line + file) context to return the caller to the log
func getContext(level string) *MessageContext {

	_, file, line, _ := runtime.Caller(3)
	file = path.Base(file)

	return &MessageContext{
		Level:     level,
		File:      file,
		TimeStamp: time.Now(),
		Line:      line,
	}
}

//Output debug logging messages
func Debug(msg string, args ...interface{}) {
	if level&DEBUG != 0 {
		writeMessage("DEBUG", msg, args...)
	}
}

//format the message
func writeMessage(level string, msg string, args ...interface{}) {
	ctx := getContext(level)

	// We go over the args, and replace any function pointer with the signature
	// func() interface{} with the return value of executing it now.
	// This allows lazy evaluation of arguments which are return values
	for i, arg := range args {
		switch arg.(type) {
		case func() interface{}:
			args[i] = arg.(func() interface{})()
		default:

		}
	}

	err := currentHandler.Emit(ctx, msg, args...)
	if err != nil {
		log.Printf("Error writing log message: %s\n", err)
		log.Printf(DefaultFormatter.Format(ctx, msg, args...))
	}

}

//output INFO level messages
func Info(msg string, args ...interface{}) {

	if level&INFO != 0 {

		writeMessage("INFO", msg, args...)

	}
}

// Output WARNING level messages
func Warning(msg string, args ...interface{}) {
	if level&WARN != 0 {
		writeMessage("WARNING", msg, args...)
	}
}

// Same as Warning() but return a formatted error object, regardless of logging level
func Warningf(msg string, args ...interface{}) error {
	err := fmt.Errorf(msg, args...)
	if level&WARN != 0 {
		writeMessage("WARNING", err.Error())
	}

	return err
}

// Output ERROR level messages
func Error(msg string, args ...interface{}) {
	if level&ERROR != 0 {
		writeMessage("ERROR", msg, args...)
	}
}

// Same as Error() but also returns a new formatted error object with the message regardless of logging level
func Errorf(msg string, args ...interface{}) error {
	err := fmt.Errorf(msg, args...)
	if level&ERROR != 0 {
		writeMessage("ERROR", err.Error())
	}
	return err
}

// Output NOTICE level messages
func Notice(msg string, args ...interface{}) {
	if level&NOTICE != 0 {
		writeMessage("NOTICE", msg, args...)
	}
}

// Output a CRITICAL level message while showing a stack trace
func Critical(msg string, args ...interface{}) {
	if level&CRITICAL != 0 {
		writeMessage("CRITICAL", msg, args...)
		log.Println(string(debug.Stack()))
	}
}

// Same as critical but also returns an error object with the message regardless of logging level
func Criticalf(msg string, args ...interface{}) error {

	err := fmt.Errorf(msg, args...)
	if level&CRITICAL != 0 {
		writeMessage("CRITICAL", err.Error())
		log.Println(string(debug.Stack()))
	}
	return err
}

// Raise a PANIC while writing the stack trace to the log
func Panic(msg string, args ...interface{}) {
	log.Println(string(debug.Stack()))
	log.Panicf(msg, args...)

}

func init() {
	log.SetFlags(0)
}
<commit_msg>Added bridging of the standard logger to our own logger<commit_after>// A simple logging module that mimics the behavior of Python's logging module.
//
// All it does basically is wrap Go's logger with nice multi-level logging calls, and
// allows you to set the logging level of your app in runtime.
//
// Logging is done just like calling fmt.Sprintf:
// 		logging.Info("This object is %s and that is %s", obj, that)
//
// example output:
//	2013/05/07 01:20:26 INFO @ db.go:528: Registering plugin REPLICATION
//	2013/05/07 01:20:26 INFO @ db.go:562: Registered 6 plugins and 22 commands
//	2013/05/07 01:20:26 INFO @ slave.go:277: Running replication watchdog loop!
//	2013/05/07 01:20:26 INFO @ redis.go:49: Redis adapter listening on 0.0.0.0:2000
//	2013/05/07 01:20:26 WARN @ main.go:69: Starting adapter...
//	2013/05/07 01:20:26 INFO @ db.go:966: Finished dump load. Loaded 2 objects from dump
//	2013/05/07 01:22:26 INFO @ db.go:329: Checking persistence... 0 changes since 2m0.000297531s
//	2013/05/07 01:22:26 INFO @ db.go:337: No need to save the db. no changes...
//	2013/05/07 01:22:26 DEBUG @ db.go:341: Sleeping for 2m0s
//
package logging

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"runtime/debug"
	"strings"
	"time"
)

const (
	DEBUG    = 1
	INFO     = 2
	WARNING  = 4
	WARN     = 4
	ERROR    = 8
	NOTICE   = 16 //notice is like info but for really important stuff ;)
	CRITICAL = 32
	QUIET    = ERROR | NOTICE | CRITICAL               //setting for errors only
	NORMAL   = INFO | WARN | ERROR | NOTICE | CRITICAL // default setting - all besides debug
	ALL      = 255
	NOTHING  = 0
)

var levels_ascending = []int{DEBUG, INFO, WARNING, ERROR, NOTICE, CRITICAL}

var LevlelsByName = map[string]int{
	"DEBUG":    DEBUG,
	"INFO":     INFO,
	"WARNING":  WARN,
	"WARN":     WARN,
	"ERROR":    ERROR,
	"NOTICE":   NOTICE,
	"CRITICAL": CRITICAL,
	"QUIET":    QUIET,
	"NORMAL":   NORMAL,
	"ALL":      ALL,
	"NOTHING":  NOTHING,
}

//default logging level is ALL
var level int = ALL

// Set the logging level.
//
// Contrary to Python that specifies a minimal level, this logger is set with a bit mask
// of active levels.
//
// e.g. for INFO and ERROR use:
// 		SetLevel(logging.INFO | logging.ERROR)
//
// For everything but debug and info use:
// 		SetLevel(logging.ALL &^ (logging.INFO | logging.DEBUG))
//
func SetLevel(l int) {
	level = l
}

// Set a minimal level for loggin, setting all levels higher than this level as well.
//
// the severity order is DEBUG, INFO, WARNING, ERROR, CRITICAL
func SetMinimalLevel(l int) {

	newLevel := 0
	for _, level := range levels_ascending {
		if level >= l {
			newLevel |= level
		}
	}
	SetLevel(newLevel)

}

// Set minimal level by string, useful for config files and command line arguments. Case insensitive.
//
// Possible level names are DEBUG, INFO, WARNING, ERROR, NOTICE, CRITICAL
func SetMinimalLevelByName(l string) error {
	l = strings.ToUpper(strings.Trim(l, " "))
	level, found := LevlelsByName[l]
	if !found {
		Error("Could not set level - not found level %s", l)
		return fmt.Errorf("Invalid level %s", l)
	}

	SetMinimalLevel(level)
	return nil
}

// Set the output writer. for now it just wraps log.SetOutput()
func SetOutput(w io.Writer) {
	log.SetOutput(w)
}

//a pluggable logger interface
type LoggingHandler interface {
	SetFormatter(Formatter)
	Emit(ctx *MessageContext, message string, args ...interface{}) error
}

type strandardHandler struct {
	formatter Formatter
}

func (l *strandardHandler) SetFormatter(f Formatter) {
	l.formatter = f
}

// default handling interface - just
func (l *strandardHandler) Emit(ctx *MessageContext, message string, args ...interface{}) error {
	fmt.Fprintln(os.Stderr, l.formatter.Format(ctx, message, args...))
	return nil
}

var currentHandler LoggingHandler = &strandardHandler{
	DefaultFormatter,
}

// Set the current handler of the library. We currently support one handler, but it might be nice to have more
func SetHandler(h LoggingHandler) {
	currentHandler = h
}

func CurrentHandler() LoggingHandler {
	return currentHandler
}

type MessageContext struct {
	Level     string
	File      string
	Line      int
	TimeStamp time.Time
}

//get the stack (line + file) context to return the caller to the log
func getContext(level string, skipDepth int) *MessageContext {

	_, file, line, _ := runtime.Caller(skipDepth)
	file = path.Base(file)

	return &MessageContext{
		Level:     level,
		File:      file,
		TimeStamp: time.Now(),
		Line:      line,
	}
}

//Output debug logging messages
func Debug(msg string, args ...interface{}) {
	if level&DEBUG != 0 {
		writeMessage("DEBUG", msg, args...)
	}
}

//format the message
func writeMessage(level string, msg string, args ...interface{}) {
	writeMessageDepth(3, level, msg, args...)
}

func writeMessageDepth(depth int, level string, msg string, args ...interface{}) {
	ctx := getContext(level, depth)

	// We go over the args, and replace any function pointer with the signature
	// func() interface{} with the return value of executing it now.
	// This allows lazy evaluation of arguments which are return values
	for i, arg := range args {
		switch arg.(type) {
		case func() interface{}:
			args[i] = arg.(func() interface{})()
		default:

		}
	}

	err := currentHandler.Emit(ctx, msg, args...)
	if err != nil {
		log.Printf("Error writing log message: %s\n", err)
		log.Printf(DefaultFormatter.Format(ctx, msg, args...))
	}

}

//output INFO level messages
func Info(msg string, args ...interface{}) {

	if level&INFO != 0 {

		writeMessage("INFO", msg, args...)

	}
}

// Output WARNING level messages
func Warning(msg string, args ...interface{}) {
	if level&WARN != 0 {
		writeMessage("WARNING", msg, args...)
	}
}

// Same as Warning() but return a formatted error object, regardless of logging level
func Warningf(msg string, args ...interface{}) error {
	err := fmt.Errorf(msg, args...)
	if level&WARN != 0 {
		writeMessage("WARNING", err.Error())
	}

	return err
}

// Output ERROR level messages
func Error(msg string, args ...interface{}) {
	if level&ERROR != 0 {
		writeMessage("ERROR", msg, args...)
	}
}

// Same as Error() but also returns a new formatted error object with the message regardless of logging level
func Errorf(msg string, args ...interface{}) error {
	err := fmt.Errorf(msg, args...)
	if level&ERROR != 0 {
		writeMessage("ERROR", err.Error())
	}
	return err
}

// Output NOTICE level messages
func Notice(msg string, args ...interface{}) {
	if level&NOTICE != 0 {
		writeMessage("NOTICE", msg, args...)
	}
}

// Output a CRITICAL level message while showing a stack trace
func Critical(msg string, args ...interface{}) {
	if level&CRITICAL != 0 {
		writeMessage("CRITICAL", msg, args...)
		log.Println(string(debug.Stack()))
	}
}

// Same as critical but also returns an error object with the message regardless of logging level
func Criticalf(msg string, args ...interface{}) error {

	err := fmt.Errorf(msg, args...)
	if level&CRITICAL != 0 {
		writeMessage("CRITICAL", err.Error())
		log.Println(string(debug.Stack()))
	}
	return err
}

// Raise a PANIC while writing the stack trace to the log
func Panic(msg string, args ...interface{}) {
	log.Println(string(debug.Stack()))
	log.Panicf(msg, args...)

}

func init() {
	log.SetFlags(0)
}

// bridge bridges the logger and the default go log, with a given level
type bridge struct {
	level     int
	levelName string
}

func (lb bridge) Write(p []byte) (n int, err error) {
	if level&lb.level != 0 {
		writeMessageDepth(6, lb.levelName, string(bytes.TrimRight(p, "\r\n")))
	}
	return len(p), nil
}

// BridgeStdLog bridges all messages written using the standard library's log.Print* and makes them output
// through this logger, at a given level.
func BridgeStdLog(level int) {

	for k, l := range LevlelsByName {
		if l == level {
			b := bridge{
				level:     l,
				levelName: k,
			}

			log.SetOutput(b)
		}
	}

}
<|endoftext|>