<commit_before>// Package recon scrapes URLs for OpenGraph information.
package recon

import (
	"fmt"
	"golang.org/x/net/html"
	"io"
	"math"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"sort"
	"time"

	"image/gif"
	"image/jpeg"
	"image/png"
)

// Parser is the client object and holds the relevant information needed when parsing a URL
type Parser struct {
	client   *http.Client
	req      *http.Request
	metaTags []metaTag
	imgTags  []imgTag
}

// ParseResult is what comes back from a Parse
type ParseResult struct {
	// URL is either the URL as-passed or the defined URL (via og:url) if present
	URL string `json:"url"`

	// Host
	Host        string    `json:"host"`
	Site        string    `json:"site_name"`
	Title       string    `json:"title"`
	Type        string    `json:"type"`
	Description string    `json:"description"`
	Author      string    `json:"author"`
	Publisher   string    `json:"publisher"`
	Images      []Image   `json:"images"`
	Scraped     time.Time `json:"scraped"`
}

type metaTag struct {
	name     string
	value    string
	priority float64
}

type imgTag struct {
	url       string
	alt       string
	preferred bool
}

type Image struct {
	URL         string  `json:"url"`
	Type        string  `json:"type"`
	Width       int     `json:"width"`
	Height      int     `json:"height"`
	Alt         string  `json:"alt"`
	AspectRatio float64 `json:"aspectRatio"`
	Preferred   bool    `json:"preferred,omitempty"`
}

type byPreferred []Image

var targetedProperties = map[string]float64{
	"og:site_name":   1,
	"og:title":       1,
	"og:type":        1,
	"og:description": 1,
	"og:author":      1,
	"og:publisher":   1,
	"og:url":         1,
	"og:image":       1,

	"site_name":   0.5,
	"title":       0.5,
	"type":        0.5,
	"description": 0.5,
	"author":      0.5,
	"publisher":   0.5,
}

var propertyMap = map[string][]string{
	"URL":         {"og:url"},
	"Site":        {"og:site_name", "site_name"},
	"Title":       {"og:title", "title"},
	"Type":        {"og:type", "type"},
	"Description": {"og:description", "description"},
	"Author":      {"og:author", "author"},
	"Publisher":   {"og:publisher", "publisher"},
}

// OptimalAspectRatio is the target aspect ratio that recon favors when looking at images
var OptimalAspectRatio = 1.91

// NewParser returns a new Parser object
func NewParser() Parser {
	p := Parser{}
	return p
}

func (p *Parser) reset() {
	client := http.DefaultClient
	client.Jar, _ = cookiejar.New(nil)

	p.client = client
	p.imgTags = make([]imgTag, 0)
	p.metaTags = make([]metaTag, 0)
}

// Parse takes a url and attempts to parse it using a default minimum confidence of 0 (all information is used).
func (p *Parser) Parse(url string) (ParseResult, error) {
	return p.ParseWithConfidence(url, 0)
}

// ParseWithConfidence takes a url and attempts to parse it, only considering information with a confidence above the minimum confidence provided (should be between 0 and 1).
func (p *Parser) ParseWithConfidence(url string, confidence float64) (ParseResult, error) {
	if confidence < 0 || confidence > 1 {
		return ParseResult{}, fmt.Errorf("ParseWithConfidence: confidence should between 0 and 1 inclusive, is %d", confidence)
	}

	p.reset()

	req, err := http.NewRequest("GET", url, nil)

	resp, err := p.client.Do(req)
	if err != nil {
		return ParseResult{}, fmt.Errorf("http error: %s", err)
	}

	p.req = req

	decoder := html.NewTokenizer(resp.Body)
	done := false
	for !done {
		tt := decoder.Next()
		switch tt {
		case html.ErrorToken:
			if decoder.Err() == io.EOF {
				done = true
			} else {
				// fmt.Println(decoder.Err())
			}
		case html.SelfClosingTagToken:
			t := decoder.Token()
			if t.Data == "meta" {
				res := p.parseMeta(t)
				if res.priority > 0 {
					p.metaTags = append(p.metaTags, res)
				}
			} else if t.Data == "img" {
				res := p.parseImg(t)
				if res.url != "" {
					p.imgTags = append(p.imgTags, res)
				}
			}
		}
	}

	res := p.buildResult()

	return res, nil
}

func (p *Parser) parseMeta(t html.Token) metaTag {
	var content string
	var tag string
	var priority float64

	for _, v := range t.Attr {
		if v.Key == "property" || v.Key == "name" {
			if _priority, exists := targetedProperties[v.Val]; exists {
				tag = v.Val
				priority = _priority
			}
		} else if v.Key == "content" {
			content = v.Val
		}
	}

	if priority > 0 {
		return metaTag{name: tag, value: content, priority: priority}
	} else {
		return metaTag{}
	}
}

func (p *Parser) parseImg(t html.Token) (i imgTag) {
	for _, v := range t.Attr {
		if v.Key == "src" {
			i.url = v.Val
		} else if v.Key == "alt" {
			i.alt = v.Val
		}
	}

	if i.url != "" {
		return
	} else {
		return imgTag{}
	}
}

func (p *Parser) buildResult() ParseResult {
	res := ParseResult{}

	res.URL = p.req.URL.String()
	res.Host = p.req.URL.Host
	if canonical_url_str := p.getMaxProperty("URL"); canonical_url_str != "" {
		canonical_url, err := url.Parse(canonical_url_str)
		if err == nil {
			res.URL = canonical_url.String()
			res.Host = canonical_url.Host
		}
	}

	res.Site = p.getMaxProperty("Site")
	res.Title = p.getMaxProperty("Title")
	res.Type = p.getMaxProperty("Type")
	res.Description = p.getMaxProperty("Description")
	res.Author = p.getMaxProperty("Author")
	res.Publisher = p.getMaxProperty("Publisher")

	for _, tag := range p.metaTags {
		if tag.name == "og:image" {
			p.imgTags = append(p.imgTags, imgTag{
				url:       tag.value,
				preferred: true,
			})
		}
	}

	res.Images = p.analyzeImages()

	res.Scraped = time.Now()
	return res
}

func (p *Parser) getMaxProperty(key string) (val string) {
	max_priority := 0.0

	for _, search_tag := range propertyMap[key] {
		for _, tag := range p.metaTags {
			if tag.name == search_tag && tag.priority > max_priority {
				val = tag.value
				max_priority = tag.priority
			}
		}
	}

	return
}

func (p *Parser) analyzeImages() []Image {
	type image struct {
		url          string
		data         io.Reader
		alt          string
		content_type string
		preferred    bool
	}

	ch := make(chan image)
	returned_images := make([]Image, 0)
	found_images := 0

	for _, tag := range p.imgTags {
		go func(tag imgTag) {
			img_url, _ := url.Parse(tag.url)
			img_url = p.req.URL.ResolveReference(img_url)

			req, _ := http.NewRequest("GET", img_url.String(), nil)
			resp, _ := p.client.Do(req)

			img := image{
				url:          img_url.String(),
				content_type: resp.Header.Get("Content-Type"),

				data: resp.Body,

				alt:       tag.alt,
				preferred: tag.preferred,
			}

			ch <- img
		}(tag)
		found_images++
	}

	if found_images == 0 {
		return returned_images
	}

	for {
		select {
		case incoming_img := <-ch:
			ret_image := Image{}
			switch incoming_img.content_type {
			case "image/jpeg":
				img, _ := jpeg.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			case "image/gif":
				img, _ := gif.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			case "image/png":
				img, _ := png.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			}

			if ret_image.Height > 0 {
				ret_image.AspectRatio = round(float64(ret_image.Width)/float64(ret_image.Height), 1e-4)
			}

			ret_image.Type = incoming_img.content_type
			ret_image.URL = incoming_img.url
			ret_image.Alt = incoming_img.alt
			ret_image.Preferred = incoming_img.preferred

			returned_images = append(returned_images, ret_image)
		}

		if len(returned_images) >= len(p.imgTags) {
			break
		}
	}

	sort.Sort(byPreferred(returned_images))

	return returned_images
}

func (t byPreferred) Less(a, b int) bool {
	if t[a].Preferred && !t[b].Preferred {
		return true
	}

	if !t[a].Preferred && t[b].Preferred {
		return false
	}

	a_diff := math.Abs(t[a].AspectRatio - OptimalAspectRatio)
	b_diff := math.Abs(t[b].AspectRatio - OptimalAspectRatio)

	return a_diff < b_diff
}

func (t byPreferred) Swap(a, b int) {
	t[a], t[b] = t[b], t[a]
}

func (t byPreferred) Len() int {
	return len(t)
}

func round(a float64, prec float64) float64 {
	return math.Floor(a/prec+0.5) * prec
}
<commit_msg>Making confidence actually functional, more documentation<commit_after>// Package recon scrapes URLs for OpenGraph information.
package recon

import (
	"fmt"
	"golang.org/x/net/html"
	"io"
	"math"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"sort"
	"time"

	"image/gif"
	"image/jpeg"
	"image/png"
)

// Parser is the client object and holds the relevant information needed when parsing a URL
type Parser struct {
	client   *http.Client
	req      *http.Request
	metaTags []metaTag
	imgTags  []imgTag
}

// ParseResult is what comes back from a Parse
type ParseResult struct {
	// URL is either the URL as-passed or the defined URL (via og:url) if present
	URL string `json:"url"`

	// Host is the domain of the URL as-passed or the defined URL if present
	Host string `json:"host"`

	// Site is the name of the site as defined via og:site_name or site_name
	Site string `json:"site_name"`

	// Title is the title of the page as defined via og:title or title
	Title string `json:"title"`

	// Type is the type of the page (article, video, etc.) as defined via og:type or type.
	Type string `json:"type"`

	// Description is the description of the page as defined via og:description or description.
	Description string `json:"description"`

	// Author is the author of the page as defined via og:author or author.
	Author string `json:"author"`

	// Publisher is the publisher of the page as defined via og:publisher or publisher.
	Publisher string `json:"publisher"`

	// Images is the collection of images parsed from the page using either og:image meta tags or <img> tags.
	Images []Image `json:"images"`

	// Scraped is the time when the page was scraped (or the time Parse was run).
	Scraped time.Time `json:"scraped"`
}

type metaTag struct {
	name     string
	value    string
	priority float64
}

type imgTag struct {
	url       string
	alt       string
	preferred bool
}

// Image contains information about parsed images on the page
type Image struct {
	URL         string  `json:"url"`
	Type        string  `json:"type"`
	Width       int     `json:"width"`
	Height      int     `json:"height"`
	Alt         string  `json:"alt"`
	AspectRatio float64 `json:"aspectRatio"`
	Preferred   bool    `json:"preferred,omitempty"`
}

type byPreferred []Image

var targetedProperties = map[string]float64{
	"og:site_name":   1,
	"og:title":       1,
	"og:type":        1,
	"og:description": 1,
	"og:author":      1,
	"og:publisher":   1,
	"og:url":         1,
	"og:image":       1,

	"site_name":   0.5,
	"title":       0.5,
	"type":        0.5,
	"description": 0.5,
	"author":      0.5,
	"publisher":   0.5,
}

var propertyMap = map[string][]string{
	"URL":         {"og:url"},
	"Site":        {"og:site_name", "site_name"},
	"Title":       {"og:title", "title"},
	"Type":        {"og:type", "type"},
	"Description": {"og:description", "description"},
	"Author":      {"og:author", "author"},
	"Publisher":   {"og:publisher", "publisher"},
}

// OptimalAspectRatio is the target aspect ratio that recon favors when looking at images
var OptimalAspectRatio = 1.91

// NewParser returns a new Parser object
func NewParser() Parser {
	p := Parser{}
	return p
}

func (p *Parser) reset() {
	client := http.DefaultClient
	client.Jar, _ = cookiejar.New(nil)

	p.client = client
	p.imgTags = make([]imgTag, 0)
	p.metaTags = make([]metaTag, 0)
}

// Parse takes a url and attempts to parse it using a default minimum confidence of 0 (all information is used).
func (p *Parser) Parse(url string) (ParseResult, error) {
	return p.ParseWithConfidence(url, 0)
}

// ParseWithConfidence takes a url and attempts to parse it, only considering information with a confidence above the minimum confidence provided (should be between 0 and 1).
func (p *Parser) ParseWithConfidence(url string, confidence float64) (ParseResult, error) {
	if confidence < 0 || confidence > 1 {
		return ParseResult{}, fmt.Errorf("ParseWithConfidence: confidence should between 0 and 1 inclusive, is %d", confidence)
	}

	p.reset()

	req, err := http.NewRequest("GET", url, nil)

	resp, err := p.client.Do(req)
	if err != nil {
		return ParseResult{}, fmt.Errorf("http error: %s", err)
	}

	p.req = req

	decoder := html.NewTokenizer(resp.Body)
	done := false
	for !done {
		tt := decoder.Next()
		switch tt {
		case html.ErrorToken:
			if decoder.Err() == io.EOF {
				done = true
			} else {
				// fmt.Println(decoder.Err())
			}
		case html.SelfClosingTagToken:
			t := decoder.Token()
			if t.Data == "meta" {
				res := p.parseMeta(t)
				if res.priority > confidence {
					p.metaTags = append(p.metaTags, res)
				}
			} else if t.Data == "img" {
				res := p.parseImg(t)
				if res.url != "" {
					p.imgTags = append(p.imgTags, res)
				}
			}
		}
	}

	res := p.buildResult()

	return res, nil
}

func (p *Parser) parseMeta(t html.Token) metaTag {
	var content string
	var tag string
	var priority float64

	for _, v := range t.Attr {
		if v.Key == "property" || v.Key == "name" {
			if _priority, exists := targetedProperties[v.Val]; exists {
				tag = v.Val
				priority = _priority
			}
		} else if v.Key == "content" {
			content = v.Val
		}
	}

	if priority > 0 {
		return metaTag{name: tag, value: content, priority: priority}
	}

	return metaTag{}
}

func (p *Parser) parseImg(t html.Token) (i imgTag) {
	for _, v := range t.Attr {
		if v.Key == "src" {
			i.url = v.Val
		} else if v.Key == "alt" {
			i.alt = v.Val
		}
	}

	if i.url != "" {
		return
	}

	return imgTag{}
}

func (p *Parser) buildResult() ParseResult {
	res := ParseResult{}

	res.URL = p.req.URL.String()
	res.Host = p.req.URL.Host
	if canonical_url_str := p.getMaxProperty("URL"); canonical_url_str != "" {
		canonical_url, err := url.Parse(canonical_url_str)
		if err == nil {
			res.URL = canonical_url.String()
			res.Host = canonical_url.Host
		}
	}

	res.Site = p.getMaxProperty("Site")
	res.Title = p.getMaxProperty("Title")
	res.Type = p.getMaxProperty("Type")
	res.Description = p.getMaxProperty("Description")
	res.Author = p.getMaxProperty("Author")
	res.Publisher = p.getMaxProperty("Publisher")

	for _, tag := range p.metaTags {
		if tag.name == "og:image" {
			p.imgTags = append(p.imgTags, imgTag{
				url:       tag.value,
				preferred: true,
			})
		}
	}

	res.Images = p.analyzeImages()

	res.Scraped = time.Now()
	return res
}

func (p *Parser) getMaxProperty(key string) (val string) {
	max_priority := 0.0

	for _, search_tag := range propertyMap[key] {
		for _, tag := range p.metaTags {
			if tag.name == search_tag && tag.priority > max_priority {
				val = tag.value
				max_priority = tag.priority
			}
		}
	}

	return
}

func (p *Parser) analyzeImages() []Image {
	type image struct {
		url          string
		data         io.Reader
		alt          string
		content_type string
		preferred    bool
	}

	ch := make(chan image)
	returned_images := make([]Image, 0)
	found_images := 0

	for _, tag := range p.imgTags {
		go func(tag imgTag) {
			img_url, _ := url.Parse(tag.url)
			img_url = p.req.URL.ResolveReference(img_url)

			req, _ := http.NewRequest("GET", img_url.String(), nil)
			resp, _ := p.client.Do(req)

			img := image{
				url:          img_url.String(),
				content_type: resp.Header.Get("Content-Type"),

				data: resp.Body,

				alt:       tag.alt,
				preferred: tag.preferred,
			}

			ch <- img
		}(tag)
		found_images++
	}

	if found_images == 0 {
		return returned_images
	}

	for {
		select {
		case incoming_img := <-ch:
			ret_image := Image{}
			switch incoming_img.content_type {
			case "image/jpeg":
				img, _ := jpeg.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			case "image/gif":
				img, _ := gif.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			case "image/png":
				img, _ := png.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			}

			if ret_image.Height > 0 {
				ret_image.AspectRatio = round(float64(ret_image.Width)/float64(ret_image.Height), 1e-4)
			}

			ret_image.Type = incoming_img.content_type
			ret_image.URL = incoming_img.url
			ret_image.Alt = incoming_img.alt
			ret_image.Preferred = incoming_img.preferred

			returned_images = append(returned_images, ret_image)
		}

		if len(returned_images) >= len(p.imgTags) {
			break
		}
	}

	sort.Sort(byPreferred(returned_images))

	return returned_images
}

func (t byPreferred) Less(a, b int) bool {
	if t[a].Preferred && !t[b].Preferred {
		return true
	}

	if !t[a].Preferred && t[b].Preferred {
		return false
	}

	a_diff := math.Abs(t[a].AspectRatio - OptimalAspectRatio)
	b_diff := math.Abs(t[b].AspectRatio - OptimalAspectRatio)

	return a_diff < b_diff
}

func (t byPreferred) Swap(a, b int) {
	t[a], t[b] = t[b], t[a]
}

func (t byPreferred) Len() int {
	return len(t)
}

func round(a float64, prec float64) float64 {
	return math.Floor(a/prec+0.5) * prec
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Aubrey Barnard.  This is free software.  See
// LICENSE.txt for details.

// Go package that provides an interface to the Fortran implementation
// of the L-BFGS-B optimization algorithm.  The Fortran code is provided
// as a C-compatible library and this is the Go API for that library.
// There is a sliver of C code (in lbfgsb_go_interface.*) needed to
// connect this Go package to the Fortran library.

package lbfgsb

// Declarations for Cgo

// #cgo LDFLAGS: -L. -llbfgsb -lgfortran -lm
// #include "lbfgsb_go_interface.h"
import "C"

import (
	"fmt"
	"math"
	"reflect"
	"unsafe"
)

// Private constants
const (
	// 3 or so 80-character lines
	bufferSize = 250
)

// Lbfgsb provides the functionality of the Fortran L-BFGS-B optimizer
// as a Go object.
type Lbfgsb struct {
	// Problem specification.  Bounds may be nil or allocated fully.
	// Individual bounds may be omitted by placing NaNs or Infs.
	lowerBounds []float64
	upperBounds []float64
	// Parameters
	approximationSize int
	fTolerance        float64
	gTolerance        float64
	debug             int
}

// Minimize optimizes the given objective using the L-BFGS-B algorithm.
// Implements OptimizationFunctionMinimizer.Minimize.
func (lbfgsb Lbfgsb) Minimize(
	objective FunctionWithGradient,
	initialPoint []float64,
	parameters map[string]interface{}) (
		minimum PointValueGradient,
		exitStatus ExitStatus) {

	// TODO OMG! split this out into some helper functions

	// Check everyone agrees on the dimensionality
	dim := len(initialPoint)
	dim_c := C.int(dim)
	if lbfgsb.lowerBounds != nil && len(lbfgsb.lowerBounds) != dim {
		exitStatus.Code = USAGE_ERROR
		exitStatus.Message = fmt.Sprintf("Dimensionality disagreement: initialPoint: %v, lowerBounds: %v", dim, len(lbfgsb.lowerBounds))
	}
	if lbfgsb.upperBounds != nil && len(lbfgsb.upperBounds) != dim {
		exitStatus.Code = USAGE_ERROR
		exitStatus.Message = fmt.Sprintf("Dimensionality disagreement: initialPoint: %v, upperBounds: %v", dim, len(lbfgsb.upperBounds))
	}

	// Process the parameters.  Argument 'parameters' overrides but does
	// not replace class-level parameters.  The bounds are not part of
	// the parameters, they are part of the problem specification.
	var paramVal interface{}
	var ok bool
	// Approximation size
	approximationSize := lbfgsb.approximationSize
	if paramVal, ok = parameters["approximationSize"]; ok {
		approximationSize, ok = paramVal.(int)
		if !ok || approximationSize <= 0 {
			exitStatus.Code = USAGE_ERROR
			exitStatus.Message = fmt.Sprintf("Bad parameter value: approximationSize: %v.  Expected integer >= 1.", paramVal)
			return
		}
	}
	// F tolerance
	fTolerance := lbfgsb.fTolerance
	if paramVal, ok = parameters["fTolerance"]; ok {
		fTolerance, ok = paramVal.(float64)
		if !ok || fTolerance <= 0.0 {
			exitStatus.Code = USAGE_ERROR
			exitStatus.Message = fmt.Sprintf("Bad parameter value: fTolerance: %v.  Expected float >= 0.", paramVal)
			return
		}
	}
	// G tolerance
	gTolerance := lbfgsb.gTolerance
	if paramVal, ok = parameters["gTolerance"]; ok {
		gTolerance, ok = paramVal.(float64)
		if !ok || gTolerance <= 0.0 {
			exitStatus.Code = USAGE_ERROR
			exitStatus.Message = fmt.Sprintf("Bad parameter value: gTolerance: %v.  Expected float >= 0.", paramVal)
			return
		}
	}
	// Debug
	debug := lbfgsb.debug
	if paramVal, ok = parameters["debug"]; ok {
		debug, ok = paramVal.(int)
		if !ok || debug < 0 {
			exitStatus.Code = USAGE_ERROR
			exitStatus.Message = fmt.Sprintf("Bad parameter value: debug: %v.  Expected integer >= 0.", paramVal)
			return
		}
	}

	// Set up bounds control.  Use a C-compatible type.
	boundsControl := make([]int32, dim)
	if lbfgsb.lowerBounds != nil {
		for index, bound := range lbfgsb.lowerBounds {
			if !math.IsNaN(bound) && !math.IsInf(bound, -1) {
				boundsControl[index] = 1
			}
		}
	}
	if lbfgsb.upperBounds != nil {
		for index, bound := range lbfgsb.upperBounds {
			if !math.IsNaN(bound) && !math.IsInf(bound, -1) {
				// Map 0 -> 3, 1 -> 2
				boundsControl[index] = 3 - boundsControl[index]
			}
		}
	}

	// Set up callback
	callbackData := &callbackData{objective: objective}
	callbackData_c := unsafe.Pointer(callbackData)

	// Allocate arrays for return value
	minimum.X = make([]float64, dim)
	minimum.G = make([]float64, dim)

	// Convert parameters for C
	approximationSize_c := C.int(approximationSize)
	fTolerance_c := C.double(fTolerance)
	gTolerance_c := C.double(gTolerance)
	debug_c := C.int(debug)

	// Prepare buffers and arrays for C.  Avoid allocation in C land by
	// allocating compatible things in Go and passing their addresses.
	// The following arrays may not be iteroperably type-safe but this
	// is how they did it on the Cgo page: http://golang.org/cmd/cgo/.
	// (One could always allocate slices of C types, pass those, and
	// then copy out and convert the contents on return.)
	var boundsControl_c *C.int = (*C.int)(&boundsControl[0])
	var lowerBounds_c *C.double = (*C.double)(&lbfgsb.lowerBounds[0])
	var upperBounds_c *C.double = (*C.double)(&lbfgsb.upperBounds[0])
	var x0_c *C.double = (*C.double)(&initialPoint[0])
	var minX_c *C.double = (*C.double)(&minimum.X[0])
	var minF_c *C.double = (*C.double)(&minimum.F)
	var minG_c *C.double = (*C.double)(&minimum.G[0])
	// Status message
	statusMessageLength_c := C.int(bufferSize)
	var statusMessageBuffer [bufferSize]C.char
	statusMessage_c := (*C.char)(&statusMessageBuffer[0])

	// Call the actual L-BFGS-B procedure
	statusCode_c := C.lbfgsb_minimize_c(
		callbackData_c, dim_c,
		boundsControl_c, lowerBounds_c, upperBounds_c,
		approximationSize_c, fTolerance_c, gTolerance_c,
		x0_c, minX_c, minF_c, minG_c,
		statusMessageLength_c, statusMessage_c, debug_c,
	)

	// Convert outputs
	// Exit status codes match between ExitStatusCode and the C enum
	exitStatus.Code = ExitStatusCode(statusCode_c)
	exitStatus.Message = C.GoStringN(statusMessage_c, statusMessageLength_c)
	// Minimum already populated because pointers to its members were
	// passed into C/Fortran

	return
}

// callbackData is a container for the actual objective function and
// related data.
type callbackData struct {
	objective FunctionWithGradient
}

// go_objective_function_callback is an adapter between the C callback
// and the Go callback for evaluating the objective function.  Exported
// to C for use as a function pointer.  Must match the signature of
// objective_function_type in lbfgsb_c.h.
//
//export go_objective_function_callback
func go_objective_function_callback(
	dim_c C.int, point_c, value_c *C.double,
	callbackData_c unsafe.Pointer,
	statusMessageLength_c C.int, statusMessage_c *C.char) (
		statusCode_c C.int) {

	var point []float64

	// Convert inputs
	dim := int(dim_c)
	wrapCArrayAsGoSlice_Float64(point_c, dim, &point)
	cbData := (*callbackData)(callbackData_c)

	// Evaluate the objective function
	value := cbData.objective.Evaluate(point)

	// Convert outputs
	*value_c = C.double(value)

	return
}

// go_objective_gradient_callback is an adapter between the C callback
// and the Go callback for evaluating the objective gradient.  Exported
// to C for use as a function pointer.  Must match the signature of
// objective_gradient_type in gd_c.h.
//
//export go_objective_gradient_callback
func go_objective_gradient_callback(
	dim_c C.int, point_c, gradient_c *C.double,
	callbackData_c unsafe.Pointer,
	statusMessageLength_c C.int, statusMessage_c *C.char) (
		statusCode_c C.int) {

	var point, gradient, gradRet []float64

	// Convert inputs
	dim := int(dim_c)
	wrapCArrayAsGoSlice_Float64(point_c, dim, &point)
	cbData := (*callbackData)(callbackData_c)

	// Evaluate the gradient of the objective function
	gradRet = cbData.objective.EvaluateGradient(point)

	// Convert outputs
	wrapCArrayAsGoSlice_Float64(gradient_c, dim, &gradient)
	copy(gradient, gradRet)

	return
}

// wrapCArrayAsGoSlice_Float64 allows a C array to be treated as a Go
// slice.  Based on https://code.google.com/p/go-wiki/wiki/cgo.  This
// only works if the Go and C types happen to be interoperable (binary
// compatible), but that seems to be the case so far.
func wrapCArrayAsGoSlice_Float64(array *C.double, length int,
	slice *[]float64) {

	sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(slice))
	sliceHeader.Cap = length
	sliceHeader.Len = length
	sliceHeader.Data = uintptr(unsafe.Pointer(array))
}
<commit_msg>Implemented Lbfgsb solver construction and setup.<commit_after>// Copyright (c) 2013 Aubrey Barnard.  This is free software.  See
// LICENSE.txt for details.

// Go package that provides an interface to the Fortran implementation
// of the L-BFGS-B optimization algorithm.  The Fortran code is provided
// as a C-compatible library and this is the Go API for that library.
// There is a sliver of C code (in lbfgsb_go_interface.*) needed to
// connect this Go package to the Fortran library.

package lbfgsb

// Declarations for Cgo

// #cgo LDFLAGS: -L. -llbfgsb -lgfortran -lm
// #include "lbfgsb_go_interface.h"
import "C"

import (
	"fmt"
	"math"
	"reflect"
	"unsafe"
)

// Private constants
const (
	// 3 or so 80-character lines
	bufferSize = 250
)

// NewLbfgsb creates, initializes, and returns a new Lbfgsb solver
// object.  Equivalent to 'new(Lbfgsb).Init(dimensionality)'.  A
// zero-value Lbfgsb object is valid and needs no explicit construction.
// However, this constructor is convenient and explicit.
func NewLbfgsb(dimensionality int) *Lbfgsb {
	return new(Lbfgsb).Init(dimensionality)
}

// Lbfgsb provides the functionality of the Fortran L-BFGS-B optimizer
// as a Go object.  A Lbfgsb solver object contains the setup for an
// optimization problem of a particular dimensionality.  It stores
// bounds, parameters, and results so it is relatively lightweight
// (especially if no bounds are specified).  It can be re-used for other
// problems with the same dimensionality, but using a different solver
// object for each problem is probably better organization.  A
// zero-value Lbfgsb object is valid and needs no explicit construction.
// A solver object will perform unconstrained optimization unless bounds
// are set.
type Lbfgsb struct {
	// Dimensionality of the problem.  Zero is an invalid
	// dimensionality, so this also serves as an indicator of whether
	// this object has been initialized for computation.  Once the
	// dimensionality has been set it cannot be changed.  To be ready
	// for computation the following must be greater than zero:
	// dimensionality, approximationSize, fTolerance, gTolerance.
	dimensionality int

	// Problem specification.  Bounds may be nil or allocated fully.
	// Individual bounds may be omitted by placing NaNs or Infs.
	lowerBounds []float64
	upperBounds []float64

	// Parameters
	approximationSize int
	fTolerance        float64
	gTolerance        float64
	debugLevel        int
}

// Init initializes this Lbfgsb solver for problems of the given
// dimensionality.  Also sets default parameters that are not zero
// values.  Returns this for method chaining.  Ignores calls subsequent
// to the first (because a solver is intended for only a particular
// dimensionality).
func (lbfgsb Lbfgsb) Init(dimensionality int) *Lbfgsb {
	// Only initialize if not previously initialized
	if lbfgsb.dimensionality == 0 {
		// Check for a valid dimensionality
		if dimensionality <= 0 {
			panic(fmt.Errorf("Lbfgsb: Optimization problem dimensionality %d <= 0.  Expected > 0.", dimensionality))
		}
		// Set up the solver.  Protect previous values so Init can be
		// called after other methods.
		lbfgsb.dimensionality = dimensionality
		if lbfgsb.approximationSize == 0 {
			lbfgsb.approximationSize = 5
		}
		if lbfgsb.fTolerance == 0.0 {
			lbfgsb.fTolerance = 1e-6
		}
		if lbfgsb.gTolerance == 0.0 {
			lbfgsb.gTolerance = 1e-6
		}
	}
	return &lbfgsb
}

// SetBounds sets the upper and lower bounds on the individual
// dimensions to the given intervals resulting in a constrained
// optimization problem.  Individual bounds may be (+/-)Inf.
func (lbfgsb Lbfgsb) SetBounds(bounds [][2]float64) *Lbfgsb {
	// Ensure object is initialized
	lbfgsb.Init(len(bounds))
	// Check dimensionality
	if lbfgsb.dimensionality != len(bounds) {
		panic(fmt.Errorf("Lbfgsb: Dimensionality of the bounds (%d) does not match the dimensionality of the solver (%d).", len(bounds), lbfgsb.dimensionality))
	}

	lbfgsb.lowerBounds = make([]float64, lbfgsb.dimensionality)
	lbfgsb.upperBounds = make([]float64, lbfgsb.dimensionality)
	for i, interval := range bounds {
		lbfgsb.lowerBounds[i] = interval[0]
		lbfgsb.upperBounds[i] = interval[1]
	}
	return &lbfgsb
}

// SetBoundsAll sets the bounds of all the dimensions to [lower,upper].
// Init must be called first to set the dimensionality.
func (lbfgsb Lbfgsb) SetBoundsAll(lower, upper float64) *Lbfgsb {
	// Check object has been initialized
	if lbfgsb.dimensionality == 0 {
		panic(fmt.Errorf("Lbfgsb: Init() must be called before SetAllBounds()."))
	}

	lbfgsb.lowerBounds = make([]float64, lbfgsb.dimensionality)
	lbfgsb.upperBounds = make([]float64, lbfgsb.dimensionality)
	for i := 0; i < lbfgsb.dimensionality; i++ {
		lbfgsb.lowerBounds[i] = lower
		lbfgsb.upperBounds[i] = upper
	}
	return &lbfgsb
}

// SetBoundsSparse sets the bounds to only those in the given map;
// others are unbounded.  Each entry in the map is a (zero-based)
// dimension index mapped to a slice representing an interval.
// Individual bounds may be (+/-)Inf.  Init must be called first to set
// the dimensionality.
//
// The slice is interpreted as an interval as follows:
//
//     nil | []: [-Inf, +Inf]
//     [x]: [-|x|, |x|]
//     [l, u, ...]: [l, u]
func (lbfgsb Lbfgsb) SetBoundsSparse(sparseBounds map[int][]float64) *Lbfgsb {
	// Check object has been initialized
	if lbfgsb.dimensionality == 0 {
		panic(fmt.Errorf("Lbfgsb: Init() must be called before SetAllBounds()."))
	}

	// If no bounds are given, clear the bounds
	if sparseBounds == nil || len(sparseBounds) == 0 {
		return lbfgsb.ClearBounds()
	}

	lbfgsb.lowerBounds = make([]float64, lbfgsb.dimensionality)
	lbfgsb.upperBounds = make([]float64, lbfgsb.dimensionality)
	nInf := math.Inf(-1)
	pInf := math.Inf(+1)
	for i := 0; i < lbfgsb.dimensionality; i++ {
		interval, exists := sparseBounds[i]
		if exists {
			if interval == nil || len(interval) == 0 {
				lbfgsb.lowerBounds[i] = nInf
				lbfgsb.upperBounds[i] = pInf
			} else if len(interval) == 1 {
				lbfgsb.upperBounds[i] = math.Abs(interval[0])
				lbfgsb.lowerBounds[i] = -lbfgsb.upperBounds[i]
			} else {
				lbfgsb.lowerBounds[i] = interval[0]
				lbfgsb.upperBounds[i] = interval[1]
			}
		} else {
			lbfgsb.lowerBounds[i] = nInf
			lbfgsb.upperBounds[i] = pInf
		}
	}
	return &lbfgsb
}

// ClearBounds clears all bounds resulting in an unconstrained
// optimization problem.
func (lbfgsb Lbfgsb) ClearBounds() *Lbfgsb {
	lbfgsb.lowerBounds = nil
	lbfgsb.upperBounds = nil
	return &lbfgsb
}

// SetApproximationSize sets the amount of history (points and
// gradients) stored and used to approximate the inverse Hessian matrix.
// More history allows better approximation at the cost of more memory.
// The recommended range is [3,20].  Defaults to 5.
func (lbfgsb Lbfgsb) SetApproximationSize(size int) *Lbfgsb {
	if size <= 0 {
		panic(fmt.Errorf("Lbfgsb: Approximation size %d <= 0.  Expected > 0.", size))
	}
	lbfgsb.approximationSize = size
	return &lbfgsb
}

// SetFTolerance sets the tolerance of the precision of the objective
// function required for convergence.  Defaults to 1e-6.
func (lbfgsb Lbfgsb) SetFTolerance(fTolerance float64) *Lbfgsb {
	if fTolerance <= 0.0 {
		panic(fmt.Errorf("Lbfgsb: F tolerance %g <= 0.  Expected > 0.", fTolerance))
	}
	lbfgsb.fTolerance = fTolerance
	return &lbfgsb
}

// SetGTolerance sets the tolerance of the precision of the objective
// gradient required for convergence.  Defaults to 1e-6.
func (lbfgsb Lbfgsb) SetGTolerance(gTolerance float64) *Lbfgsb {
	if gTolerance <= 0.0 {
		panic(fmt.Errorf("Lbfgsb: G tolerance %g <= 0.  Expected > 0.", gTolerance))
	}
	lbfgsb.gTolerance = gTolerance
	return &lbfgsb
}

// SetDebugLevel sets the level of output verbosity.  Defaults to 0, no
// output.
func (lbfgsb Lbfgsb) SetDebugLevel(level int) *Lbfgsb {
	if level < 0 {
		panic(fmt.Errorf("Lbfgsb: Debug level %d < 0.  Expected >= 0.", level))
	}
	lbfgsb.debugLevel = level
	return &lbfgsb
}

// Minimize optimizes the given objective using the L-BFGS-B algorithm.
// Implements OptimizationFunctionMinimizer.Minimize.
func (lbfgsb Lbfgsb) Minimize(
	objective FunctionWithGradient,
	initialPoint []float64,
	parameters map[string]interface{}) (
		minimum PointValueGradient,
		exitStatus ExitStatus) {

	// Make sure object has been initialized
	lbfgsb.Init(len(initialPoint))

	// TODO OMG! split this out into some helper functions

	// Check dimensionality
	dim := len(initialPoint)
	dim_c := C.int(dim)
	if lbfgsb.dimensionality != dim {
		exitStatus.Code = USAGE_ERROR
		exitStatus.Message = fmt.Sprintf("Lbfgsb: Dimensionality of the initial point (%d) does not match the dimensionality of the solver (%d).", dim, lbfgsb.dimensionality)
	}

	// Process the parameters.  Argument 'parameters' overrides but does
	// not replace class-level parameters.  The bounds are not part of
	// the parameters, they are part of the problem specification.
	var paramVal interface{}
	var ok bool
	// Approximation size
	approximationSize := lbfgsb.approximationSize
	if paramVal, ok = parameters["approximationSize"]; ok {
		approximationSize, ok = paramVal.(int)
		if !ok || approximationSize <= 0 {
			exitStatus.Code = USAGE_ERROR
			exitStatus.Message = fmt.Sprintf("Bad parameter value: approximationSize: %v.  Expected integer >= 1.", paramVal)
			return
		}
	}
	// F tolerance
	fTolerance := lbfgsb.fTolerance
	if paramVal, ok = parameters["fTolerance"]; ok {
		fTolerance, ok = paramVal.(float64)
		if !ok || fTolerance <= 0.0 {
			exitStatus.Code = USAGE_ERROR
			exitStatus.Message = fmt.Sprintf("Bad parameter value: fTolerance: %v.  Expected float >= 0.", paramVal)
			return
		}
	}
	// G tolerance
	gTolerance := lbfgsb.gTolerance
	if paramVal, ok = parameters["gTolerance"]; ok {
		gTolerance, ok = paramVal.(float64)
		if !ok || gTolerance <= 0.0 {
			exitStatus.Code = USAGE_ERROR
			exitStatus.Message = fmt.Sprintf("Bad parameter value: gTolerance: %v.  Expected float >= 0.", paramVal)
			return
		}
	}
	// Debug level
	debugLevel := lbfgsb.debugLevel
	if paramVal, ok = parameters["debugLevel"]; ok {
		debugLevel, ok = paramVal.(int)
		if !ok || debugLevel < 0 {
			exitStatus.Code = USAGE_ERROR
			exitStatus.Message = fmt.Sprintf("Bad parameter value: debugLevel: %v.  Expected integer >= 0.", paramVal)
			return
		}
	}

	// Set up bounds control.  Use a C-compatible type.
	boundsControl := make([]int32, dim)
	if lbfgsb.lowerBounds != nil {
		for index, bound := range lbfgsb.lowerBounds {
			if !math.IsNaN(bound) && !math.IsInf(bound, -1) {
				boundsControl[index] = 1
			}
		}
	}
	if lbfgsb.upperBounds != nil {
		for index, bound := range lbfgsb.upperBounds {
			if !math.IsNaN(bound) && !math.IsInf(bound, -1) {
				// Map 0 -> 3, 1 -> 2
				boundsControl[index] = 3 - boundsControl[index]
			}
		}
	}

	// Set up callback
	callbackData := &callbackData{objective: objective}
	callbackData_c := unsafe.Pointer(callbackData)

	// Allocate arrays for return value
	minimum.X = make([]float64, dim)
	minimum.G = make([]float64, dim)

	// Convert parameters for C
	approximationSize_c := C.int(approximationSize)
	fTolerance_c := C.double(fTolerance)
	gTolerance_c := C.double(gTolerance)
	debugLevel_c := C.int(debugLevel)

	// Prepare buffers and arrays for C.  Avoid allocation in C land by
	// allocating compatible things in Go and passing their addresses.
	// The following arrays may not be iteroperably type-safe but this
	// is how they did it on the Cgo page: http://golang.org/cmd/cgo/.
	// (One could always allocate slices of C types, pass those, and
	// then copy out and convert the contents on return.)
	var boundsControl_c *C.int = (*C.int)(&boundsControl[0])
	var lowerBounds_c *C.double = (*C.double)(&lbfgsb.lowerBounds[0])
	var upperBounds_c *C.double = (*C.double)(&lbfgsb.upperBounds[0])
	var x0_c *C.double = (*C.double)(&initialPoint[0])
	var minX_c *C.double = (*C.double)(&minimum.X[0])
	var minF_c *C.double = (*C.double)(&minimum.F)
	var minG_c *C.double = (*C.double)(&minimum.G[0])
	// Status message
	statusMessageLength_c := C.int(bufferSize)
	var statusMessageBuffer [bufferSize]C.char
	statusMessage_c := (*C.char)(&statusMessageBuffer[0])

	// Call the actual L-BFGS-B procedure
	statusCode_c := C.lbfgsb_minimize_c(
		callbackData_c, dim_c,
		boundsControl_c, lowerBounds_c, upperBounds_c,
		approximationSize_c, fTolerance_c, gTolerance_c,
		x0_c, minX_c, minF_c, minG_c,
		statusMessageLength_c, statusMessage_c, debugLevel_c,
	)

	// Convert outputs
	// Exit status codes match between ExitStatusCode and the C enum
	exitStatus.Code = ExitStatusCode(statusCode_c)
	exitStatus.Message = C.GoStringN(statusMessage_c, statusMessageLength_c)
	// Minimum already populated because pointers to its members were
	// passed into C/Fortran

	return
}

// callbackData is a container for the actual objective function and
// related data.
type callbackData struct {
	objective FunctionWithGradient
}

// go_objective_function_callback is an adapter between the C callback
// and the Go callback for evaluating the objective function.  Exported
// to C for use as a function pointer.  Must match the signature of
// objective_function_type in lbfgsb_c.h.
//
//export go_objective_function_callback
func go_objective_function_callback(
	dim_c C.int, point_c, value_c *C.double,
	callbackData_c unsafe.Pointer,
	statusMessageLength_c C.int, statusMessage_c *C.char) (
		statusCode_c C.int) {

	var point []float64

	// Convert inputs
	dim := int(dim_c)
	wrapCArrayAsGoSlice_Float64(point_c, dim, &point)
	cbData := (*callbackData)(callbackData_c)

	// Evaluate the objective function
	value := cbData.objective.Evaluate(point)

	// Convert outputs
	*value_c = C.double(value)

	return
}

// go_objective_gradient_callback is an adapter between the C callback
// and the Go callback for evaluating the objective gradient.  Exported
// to C for use as a function pointer.  Must match the signature of
// objective_gradient_type in gd_c.h.
//
//export go_objective_gradient_callback
func go_objective_gradient_callback(
	dim_c C.int, point_c, gradient_c *C.double,
	callbackData_c unsafe.Pointer,
	statusMessageLength_c C.int, statusMessage_c *C.char) (
		statusCode_c C.int) {

	var point, gradient, gradRet []float64

	// Convert inputs
	dim := int(dim_c)
	wrapCArrayAsGoSlice_Float64(point_c, dim, &point)
	cbData := (*callbackData)(callbackData_c)

	// Evaluate the gradient of the objective function
	gradRet = cbData.objective.EvaluateGradient(point)

	// Convert outputs
	wrapCArrayAsGoSlice_Float64(gradient_c, dim, &gradient)
	copy(gradient, gradRet)

	return
}

// wrapCArrayAsGoSlice_Float64 allows a C array to be treated as a Go
// slice.  Based on https://code.google.com/p/go-wiki/wiki/cgo.  This
// only works if the Go and C types happen to be interoperable (binary
// compatible), but that seems to be the case so far.
func wrapCArrayAsGoSlice_Float64(array *C.double, length int,
	slice *[]float64) {

	sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(slice))
	sliceHeader.Cap = length
	sliceHeader.Len = length
	sliceHeader.Data = uintptr(unsafe.Pointer(array))
}
<|endoftext|>


<commit_before>package irc

import (
	"bytes"
	"errors"
	"strings"
)

var tagDecodeSlashMap = map[rune]rune{
	':':  ';',
	's':  ' ',
	'\\': '\\',
	'r':  '\r',
	'n':  '\n',
}

var tagEncodeMap = map[rune]string{
	';':  "\\:",
	' ':  "\\s",
	'\\': "\\\\",
	'\r': "\\r",
	'\n': "\\n",
}

var (
	// ErrZeroLengthMessage is returned when parsing if the input is
	// zero-length.
	ErrZeroLengthMessage = errors.New("irc: Cannot parse zero-length message")

	// ErrMissingDataAfterPrefix is returned when parsing if there is
	// no message data after the prefix.
	ErrMissingDataAfterPrefix = errors.New("irc: No message data after prefix")

	// ErrMissingDataAfterTags is returned when parsing if there is no
	// message data after the tags.
	ErrMissingDataAfterTags = errors.New("irc: No message data after tags")

	// ErrMissingCommand is returned when parsing if there is no
	// command in the parsed message.
	ErrMissingCommand = errors.New("irc: Missing message command")
)

// TagValue represents the value of a tag.
type TagValue string

// ParseTagValue parses a TagValue from the connection. If you need to
// set a TagValue, you probably want to just set the string itself, so
// it will be encoded properly.
func ParseTagValue(v string) TagValue {
	ret := &bytes.Buffer{}

	input := bytes.NewBufferString(v)

	for {
		c, _, err := input.ReadRune()
		if err != nil {
			break
		}

		if c == '\\' {
			c2, _, err := input.ReadRune()
			if err != nil {
				ret.WriteRune(c)
				break
			}

			if replacement, ok := tagDecodeSlashMap[c2]; ok {
				ret.WriteRune(replacement)
			} else {
				ret.WriteRune(c)
				ret.WriteRune(c2)
			}
		} else {
			ret.WriteRune(c)
		}
	}

	return TagValue(ret.String())
}

// Encode converts a TagValue to the format in the connection.
func (v TagValue) Encode() string {
	ret := &bytes.Buffer{}

	for _, c := range v {
		if replacement, ok := tagEncodeMap[c]; ok {
			ret.WriteString(replacement)
		} else {
			ret.WriteRune(c)
		}
	}

	return ret.String()
}

// Tags represents the IRCv3 message tags.
type Tags map[string]TagValue

// ParseTags takes a tag string and parses it into a tag map. It will
// always return a tag map, even if there are no valid tags.
func ParseTags(line string) Tags {
	ret := Tags{}

	tags := strings.Split(line, ";")
	for _, tag := range tags {
		parts := strings.SplitN(tag, "=", 2)
		if len(parts) < 2 {
			ret[parts[0]] = ""
			continue
		}

		ret[parts[0]] = ParseTagValue(parts[1])
	}

	return ret
}

// GetTag is a convenience method to look up a tag in the map.
func (t Tags) GetTag(key string) (string, bool) {
	ret, ok := t[key]
	return string(ret), ok
}

// Copy will create a new copy of all IRC tags attached to this
// message.
func (t Tags) Copy() Tags {
	ret := Tags{}

	for k, v := range t {
		ret[k] = v
	}

	return ret
}

// String ensures this is stringable
func (t Tags) String() string {
	buf := &bytes.Buffer{}

	for k, v := range t {
		buf.WriteByte(';')
		buf.WriteString(k)
		if v != "" {
			buf.WriteByte('=')
			buf.WriteString(v.Encode())
		}
	}

	// We don't need the first byte because that's an extra ';'
	// character.
	buf.ReadByte()

	return buf.String()
}

// Prefix represents the prefix of a message, generally the user who sent it
type Prefix struct {
	// Name will contain the nick of who sent the message, the
	// server who sent the message, or a blank string
	Name string

	// User will either contain the user who sent the message or a blank string
	User string

	// Host will either contain the host of who sent the message or a blank string
	Host string
}

// ParsePrefix takes an identity string and parses it into an
// identity struct. It will always return an Prefix struct and never
// nil.
func ParsePrefix(line string) *Prefix {
	// Start by creating an Prefix with nothing but the host
	id := &Prefix{
		Name: line,
	}

	uh := strings.SplitN(id.Name, "@", 2)
	if len(uh) == 2 {
		id.Name, id.Host = uh[0], uh[1]
	}

	nu := strings.SplitN(id.Name, "!", 2)
	if len(nu) == 2 {
		id.Name, id.User = nu[0], nu[1]
	}

	return id
}

// Copy will create a new copy of an Prefix
func (p *Prefix) Copy() *Prefix {
	if p == nil {
		return nil
	}

	newPrefix := &Prefix{}

	*newPrefix = *p

	return newPrefix
}

// String ensures this is stringable
func (p *Prefix) String() string {
	buf := &bytes.Buffer{}
	buf.WriteString(p.Name)

	if p.User != "" {
		buf.WriteString("!")
		buf.WriteString(p.User)
	}

	if p.Host != "" {
		buf.WriteString("@")
		buf.WriteString(p.Host)
	}

	return buf.String()
}

// Message represents a line parsed from the server
type Message struct {
	// Each message can have IRCv3 tags
	Tags

	// Each message can have a Prefix
	*Prefix

	// Command is which command is being called.
	Command string

	// Params are all the arguments for the command.
	Params []string
}

// MustParseMessage calls ParseMessage and either returns the message
// or panics if an error is returned.
func MustParseMessage(line string) *Message {
	m, err := ParseMessage(line)
	if err != nil {
		panic(err.Error())
	}
	return m
}

// ParseMessage takes a message string (usually a whole line) and
// parses it into a Message struct. This will return nil in the case
// of invalid messages.
func ParseMessage(line string) (*Message, error) {
	// Trim the line and make sure we have data
	line = strings.TrimSpace(line)
	if len(line) == 0 {
		return nil, ErrZeroLengthMessage
	}

	c := &Message{
		Tags:   Tags{},
		Prefix: &Prefix{},
	}

	if line[0] == '@' {
		split := strings.SplitN(line, " ", 2)
		if len(split) < 2 {
			return nil, ErrMissingDataAfterTags
		}

		c.Tags = ParseTags(split[0][1:])
		line = split[1]
	}

	if line[0] == ':' {
		split := strings.SplitN(line, " ", 2)
		if len(split) < 2 {
			return nil, ErrMissingDataAfterPrefix
		}

		// Parse the identity, if there was one
		c.Prefix = ParsePrefix(split[0][1:])
		line = split[1]
	}

	// Split out the trailing then the rest of the args. Because
	// we expect there to be at least one result as an arg (the
	// command) we don't need to special case the trailing arg and
	// can just attempt a split on " :"
	split := strings.SplitN(line, " :", 2)
	c.Params = strings.FieldsFunc(split[0], func(r rune) bool {
		return r == ' '
	})

	// If there are no args, we need to bail because we need at
	// least the command.
	if len(c.Params) == 0 {
		return nil, ErrMissingCommand
	}

	// If we had a trailing arg, append it to the other args
	if len(split) == 2 {
		c.Params = append(c.Params, split[1])
	}

	// Because of how it's parsed, the Command will show up as the
	// first arg.
	c.Command = c.Params[0]
	c.Params = c.Params[1:]

	return c, nil
}

// Trailing returns the last argument in the Message or an empty string
// if there are no args
func (m *Message) Trailing() string {
	if len(m.Params) < 1 {
		return ""
	}

	return m.Params[len(m.Params)-1]
}

// FromChannel is mostly for PRIVMSG messages (and similar derived
// messages) It will check if the message came from a channel or a
// person. This is not always accurate, because some servers allow for
// other characters at the start of a channel.
//
// TODO: Remove in favor of *Client.FromChannel(*Message)
func (m *Message) FromChannel() bool {
	if len(m.Params) < 1 || len(m.Params[0]) < 1 {
		return false
	}

	switch m.Params[0][0] {
	case '#', '&':
		return true
	default:
		return false
	}
}

// Copy will create a new copy of an message
func (m *Message) Copy() *Message {
	// Create a new message
	newMessage := &Message{}

	// Copy stuff from the old message
	*newMessage = *m

	// Copy any IRcv3 tags
	newMessage.Tags = m.Tags.Copy()

	// Copy the Prefix
	newMessage.Prefix = m.Prefix.Copy()

	// Copy the Params slice
	newMessage.Params = append(make([]string, 0, len(m.Params)), m.Params...)

	return newMessage
}

// String ensures this is stringable
func (m *Message) String() string {
	buf := &bytes.Buffer{}

	// Write any IRCv3 tags if they exist in the message
	if len(m.Tags) > 0 {
		buf.WriteByte('@')
		buf.WriteString(m.Tags.String())
		buf.WriteByte(' ')
	}

	// Add the prefix if we have one
	if m.Prefix != nil && m.Prefix.Name != "" {
		buf.WriteByte(':')
		buf.WriteString(m.Prefix.String())
		buf.WriteByte(' ')
	}

	// Add the command since we know we'll always have one
	buf.WriteString(m.Command)

	if len(m.Params) > 0 {
		args := m.Params[:len(m.Params)-1]
		trailing := m.Params[len(m.Params)-1]

		if len(args) > 0 {
			buf.WriteByte(' ')
			buf.WriteString(strings.Join(args, " "))
		}

		// If trailing is zero-length, contains a space or starts with
		// a : we need to actually specify that it's trailing.
		if len(trailing) == 0 || strings.ContainsRune(trailing, ' ') || trailing[0] == ':' {
			buf.WriteString(" :")
		} else {
			buf.WriteString(" ")
		}
		buf.WriteString(trailing)
	}

	return buf.String()
}
<commit_msg>Always uppercase the command<commit_after>package irc

import (
	"bytes"
	"errors"
	"strings"
)

var tagDecodeSlashMap = map[rune]rune{
	':':  ';',
	's':  ' ',
	'\\': '\\',
	'r':  '\r',
	'n':  '\n',
}

var tagEncodeMap = map[rune]string{
	';':  "\\:",
	' ':  "\\s",
	'\\': "\\\\",
	'\r': "\\r",
	'\n': "\\n",
}

var (
	// ErrZeroLengthMessage is returned when parsing if the input is
	// zero-length.
	ErrZeroLengthMessage = errors.New("irc: Cannot parse zero-length message")

	// ErrMissingDataAfterPrefix is returned when parsing if there is
	// no message data after the prefix.
	ErrMissingDataAfterPrefix = errors.New("irc: No message data after prefix")

	// ErrMissingDataAfterTags is returned when parsing if there is no
	// message data after the tags.
	ErrMissingDataAfterTags = errors.New("irc: No message data after tags")

	// ErrMissingCommand is returned when parsing if there is no
	// command in the parsed message.
	ErrMissingCommand = errors.New("irc: Missing message command")
)

// TagValue represents the value of a tag.
type TagValue string

// ParseTagValue parses a TagValue from the connection. If you need to
// set a TagValue, you probably want to just set the string itself, so
// it will be encoded properly.
func ParseTagValue(v string) TagValue {
	ret := &bytes.Buffer{}

	input := bytes.NewBufferString(v)

	for {
		c, _, err := input.ReadRune()
		if err != nil {
			break
		}

		if c == '\\' {
			c2, _, err := input.ReadRune()
			if err != nil {
				ret.WriteRune(c)
				break
			}

			if replacement, ok := tagDecodeSlashMap[c2]; ok {
				ret.WriteRune(replacement)
			} else {
				ret.WriteRune(c)
				ret.WriteRune(c2)
			}
		} else {
			ret.WriteRune(c)
		}
	}

	return TagValue(ret.String())
}

// Encode converts a TagValue to the format in the connection.
func (v TagValue) Encode() string {
	ret := &bytes.Buffer{}

	for _, c := range v {
		if replacement, ok := tagEncodeMap[c]; ok {
			ret.WriteString(replacement)
		} else {
			ret.WriteRune(c)
		}
	}

	return ret.String()
}

// Tags represents the IRCv3 message tags.
type Tags map[string]TagValue

// ParseTags takes a tag string and parses it into a tag map. It will
// always return a tag map, even if there are no valid tags.
func ParseTags(line string) Tags {
	ret := Tags{}

	tags := strings.Split(line, ";")
	for _, tag := range tags {
		parts := strings.SplitN(tag, "=", 2)
		if len(parts) < 2 {
			ret[parts[0]] = ""
			continue
		}

		ret[parts[0]] = ParseTagValue(parts[1])
	}

	return ret
}

// GetTag is a convenience method to look up a tag in the map.
func (t Tags) GetTag(key string) (string, bool) {
	ret, ok := t[key]
	return string(ret), ok
}

// Copy will create a new copy of all IRC tags attached to this
// message.
func (t Tags) Copy() Tags {
	ret := Tags{}

	for k, v := range t {
		ret[k] = v
	}

	return ret
}

// String ensures this is stringable
func (t Tags) String() string {
	buf := &bytes.Buffer{}

	for k, v := range t {
		buf.WriteByte(';')
		buf.WriteString(k)
		if v != "" {
			buf.WriteByte('=')
			buf.WriteString(v.Encode())
		}
	}

	// We don't need the first byte because that's an extra ';'
	// character.
	buf.ReadByte()

	return buf.String()
}

// Prefix represents the prefix of a message, generally the user who sent it
type Prefix struct {
	// Name will contain the nick of who sent the message, the
	// server who sent the message, or a blank string
	Name string

	// User will either contain the user who sent the message or a blank string
	User string

	// Host will either contain the host of who sent the message or a blank string
	Host string
}

// ParsePrefix takes an identity string and parses it into an
// identity struct. It will always return an Prefix struct and never
// nil.
func ParsePrefix(line string) *Prefix {
	// Start by creating an Prefix with nothing but the host
	id := &Prefix{
		Name: line,
	}

	uh := strings.SplitN(id.Name, "@", 2)
	if len(uh) == 2 {
		id.Name, id.Host = uh[0], uh[1]
	}

	nu := strings.SplitN(id.Name, "!", 2)
	if len(nu) == 2 {
		id.Name, id.User = nu[0], nu[1]
	}

	return id
}

// Copy will create a new copy of an Prefix
func (p *Prefix) Copy() *Prefix {
	if p == nil {
		return nil
	}

	newPrefix := &Prefix{}

	*newPrefix = *p

	return newPrefix
}

// String ensures this is stringable
func (p *Prefix) String() string {
	buf := &bytes.Buffer{}
	buf.WriteString(p.Name)

	if p.User != "" {
		buf.WriteString("!")
		buf.WriteString(p.User)
	}

	if p.Host != "" {
		buf.WriteString("@")
		buf.WriteString(p.Host)
	}

	return buf.String()
}

// Message represents a line parsed from the server
type Message struct {
	// Each message can have IRCv3 tags
	Tags

	// Each message can have a Prefix
	*Prefix

	// Command is which command is being called.
	Command string

	// Params are all the arguments for the command.
	Params []string
}

// MustParseMessage calls ParseMessage and either returns the message
// or panics if an error is returned.
func MustParseMessage(line string) *Message {
	m, err := ParseMessage(line)
	if err != nil {
		panic(err.Error())
	}
	return m
}

// ParseMessage takes a message string (usually a whole line) and
// parses it into a Message struct. This will return nil in the case
// of invalid messages.
func ParseMessage(line string) (*Message, error) {
	// Trim the line and make sure we have data
	line = strings.TrimSpace(line)
	if len(line) == 0 {
		return nil, ErrZeroLengthMessage
	}

	c := &Message{
		Tags:   Tags{},
		Prefix: &Prefix{},
	}

	if line[0] == '@' {
		split := strings.SplitN(line, " ", 2)
		if len(split) < 2 {
			return nil, ErrMissingDataAfterTags
		}

		c.Tags = ParseTags(split[0][1:])
		line = split[1]
	}

	if line[0] == ':' {
		split := strings.SplitN(line, " ", 2)
		if len(split) < 2 {
			return nil, ErrMissingDataAfterPrefix
		}

		// Parse the identity, if there was one
		c.Prefix = ParsePrefix(split[0][1:])
		line = split[1]
	}

	// Split out the trailing then the rest of the args. Because
	// we expect there to be at least one result as an arg (the
	// command) we don't need to special case the trailing arg and
	// can just attempt a split on " :"
	split := strings.SplitN(line, " :", 2)
	c.Params = strings.FieldsFunc(split[0], func(r rune) bool {
		return r == ' '
	})

	// If there are no args, we need to bail because we need at
	// least the command.
	if len(c.Params) == 0 {
		return nil, ErrMissingCommand
	}

	// If we had a trailing arg, append it to the other args
	if len(split) == 2 {
		c.Params = append(c.Params, split[1])
	}

	// Because of how it's parsed, the Command will show up as the
	// first arg.
	c.Command = strings.ToUpper(c.Params[0])
	c.Params = c.Params[1:]

	return c, nil
}

// Trailing returns the last argument in the Message or an empty string
// if there are no args
func (m *Message) Trailing() string {
	if len(m.Params) < 1 {
		return ""
	}

	return m.Params[len(m.Params)-1]
}

// FromChannel is mostly for PRIVMSG messages (and similar derived
// messages) It will check if the message came from a channel or a
// person. This is not always accurate, because some servers allow for
// other characters at the start of a channel.
//
// TODO: Remove in favor of *Client.FromChannel(*Message)
func (m *Message) FromChannel() bool {
	if len(m.Params) < 1 || len(m.Params[0]) < 1 {
		return false
	}

	switch m.Params[0][0] {
	case '#', '&':
		return true
	default:
		return false
	}
}

// Copy will create a new copy of an message
func (m *Message) Copy() *Message {
	// Create a new message
	newMessage := &Message{}

	// Copy stuff from the old message
	*newMessage = *m

	// Copy any IRcv3 tags
	newMessage.Tags = m.Tags.Copy()

	// Copy the Prefix
	newMessage.Prefix = m.Prefix.Copy()

	// Copy the Params slice
	newMessage.Params = append(make([]string, 0, len(m.Params)), m.Params...)

	return newMessage
}

// String ensures this is stringable
func (m *Message) String() string {
	buf := &bytes.Buffer{}

	// Write any IRCv3 tags if they exist in the message
	if len(m.Tags) > 0 {
		buf.WriteByte('@')
		buf.WriteString(m.Tags.String())
		buf.WriteByte(' ')
	}

	// Add the prefix if we have one
	if m.Prefix != nil && m.Prefix.Name != "" {
		buf.WriteByte(':')
		buf.WriteString(m.Prefix.String())
		buf.WriteByte(' ')
	}

	// Add the command since we know we'll always have one
	buf.WriteString(m.Command)

	if len(m.Params) > 0 {
		args := m.Params[:len(m.Params)-1]
		trailing := m.Params[len(m.Params)-1]

		if len(args) > 0 {
			buf.WriteByte(' ')
			buf.WriteString(strings.Join(args, " "))
		}

		// If trailing is zero-length, contains a space or starts with
		// a : we need to actually specify that it's trailing.
		if len(trailing) == 0 || strings.ContainsRune(trailing, ' ') || trailing[0] == ':' {
			buf.WriteString(" :")
		} else {
			buf.WriteString(" ")
		}
		buf.WriteString(trailing)
	}

	return buf.String()
}
<|endoftext|>


<commit_before>package processor

import (
	"bufio"
	"github.com/docker/engine-api/client"
	"github.com/gaia-docker/tugbot-collect/log"
	"golang.org/x/net/context"
	"io"
	"os"
	"path/filepath"
	"strings"
	"github.com/docker/engine-api/types"
)

var logger = log.GetLogger("processor")

type Processor struct {
	Tasks            chan string
	dockerClient     *client.Client
	outputdir        string
	resultserviceurl string
	resultsdirlabel  string
	dockerrm         bool
}

func NewProcessor(p_dockerClient *client.Client, p_outputdir string, p_resultserviceurl string, p_resultsdirlabel string, p_dockerrm bool) Processor {
	p := Processor{
		dockerClient:    p_dockerClient,
		outputdir:       p_outputdir,
		resultserviceurl:   p_resultserviceurl,
		resultsdirlabel: p_resultsdirlabel,
		dockerrm:        p_dockerrm,
	}
	p.Tasks = make(chan string, 10)
	return p
}

func (p Processor) Run() {
	go func() {
		for task := range p.Tasks {

			//We run each task in a separate goroutine to parallel the extraction
			go func(contId string) {
				logger.Info("processor is going to extract results from container with id: ", contId)

				ctx, cancel := context.WithCancel(context.Background())
				defer cancel()

				contInfo, err := p.dockerClient.ContainerInspect(ctx, contId)
				if err != nil {
					logger.Error("failed to inspect container with id: ", contId, ", cannot determine results dir location - discard processing this container")
					return
				}

				//copy the result dir from inside the container into memory
				//The result dir will be return in tar format
				resultDir := contInfo.Config.Labels[p.resultsdirlabel]
				logger.Info("going to extract results for container with id: ", contId, ", from this location inside the test container: ", resultDir)
				reader, pathstat, err := p.dockerClient.CopyFromContainer(ctx, contId, resultDir)
				if err != nil {
					logger.Error("failed to copy result dir from container with id: ", contId, ", error is: ", err, " - discard processing this container")
					return
				}

				//check that the resultdir is really a directory
				if !pathstat.Mode.IsDir() {
					logger.Error("results location:", resultDir, ", inside the container with id: ", contId, ", is not a directory - discard processing of this container")
					return
				}

				//write the results to disk
				writeToDiskStatus := make(chan error)
				writeToDisk(p.outputdir, contInfo.Config.Image, contId, reader, writeToDiskStatus)
				if <-writeToDiskStatus != nil {
					logger.Error("failed to write to disk")
					return
				}

				logger.Info("sucessfully extracted results from container with id: ", contId)

				if p.dockerrm {
					err = p.dockerClient.ContainerRemove(ctx, contId, types.ContainerRemoveOptions{})
					if err != nil {
						logger.Error("failed to remove container with id: ", contId, ", why: ", err)
						return
					}

					logger.Info("sucessfully removed (docker rm) container with id: ", contId)
				}
			}(task)
		}
	}()
}

func writeToDisk(outputDir, imageName, contId string, reader io.ReadCloser, returnVal chan error) {

	go func() {
		if outputDir == "/dev/null" {
			logger.Info("output dir is ", outputDir, ", skip writing to disk")
			returnVal <- nil
			return
		}

		//create output dir (is dir is already exist MkdirAll will return nil
		outDirFullPath := outputDir + string(filepath.Separator) + strings.Replace(imageName, "/", "_", -1) + "-" + contId[:11]
		err := os.MkdirAll(outDirFullPath, 0777)
		if err != nil {
			logger.Error("failed to create output dir: ", outDirFullPath, ", for container with id: ", contId, ", error is: ", err, " - discard processing this container")
			returnVal <- err
			return
		}
		logger.Info("going to write results from container with id: ", contId, ", into this location: ", outDirFullPath)

		// open output file
		fo, err := os.Create(outDirFullPath + string(filepath.Separator) + "results.tar")
		if err != nil {
			logger.Error("failed to create tar file on FS for container with id: ", contId, ", error is: ", err, " - discard processing this container")
			returnVal <- err
			return
		}
		// close fo on exit and check for its returned error
		defer func() {
			if err := fo.Close(); err != nil {
				logger.Error("failed to close FS tar file for container with id: ", contId, ", error is: ", err)
			}
		}()

		// make a buffer to keep chunks that are read
		// buffer size==32K
		buf := make([]byte, 32 * 1024)
		writer := bufio.NewWriter(fo)

		if _, err := io.CopyBuffer(writer, reader, buf); err != nil {
			logger.Error("failed to write to tar file for container with id: ", contId, ", error is: ", err, " - discard processing this container")
			returnVal <- err
			return
		}

		//All is good, returning nil on the channel
		returnVal <- nil
	}()
}
<commit_msg>fix some golint errors<commit_after>package processor

import (
	"bufio"
	"github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
	"github.com/gaia-docker/tugbot-collect/log"
	"golang.org/x/net/context"
	"io"
	"os"
	"path/filepath"
	"strings"
)

var logger = log.GetLogger("processor")

//Process any container that is pushed to the Tasks channel - collecting the results and write to disk / publish to the results service
type Processor struct {
	Tasks            chan string
	dockerClient     *client.Client
	outputDir        string
	resultServiceUrl string
	resultsDirLabel  string
	dockerRM         bool
}

func NewProcessor(pDockerClient *client.Client, pOutputDir string, pResultServiceUrl string, pResultsDirLabel string, pDockerRM bool) Processor {
	p := Processor{
		dockerClient:     pDockerClient,
		outputDir:        pOutputDir,
		resultServiceUrl: pResultServiceUrl,
		resultsDirLabel:  pResultsDirLabel,
		dockerRM:         pDockerRM,
	}
	p.Tasks = make(chan string, 10)
	return p
}

func (p Processor) Run() {
	go func() {
		for task := range p.Tasks {

			//We run each task in a separate goroutine to parallel the extraction
			go func(contId string) {
				logger.Info("processor is going to extract results from container with id: ", contId)

				ctx, cancel := context.WithCancel(context.Background())
				defer cancel()

				contInfo, err := p.dockerClient.ContainerInspect(ctx, contId)
				if err != nil {
					logger.Error("failed to inspect container with id: ", contId, ", cannot determine results dir location - discard processing this container")
					return
				}

				//copy the result dir from inside the container into memory
				//The result dir will be return in tar format
				resultDir := contInfo.Config.Labels[p.resultsDirLabel]
				logger.Info("going to extract results for container with id: ", contId, ", from this location inside the test container: ", resultDir)
				reader, pathstat, err := p.dockerClient.CopyFromContainer(ctx, contId, resultDir)
				if err != nil {
					logger.Error("failed to copy result dir from container with id: ", contId, ", error is: ", err, " - discard processing this container")
					return
				}

				//check that the resultdir is really a directory
				if !pathstat.Mode.IsDir() {
					logger.Error("results location:", resultDir, ", inside the container with id: ", contId, ", is not a directory - discard processing of this container")
					return
				}

				//write the results to disk
				writeToDiskStatus := make(chan error)
				writeToDisk(p.outputDir, contInfo.Config.Image, contId, reader, writeToDiskStatus)
				if <-writeToDiskStatus != nil {
					logger.Error("failed to write to disk")
					return
				}

				logger.Info("sucessfully extracted results from container with id: ", contId)

				if p.dockerRM {
					err = p.dockerClient.ContainerRemove(ctx, contId, types.ContainerRemoveOptions{})
					if err != nil {
						logger.Error("failed to remove container with id: ", contId, ", why: ", err)
						return
					}

					logger.Info("sucessfully removed (docker rm) container with id: ", contId)
				}
			}(task)
		}
	}()
}

func writeToDisk(outputDir, imageName, contId string, reader io.ReadCloser, returnVal chan error) {

	go func() {
		if outputDir == "/dev/null" {
			logger.Info("output dir is ", outputDir, ", skip writing to disk")
			returnVal <- nil
			return
		}

		//create output dir (is dir is already exist MkdirAll will return nil
		outDirFullPath := outputDir + string(filepath.Separator) + strings.Replace(imageName, "/", "_", -1) + "-" + contId[:11]
		err := os.MkdirAll(outDirFullPath, 0777)
		if err != nil {
			logger.Error("failed to create output dir: ", outDirFullPath, ", for container with id: ", contId, ", error is: ", err, " - discard processing this container")
			returnVal <- err
			return
		}
		logger.Info("going to write results from container with id: ", contId, ", into this location: ", outDirFullPath)

		// open output file
		fo, err := os.Create(outDirFullPath + string(filepath.Separator) + "results.tar")
		if err != nil {
			logger.Error("failed to create tar file on FS for container with id: ", contId, ", error is: ", err, " - discard processing this container")
			returnVal <- err
			return
		}
		// close fo on exit and check for its returned error
		defer func() {
			if err := fo.Close(); err != nil {
				logger.Error("failed to close FS tar file for container with id: ", contId, ", error is: ", err)
			}
		}()

		// make a buffer to keep chunks that are read
		// buffer size==32K
		buf := make([]byte, 32*1024)
		writer := bufio.NewWriter(fo)

		if _, err := io.CopyBuffer(writer, reader, buf); err != nil {
			logger.Error("failed to write to tar file for container with id: ", contId, ", error is: ", err, " - discard processing this container")
			returnVal <- err
			return
		}

		//All is good, returning nil on the channel
		returnVal <- nil
	}()
}
<|endoftext|>


<commit_before>// The command line tool for running Imsto bootstrap.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"strings"
	"sync"
	"text/template"
	"wpst.me/calf/config"
)

// Cribbed from the genius organization of the "go" command.
type Command struct {
	Run                    func(args []string) bool
	UsageLine, Short, Long string
	// Flag is a set of flags specific to this command.
	Flag    flag.FlagSet
	IsDebug *bool
}

func (cmd *Command) Name() string {
	name := cmd.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (cmd *Command) Usage() {
	fmt.Fprintf(os.Stderr, "Example: imsto %s\n", cmd.UsageLine)
	fmt.Fprintf(os.Stderr, "Default Usage:\n")
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "Description:\n")
	fmt.Fprintf(os.Stderr, "  %s\n", strings.TrimSpace(cmd.Long))
	os.Exit(2)
}

type LoggedError struct{ error }

const (
	VERSION = "0.0.2"
)

// main
var (
	IsDebug    *bool
	exitStatus = 0
	exitMu     sync.Mutex
	appDir     string
	logDir     string
)

var commands = []*Command{
	cmdImport,
	cmdOptimize,
	cmdTiring,
	cmdStage,
	cmdView,
	cmdTest,
}

func setExitStatus(n int) {
	exitMu.Lock()
	if exitStatus < n {
		exitStatus = n
	}
	exitMu.Unlock()
}

func init() {
	flag.StringVar(&appDir, "root", "", "app root dir")
	flag.StringVar(&logDir, "logs", "", "app logs dir")
	flag.Parse()
	if appDir != "" {
		config.SetAppRoot(appDir)
	}
	err := config.Load()
	if err != nil {
		log.Print("config load error: ", err)
	}
	if logDir == "" {
		dir := config.AppRoot()
		log.Printf("approot %s", dir)
		if err != nil {
			log.Print("getwd error: ", err)
		}
		if dir != "" {
			dir = path.Join(dir, "logs")
			// log.Printf("log dir: %s", dir)
			if fi, err := os.Stat(dir); err == nil {
				if fi.IsDir() {
					logDir = dir
				}
			}
		}
	}
}

func main() {
	fmt.Fprintf(os.Stdout, header)
	flag.Usage = func() { usage(1) }
	args := flag.Args()

	if len(args) < 1 || args[0] == "help" {
		if len(args) == 1 {
			usage(0)
		}
		if len(args) > 1 {
			for _, cmd := range commands {
				if cmd.Name() == args[1] {
					tmpl(os.Stdout, helpTemplate, cmd)
					return
				}
			}
		}
		usage(2)
	}

	for _, cmd := range commands {
		name := cmd.Name()
		if name == args[0] && cmd.Run != nil {
			cmd.Flag.Usage = func() { cmd.Usage() }
			cmd.Flag.Parse(args[1:])
			args = cmd.Flag.Args()
			IsDebug = cmd.IsDebug
			// if IsDebug != nil && *IsDebug {
			log.SetFlags(log.LstdFlags | log.Lshortfile)
			log.Printf("log dir: %s", logDir)
			if logDir != "" {
				logfile := path.Join(logDir, name+".log")
				log.Printf("logfile: %s", logfile)
				fd, err := os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
				if err != nil {
					log.Printf("logfile %s create failed", logfile)
					return
				}
				log.SetOutput(fd)
			}

			// }
			if cmd.Run(args) {
				fmt.Fprintf(os.Stderr, "\n")
			}
			exit()
			return
		}
	}

	errorf("unknown command %q\nRun 'imsto help' for usage.\n", args[0])
}

func errorf(format string, args ...interface{}) {
	// Ensure the user's command prompt starts on the next line.
	if !strings.HasSuffix(format, "\n") {
		format += "\n"
	}
	fmt.Fprintf(os.Stderr, format, args...)
	// panic(LoggedError{}) // Panic instead of os.Exit so that deferred will run.
}

const header = `
Welcome

`

const usageTemplate = `usage: imsto command [arguments]

The commands are:
{{range .}}
    {{.Name | printf "%-11s"}} {{.Short}}{{end}}

Use "imsto help [command]" for more information.
`

var helpTemplate = `usage: imsto {{.UsageLine}}
{{.Long}}
`

func usage(exitCode int) {
	tmpl(os.Stderr, usageTemplate, commands)
	os.Exit(exitCode)
}

func tmpl(w io.Writer, text string, data interface{}) {
	t := template.New("top")
	template.Must(t.Parse(text))
	if err := t.Execute(w, data); err != nil {
		panic(err)
	}
}

var atExitFuncs []func()

func atExit(f func()) {
	atExitFuncs = append(atExitFuncs, f)
}

func exit() {
	for _, f := range atExitFuncs {
		f()
	}
	os.Exit(exitStatus)
}

func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) (err error) {
	w.Header().Set("Content-Type", "application/json")
	var bytes []byte
	if r.FormValue("pretty") != "" {
		bytes, err = json.MarshalIndent(obj, "", "  ")
	} else {
		bytes, err = json.Marshal(obj)
	}
	if err != nil {
		return
	}
	callback := r.FormValue("callback")
	if callback == "" {
		_, err = w.Write(bytes)
	} else {
		if _, err = w.Write([]uint8(callback)); err != nil {
			return
		}
		if _, err = w.Write([]uint8("(")); err != nil {
			return
		}
		fmt.Fprint(w, string(bytes))
		if _, err = w.Write([]uint8(")")); err != nil {
			return
		}
	}
	return
}

// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, obj interface{}) {
	if err := writeJson(w, r, obj); err != nil {
		log.Printf("error writing JSON %s: %s", obj, err.Error())
	}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, err error) {
	m := make(map[string]interface{})
	m["status"] = "fail"
	m["error"] = err.Error()
	writeJsonQuiet(w, r, m)
}

func debug(params ...interface{}) {
	if *IsDebug {
		log.Print(params)
	}
}
func secure(whiteList []string, f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if len(whiteList) == 0 {
			f(w, r)
			return
		}
		host, _, err := net.SplitHostPort(r.RemoteAddr)
		if err == nil {
			for _, ip := range whiteList {
				if ip == host {
					f(w, r)
					return
				}
			}
		}
		writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host})
	}
}
<commit_msg>change default log directory<commit_after>// The command line tool for running Imsto bootstrap.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"strings"
	"sync"
	"text/template"
	"wpst.me/calf/config"
)

// Cribbed from the genius organization of the "go" command.
type Command struct {
	Run                    func(args []string) bool
	UsageLine, Short, Long string
	// Flag is a set of flags specific to this command.
	Flag    flag.FlagSet
	IsDebug *bool
}

func (cmd *Command) Name() string {
	name := cmd.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (cmd *Command) Usage() {
	fmt.Fprintf(os.Stderr, "Example: imsto %s\n", cmd.UsageLine)
	fmt.Fprintf(os.Stderr, "Default Usage:\n")
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "Description:\n")
	fmt.Fprintf(os.Stderr, "  %s\n", strings.TrimSpace(cmd.Long))
	os.Exit(2)
}

type LoggedError struct{ error }

const (
	VERSION = "0.0.2"
)

// main
var (
	IsDebug    *bool
	exitStatus = 0
	exitMu     sync.Mutex
	appDir     string
	logDir     string
)

var commands = []*Command{
	cmdImport,
	cmdOptimize,
	cmdTiring,
	cmdStage,
	cmdView,
	cmdTest,
}

func setExitStatus(n int) {
	exitMu.Lock()
	if exitStatus < n {
		exitStatus = n
	}
	exitMu.Unlock()
}

func init() {
	flag.StringVar(&appDir, "root", "", "app root dir")
	flag.StringVar(&logDir, "logs", "/var/log/imsto", "app logs dir")
	flag.Parse()
	if appDir != "" {
		config.SetAppRoot(appDir)
	}
	err := config.Load()
	if err != nil {
		log.Print("config load error: ", err)
	}
	if logDir == "" {
		dir := config.AppRoot()
		log.Printf("approot %s", dir)
		if err != nil {
			log.Print("getwd error: ", err)
		}
		if dir != "" {
			dir = path.Join(dir, "logs")
			// log.Printf("log dir: %s", dir)
			if fi, err := os.Stat(dir); err == nil {
				if fi.IsDir() {
					logDir = dir
				}
			}
		}
	}
}

func main() {
	fmt.Fprintf(os.Stdout, header)
	flag.Usage = func() { usage(1) }
	args := flag.Args()

	if len(args) < 1 || args[0] == "help" {
		if len(args) == 1 {
			usage(0)
		}
		if len(args) > 1 {
			for _, cmd := range commands {
				if cmd.Name() == args[1] {
					tmpl(os.Stdout, helpTemplate, cmd)
					return
				}
			}
		}
		usage(2)
	}

	if logDir != "" {
		_, err := os.Stat(logDir)
		if os.IsNotExist(err) {
			if err := os.Mkdir(logDir, os.FileMode(0755)); err != nil {
				log.Printf("mkdir '%s' error or access denied", logDir)
				return
			}
		} else if os.IsPermission(err) {
			log.Printf("dir '%s' access denied", logDir)
			return
		}

	}

	for _, cmd := range commands {
		name := cmd.Name()
		if name == args[0] && cmd.Run != nil {
			cmd.Flag.Usage = func() { cmd.Usage() }
			cmd.Flag.Parse(args[1:])
			args = cmd.Flag.Args()
			IsDebug = cmd.IsDebug
			// if IsDebug != nil && *IsDebug {
			log.SetFlags(log.LstdFlags | log.Lshortfile)
			log.Printf("log dir: %s", logDir)
			if logDir != "" {
				logfile := path.Join(logDir, name+"_log")
				log.Printf("logfile: %s", logfile)
				fd, err := os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
				if err != nil {
					log.Printf("logfile %s create failed", logfile)
					return
				}
				log.SetOutput(fd)
			}

			// }
			if cmd.Run(args) {
				fmt.Fprintf(os.Stderr, "\n")
			}
			exit()
			return
		}
	}

	errorf("unknown command %q\nRun 'imsto help' for usage.\n", args[0])
}

func errorf(format string, args ...interface{}) {
	// Ensure the user's command prompt starts on the next line.
	if !strings.HasSuffix(format, "\n") {
		format += "\n"
	}
	fmt.Fprintf(os.Stderr, format, args...)
	// panic(LoggedError{}) // Panic instead of os.Exit so that deferred will run.
}

const header = `
Welcome

`

const usageTemplate = `usage: imsto command [arguments]

The commands are:
{{range .}}
    {{.Name | printf "%-11s"}} {{.Short}}{{end}}

Use "imsto help [command]" for more information.
`

var helpTemplate = `usage: imsto {{.UsageLine}}
{{.Long}}
`

func usage(exitCode int) {
	tmpl(os.Stderr, usageTemplate, commands)
	os.Exit(exitCode)
}

func tmpl(w io.Writer, text string, data interface{}) {
	t := template.New("top")
	template.Must(t.Parse(text))
	if err := t.Execute(w, data); err != nil {
		panic(err)
	}
}

var atExitFuncs []func()

func atExit(f func()) {
	atExitFuncs = append(atExitFuncs, f)
}

func exit() {
	for _, f := range atExitFuncs {
		f()
	}
	os.Exit(exitStatus)
}

func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) (err error) {
	w.Header().Set("Content-Type", "application/json")
	var bytes []byte
	if r.FormValue("pretty") != "" {
		bytes, err = json.MarshalIndent(obj, "", "  ")
	} else {
		bytes, err = json.Marshal(obj)
	}
	if err != nil {
		return
	}
	callback := r.FormValue("callback")
	if callback == "" {
		_, err = w.Write(bytes)
	} else {
		if _, err = w.Write([]uint8(callback)); err != nil {
			return
		}
		if _, err = w.Write([]uint8("(")); err != nil {
			return
		}
		fmt.Fprint(w, string(bytes))
		if _, err = w.Write([]uint8(")")); err != nil {
			return
		}
	}
	return
}

// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, obj interface{}) {
	if err := writeJson(w, r, obj); err != nil {
		log.Printf("error writing JSON %s: %s", obj, err.Error())
	}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, err error) {
	m := make(map[string]interface{})
	m["status"] = "fail"
	m["error"] = err.Error()
	writeJsonQuiet(w, r, m)
}

func debug(params ...interface{}) {
	if *IsDebug {
		log.Print(params)
	}
}
func secure(whiteList []string, f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if len(whiteList) == 0 {
			f(w, r)
			return
		}
		host, _, err := net.SplitHostPort(r.RemoteAddr)
		if err == nil {
			for _, ip := range whiteList {
				if ip == host {
					f(w, r)
					return
				}
			}
		}
		writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host})
	}
}
<|endoftext|>


<commit_before>package actors

import (
	"bytes"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/bosh-bootloader/integration-test"
)

type BBL struct {
	stateDirectory string
	pathToBBL      string
	configuration  integration.Config
}

func NewBBL(stateDirectory string, pathToBBL string, configuration integration.Config) BBL {
	return BBL{
		stateDirectory: stateDirectory,
		pathToBBL:      pathToBBL,
		configuration:  configuration,
	}
}

func (b BBL) Up(loadBalancerType string) {
	args := []string{
		"--aws-access-key-id", b.configuration.AWSAccessKeyID,
		"--aws-secret-access-key", b.configuration.AWSSecretAccessKey,
		"--aws-region", b.configuration.AWSRegion,
		"--state-dir", b.stateDirectory,
		"unsupported-deploy-bosh-on-aws-for-concourse",
		"--lb-type", loadBalancerType,
	}

	if loadBalancerType == "cf" || loadBalancerType == "concourse" {
		args = append(args, []string{
			"--cert", "bbl-certs/bbl.crt",
			"--key", "bbl-certs/bbl.key",
		}...)
	}

	session := b.execute(args, os.Stdout, os.Stderr)
	Eventually(session, 40*time.Minute).Should(gexec.Exit(0))
}

func (b BBL) Destroy() {
	session := b.execute([]string{
		"--state-dir", b.stateDirectory,
		"destroy",
		"--no-confirm",
	}, os.Stdout, os.Stderr)
	Eventually(session, 10*time.Minute).Should(gexec.Exit(0))
}

func (b BBL) DirectorUsername() string {
	return b.fetchValue("director-username")
}

func (b BBL) DirectorPassword() string {
	return b.fetchValue("director-password")
}

func (b BBL) DirectorAddress() string {
	return b.fetchValue("director-address")
}

func (b BBL) CreateLB(loadBalancerType string) {
	args := []string{
		"--aws-access-key-id", b.configuration.AWSAccessKeyID,
		"--aws-secret-access-key", b.configuration.AWSSecretAccessKey,
		"--aws-region", b.configuration.AWSRegion,
		"--state-dir", b.stateDirectory,
		"unsupported-create-lbs",
		"--type", loadBalancerType,
		"--cert", "bbl-certs/bbl-intermediate.crt",
		"--key", "bbl-certs/bbl-intermediate.key",
		"--chain", "bbl-certs/bbl.crt",
	}

	session := b.execute(args, os.Stdout, os.Stderr)
	Eventually(session, 10*time.Minute).Should(gexec.Exit(0))
}

func (b BBL) UpdateLB(certPath, keyPath string) {
	args := []string{
		"--aws-access-key-id", b.configuration.AWSAccessKeyID,
		"--aws-secret-access-key", b.configuration.AWSSecretAccessKey,
		"--aws-region", b.configuration.AWSRegion,
		"--state-dir", b.stateDirectory,
		"unsupported-update-lbs",
		"--cert", certPath,
		"--key", keyPath,
	}

	session := b.execute(args, os.Stdout, os.Stderr)
	Eventually(session, 10*time.Minute).Should(gexec.Exit(0))
}

func (b BBL) fetchValue(value string) string {
	args := []string{
		"--state-dir", b.stateDirectory,
		value,
	}

	stdout := bytes.NewBuffer([]byte{})
	stderr := bytes.NewBuffer([]byte{})
	b.execute(args, stdout, stderr).Wait()

	return strings.TrimSpace(string(stdout.Bytes()))
}

func (b BBL) execute(args []string, stdout io.Writer, stderr io.Writer) *gexec.Session {
	cmd := exec.Command(b.pathToBBL, args...)
	session, err := gexec.Start(cmd, stdout, stderr)
	Expect(err).NotTo(HaveOccurred())

	return session
}
<commit_msg>Added LBs helper func for integration test<commit_after>package actors

import (
	"bytes"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/bosh-bootloader/integration-test"
)

type BBL struct {
	stateDirectory string
	pathToBBL      string
	configuration  integration.Config
}

func NewBBL(stateDirectory string, pathToBBL string, configuration integration.Config) BBL {
	return BBL{
		stateDirectory: stateDirectory,
		pathToBBL:      pathToBBL,
		configuration:  configuration,
	}
}

func (b BBL) Up(loadBalancerType string) {
	args := []string{
		"--aws-access-key-id", b.configuration.AWSAccessKeyID,
		"--aws-secret-access-key", b.configuration.AWSSecretAccessKey,
		"--aws-region", b.configuration.AWSRegion,
		"--state-dir", b.stateDirectory,
		"unsupported-deploy-bosh-on-aws-for-concourse",
		"--lb-type", loadBalancerType,
	}

	if loadBalancerType == "cf" || loadBalancerType == "concourse" {
		args = append(args, []string{
			"--cert", "bbl-certs/bbl.crt",
			"--key", "bbl-certs/bbl.key",
		}...)
	}

	session := b.execute(args, os.Stdout, os.Stderr)
	Eventually(session, 40*time.Minute).Should(gexec.Exit(0))
}

func (b BBL) Destroy() {
	session := b.execute([]string{
		"--state-dir", b.stateDirectory,
		"destroy",
		"--no-confirm",
	}, os.Stdout, os.Stderr)
	Eventually(session, 10*time.Minute).Should(gexec.Exit(0))
}

func (b BBL) DirectorUsername() string {
	return b.fetchValue("director-username")
}

func (b BBL) DirectorPassword() string {
	return b.fetchValue("director-password")
}

func (b BBL) DirectorAddress() string {
	return b.fetchValue("director-address")
}

func (b BBL) CreateLB(loadBalancerType string) {
	args := []string{
		"--aws-access-key-id", b.configuration.AWSAccessKeyID,
		"--aws-secret-access-key", b.configuration.AWSSecretAccessKey,
		"--aws-region", b.configuration.AWSRegion,
		"--state-dir", b.stateDirectory,
		"unsupported-create-lbs",
		"--type", loadBalancerType,
		"--cert", "bbl-certs/bbl-intermediate.crt",
		"--key", "bbl-certs/bbl-intermediate.key",
		"--chain", "bbl-certs/bbl.crt",
	}

	session := b.execute(args, os.Stdout, os.Stderr)
	Eventually(session, 10*time.Minute).Should(gexec.Exit(0))
}

func (b BBL) LBs() *gexec.Session {
	args := []string{
		"--aws-access-key-id", b.configuration.AWSAccessKeyID,
		"--aws-secret-access-key", b.configuration.AWSSecretAccessKey,
		"--aws-region", b.configuration.AWSRegion,
		"--state-dir", b.stateDirectory,
		"unsupported-lbs",
	}

	session := b.execute(args, os.Stdout, os.Stderr)
	Eventually(session, 10*time.Minute).Should(gexec.Exit(0))

	return session
}

func (b BBL) UpdateLB(certPath, keyPath string) {
	args := []string{
		"--aws-access-key-id", b.configuration.AWSAccessKeyID,
		"--aws-secret-access-key", b.configuration.AWSSecretAccessKey,
		"--aws-region", b.configuration.AWSRegion,
		"--state-dir", b.stateDirectory,
		"unsupported-update-lbs",
		"--cert", certPath,
		"--key", keyPath,
	}

	session := b.execute(args, os.Stdout, os.Stderr)
	Eventually(session, 10*time.Minute).Should(gexec.Exit(0))
}

func (b BBL) fetchValue(value string) string {
	args := []string{
		"--state-dir", b.stateDirectory,
		value,
	}

	stdout := bytes.NewBuffer([]byte{})
	stderr := bytes.NewBuffer([]byte{})
	b.execute(args, stdout, stderr).Wait()

	return strings.TrimSpace(string(stdout.Bytes()))
}

func (b BBL) execute(args []string, stdout io.Writer, stderr io.Writer) *gexec.Session {
	cmd := exec.Command(b.pathToBBL, args...)
	session, err := gexec.Start(cmd, stdout, stderr)
	Expect(err).NotTo(HaveOccurred())

	return session
}
<|endoftext|>


<commit_before>// gotify is a client library for the Spotify API
package gotify

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const OauthAuthorizeURL = "https://accounts.spotify.com/authorize"
const OauthTokenUrl = "https://accounts.spotify.com/api/token"

type SpotifyOauth struct {
	ClientId     string
	ClientSecret string
	RedirectUri  string
	State        string
	Scope        string
	CachePath    string
}

type Token struct {
	AccessToken  string `json:"access_token"`
	TokenType    string `json:"token_type"`
	ExpiresAt    time.Time
	RefreshToken string `json:"refresh_token"`
	TTL          int    `json:"expires_in"`
}

func (t Token) isExpired() bool {
	if !(int(time.Since(t.ExpiresAt)) >= 0) {
		return false
	}
	return true
}

func (t *Token) stringer() string {
	return fmt.Sprintf("{AccessToken: %s, TokenType: %s, ExpiresAt: %s, RefreshToken: %s, TTL: %s}", t.AccessToken, t.TokenType, t.ExpiresAt, t.RefreshToken, string(t.TTL))
}

// Reads a cached token
func GetCachedToken(oauth SpotifyOauth) (Token, error) {
	var token Token
	var err error
	if oauth.CachePath != "" {
		cachedData, err := ioutil.ReadFile(oauth.CachePath)
		if err == nil {
			err = json.Unmarshal(cachedData, &token)
			if err == nil {
				if token.isExpired() {
					token, err = RefreshAccessToken(token.RefreshToken, oauth)
				}
				if err == nil {
					return token, nil
				}
			}
		}
	}
	return Token{}, err
}

func saveTokenInfo(token Token, oauth SpotifyOauth) error {
	var err error
	if oauth.CachePath != "" {
		marshaledToken, err := json.Marshal(token)
		if err == nil {
			err := ioutil.WriteFile(oauth.CachePath, marshaledToken, 0x777)
			return nil
		}
	}
	return err
}

// Takes a SpotifyOauth struct and returns the appropriate AuthorizeUrl for
// your request
func GetAuthorizeURL(oauth SpotifyOauth) (string, error) {
	var Url *url.URL
	Url, err := url.Parse(OauthAuthorizeURL)
	if err == nil {
		parameters := url.Values{}
		parameters.Add("client_id", oauth.ClientId)
		parameters.Add("response_type", "code")
		parameters.Add("redirect_uri", oauth.RedirectUri)
		if oauth.Scope != "" {
			parameters.Add("scope", oauth.Scope)
		}
		if oauth.State != "" {
			parameters.Add("state", oauth.State)
		}
		Url.RawQuery = parameters.Encode()
		return Url.String(), nil
	}
	return "", err
}

// Takes the authorization code and a SpotifyOauth
// Returns an access token
func GetAccessToken(code string, oauth SpotifyOauth) (Token, error) {
	var err error
	parameters := url.Values{}
	parameters.Add("redirect_uri", oauth.RedirectUri)
	parameters.Add("code", code)
	parameters.Add("grant_type", "authorization_code")

	token, err := sendAccessTokenRequest(parameters, oauth)
	if err == nil {
		err = saveTokenInfo(token, oauth)
		if err == nil {
			return token, nil
		}
	}
	return Token{}, err
}

// Refreshes an expired AccessToken
// TODO: TEST
func RefreshAccessToken(refreshToken string, oauth SpotifyOauth) (Token, error) {
	parameters := url.Values{}
	parameters.Add("refresh_token", refreshToken)
	parameters.Add("grant_type", "refresh_token")

	token, err := sendAccessTokenRequest(parameters, oauth)
	if err == nil {
		if token.RefreshToken == "" {
			token.RefreshToken = refreshToken
		}
		err = saveTokenInfo(token, oauth)
		if err == nil {
			return token, nil
		}
	}
	return Token{}, err
}

func sendAccessTokenRequest(parameters url.Values, oauth SpotifyOauth) (Token, error) {
	client := &http.Client{}
	req, err := http.NewRequest("POST", OauthTokenUrl, strings.NewReader(parameters.Encode()))
	if err == nil {
		req.SetBasicAuth(oauth.ClientId, oauth.ClientSecret)
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
		resp, err := client.Do(req)
		if err == nil {
			if resp.StatusCode == http.StatusOK {
				var token Token
				err = json.NewDecoder(resp.Body).Decode(&token)
				if err == nil {
					token.ExpiresAt = time.Now().Add(time.Duration(token.TTL) * time.Second)
					return token, nil
				}
			} else {
				err = errors.New(resp.Status)
			}
		}
	}
	return Token{}, err
}

// Parses the response code from from the query string when user is redirected
// back to the application
func ParseResponseCode(response string) (string, error) {
	u, err := url.Parse(response)
	if err != nil {
		return "", err
	}
	q, _ := url.ParseQuery(u.RawQuery)
	code := q["code"][0]
	return code, nil
}
<commit_msg>minor fixes<commit_after>// gotify is a client library for the Spotify API
package gotify

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const OauthAuthorizeURL = "https://accounts.spotify.com/authorize"
const OauthTokenUrl = "https://accounts.spotify.com/api/token"

type SpotifyOauth struct {
	ClientId     string
	ClientSecret string
	RedirectUri  string
	State        string
	Scope        string
	CachePath    string
}

type Token struct {
	AccessToken  string `json:"access_token"`
	TokenType    string `json:"token_type"`
	ExpiresAt    time.Time
	RefreshToken string `json:"refresh_token"`
	TTL          int    `json:"expires_in"`
}

func (t Token) isExpired() bool {
	if !(int(time.Since(t.ExpiresAt)) >= 0) {
		return false
	}
	return true
}

// Reads a cached token
func GetCachedToken(oauth SpotifyOauth) (Token, error) {
	var token Token
	var err error
	if oauth.CachePath != "" {
		cachedData, err := ioutil.ReadFile(oauth.CachePath)
		if err == nil {
			err = json.Unmarshal(cachedData, &token)
			if err == nil {
				if token.isExpired() {
					token, err = RefreshAccessToken(token.RefreshToken, oauth)
				}
				if err == nil {
					return token, nil
				}
			}
		}
	}
	return Token{}, err
}

// Takes a SpotifyOauth struct and returns the appropriate AuthorizeUrl for
// your request
func GetAuthorizeURL(oauth SpotifyOauth) (string, error) {
	var Url *url.URL
	Url, err := url.Parse(OauthAuthorizeURL)
	if err == nil {
		parameters := url.Values{}
		parameters.Add("client_id", oauth.ClientId)
		parameters.Add("response_type", "code")
		parameters.Add("redirect_uri", oauth.RedirectUri)
		if oauth.Scope != "" {
			parameters.Add("scope", oauth.Scope)
		}
		if oauth.State != "" {
			parameters.Add("state", oauth.State)
		}
		Url.RawQuery = parameters.Encode()
		return Url.String(), nil
	}
	return "", err
}

// Takes the authorization code and a SpotifyOauth
// Returns an access token
func GetAccessToken(code string, oauth SpotifyOauth) (Token, error) {
	var err error
	parameters := url.Values{}
	parameters.Add("redirect_uri", oauth.RedirectUri)
	parameters.Add("code", code)
	parameters.Add("grant_type", "authorization_code")

	token, err := sendAccessTokenRequest(parameters, oauth)
	if err == nil {
		err = saveTokenInfo(token, oauth)
		if err == nil {
			return token, nil
		}
	}
	return Token{}, err
}

// Refreshes an expired AccessToken
func RefreshAccessToken(refreshToken string, oauth SpotifyOauth) (Token, error) {
	parameters := url.Values{}
	parameters.Add("refresh_token", refreshToken)
	parameters.Add("grant_type", "refresh_token")

	token, err := sendAccessTokenRequest(parameters, oauth)
	if err == nil {
		if token.RefreshToken == "" {
			token.RefreshToken = refreshToken
		}
		err = saveTokenInfo(token, oauth)
		if err == nil {
			return token, nil
		}
	}
	return Token{}, err
}

// Parses the response code from from the query string when user is redirected
// back to the application
func ParseResponseCode(response string) (string, error) {
	u, err := url.Parse(response)
	if err != nil {
		return "", err
	}
	q, _ := url.ParseQuery(u.RawQuery)
	code := q["code"][0]
	return code, nil
}

func sendAccessTokenRequest(parameters url.Values, oauth SpotifyOauth) (Token, error) {
	client := &http.Client{}
	req, err := http.NewRequest("POST", OauthTokenUrl, strings.NewReader(parameters.Encode()))
	if err == nil {
		req.SetBasicAuth(oauth.ClientId, oauth.ClientSecret)
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
		resp, err := client.Do(req)
		if err == nil {
			if resp.StatusCode == http.StatusOK {
				var token Token
				err = json.NewDecoder(resp.Body).Decode(&token)
				if err == nil {
					token.ExpiresAt = time.Now().Add(time.Duration(token.TTL) * time.Second)
					return token, nil
				}
			} else {
				err = errors.New(resp.Status)
			}
		}
	}
	return Token{}, err
}

func saveTokenInfo(token Token, oauth SpotifyOauth) error {
	var err error
	if oauth.CachePath != "" {
		marshaledToken, err := json.Marshal(token)
		if err == nil {
			err = ioutil.WriteFile(oauth.CachePath, marshaledToken, 0x777)
			return nil
		}
	}
	return err
}
<|endoftext|>


<commit_before>package apicache

import (
	"net/http"
	"strconv"

	"time"

	"github.com/prometheus/client_golang/prometheus"
)

// Custom transport to chain into the HTTPClient to gather statistics.
type transport struct {
	next *http.Transport
}

// RoundTrip wraps http.DefaultTransport.RoundTrip to provide stats and handle error rates.
func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {

	// Loop until success
	tries := 0
	for {
		esiRateLimiter := true

		// Time our response
		start := time.Now().Nanosecond()

		// Tickup retry counter
		tries++

		// Run the request.
		res, err := t.next.RoundTrip(req)

		// We got a response
		if res != nil {
			// Get the ESI error information
			resetS := res.Header.Get("x-esi-error-limit-reset")
			tokensS := res.Header.Get("x-esi-error-limit-remain")

			// If we cannot decode this is likely from another source.
			reset, err := strconv.Atoi(resetS)
			if err != nil {
				esiRateLimiter = false
			}
			tokens, err := strconv.Atoi(tokensS)
			if err != nil {
				esiRateLimiter = false
			}

			duration := time.Now().Nanosecond() - start
			metricAPICalls.With(
				prometheus.Labels{"host": req.Host},
			).Observe(float64(duration))

			// Sleep to prevent hammering CCP ESI if there are excessive errors
			if esiRateLimiter {
				time.Sleep(time.Duration(float64(reset*3) * (1 - (float64(tokens) / 100))))
			}

			if res.StatusCode == 420 {
				time.Sleep(time.Duration(reset))
			}

			if res.StatusCode == 420 || res.StatusCode >= 500 || res.StatusCode == 0 {
				metricAPIErrors.Inc()
				// break out after 10 tries
				if tries > 10 {
					return res, err
				}
				if !esiRateLimiter {
					time.Sleep(time.Duration(tries * 2))
				}
				continue
			}
		}

		return res, err
	}
}

var (
	metricAPICalls = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "evedata",
		Subsystem: "api",
		Name:      "calls",
		Help:      "API call statistics.",
		Buckets:   prometheus.LinearBuckets(0, 50, 25),
	},
		[]string{"host"},
	)

	metricAPIErrors = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "evedata",
		Subsystem: "api",
		Name:      "errors",
		Help:      "Count of API errors.",
	})
)

func init() {
	prometheus.MustRegister(
		metricAPICalls,
		metricAPIErrors,
	)
}
<commit_msg>tweek buckets<commit_after>package apicache

import (
	"net/http"
	"strconv"

	"time"

	"github.com/prometheus/client_golang/prometheus"
)

// Custom transport to chain into the HTTPClient to gather statistics.
type transport struct {
	next *http.Transport
}

// RoundTrip wraps http.DefaultTransport.RoundTrip to provide stats and handle error rates.
func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {

	// Loop until success
	tries := 0
	for {
		esiRateLimiter := true

		// Time our response
		start := time.Now().Nanosecond()

		// Tickup retry counter
		tries++

		// Run the request.
		res, err := t.next.RoundTrip(req)

		// We got a response
		if res != nil {
			// Get the ESI error information
			resetS := res.Header.Get("x-esi-error-limit-reset")
			tokensS := res.Header.Get("x-esi-error-limit-remain")

			// If we cannot decode this is likely from another source.
			reset, err := strconv.Atoi(resetS)
			if err != nil {
				esiRateLimiter = false
			}
			tokens, err := strconv.Atoi(tokensS)
			if err != nil {
				esiRateLimiter = false
			}

			duration := (time.Now().Nanosecond() - start) / 1000000
			metricAPICalls.With(
				prometheus.Labels{"host": req.Host},
			).Observe(float64(duration))

			// Sleep to prevent hammering CCP ESI if there are excessive errors
			if esiRateLimiter {
				time.Sleep(time.Duration(float64(reset*3) * (1 - (float64(tokens) / 100))))
			}

			if res.StatusCode == 420 {
				time.Sleep(time.Duration(reset))
			}

			if res.StatusCode == 420 || res.StatusCode >= 500 || res.StatusCode == 0 {
				metricAPIErrors.Inc()
				// break out after 10 tries
				if tries > 10 {
					return res, err
				}
				if !esiRateLimiter {
					time.Sleep(time.Duration(tries * 2))
				}
				continue
			}
		}

		return res, err
	}
}

var (
	metricAPICalls = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "evedata",
		Subsystem: "api",
		Name:      "calls",
		Help:      "API call statistics.",
		Buckets:   prometheus.ExponentialBuckets(10, 1.45, 20),
	},
		[]string{"host"},
	)

	metricAPIErrors = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "evedata",
		Subsystem: "api",
		Name:      "errors",
		Help:      "Count of API errors.",
	})
)

func init() {
	prometheus.MustRegister(
		metricAPICalls,
		metricAPIErrors,
	)
}
<|endoftext|>