<commit_before>package surf

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/headzoo/surf/errors"
	"github.com/headzoo/surf/jars"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"runtime"
	"strings"
	"syscall"
	"time"
)

const (
	// Name is used as the browser name in the default user agent.
	Name = "Surf"
	// Version is used as the version in the default user agent.
	Version = "0.4.2"
)

// Attribute represents a Browser capability.
type Attribute int

// AttributeMap represents a map of Attribute values.
type AttributeMap map[Attribute]bool

const (
	// SendRefererAttribute instructs a Browser to send the Referer header.
	SendRefererAttribute Attribute = iota
	// MetaRefreshHandlingAttribute instructs a Browser to handle the refresh meta tag.
	MetaRefreshHandlingAttribute
	// FollowRedirectsAttribute instructs a Browser to follow Location headers.
	FollowRedirectsAttribute
)

var (
	// DefaultUserAgent is the global user agent value.
	DefaultUserAgent string = fmt.Sprintf("%s/%s (%s; %s)", Name, Version, runtime.Version(), osRelease())
	// DefaultSendRefererAttribute is the global value for the AttributeSendReferer attribute.
	DefaultSendRefererAttribute bool = true
	// DefaultMetaRefreshHandlingAttribute is the global value for the AttributeHandleRefresh attribute.
	DefaultMetaRefreshHandlingAttribute bool = true
	// DefaultFollowRedirectsAttribute is the global value for the AttributeFollowRedirects attribute.
	DefaultFollowRedirectsAttribute bool = true
)

// exprPrefixesImplied are strings a selection expr may start with, and the tag is implied.
var exprPrefixesImplied = []string{":", ".", "["}

// Browsable represents an HTTP web browser.
type Browsable interface {
	Document
	Get(url string) error
	GetForm(url string, data url.Values) error
	GetBookmark(name string) error
	Post(url string, bodyType string, body io.Reader) error
	PostForm(url string, data url.Values) error
	Bookmark(name string) error
	FollowLink(expr string) error
	Links() []string
	Form(expr string) (FormElement, error)
	Forms() []FormElement
	Back() bool
	Reload() error
	SiteCookies() []*http.Cookie
	SetAttribute(a Attribute, v bool)
	ResolveUrl(u *url.URL) *url.URL
}

// Browser is the default Browser implementation.
type Browser struct {
	*Page
	UserAgent   string
	Cookies   http.CookieJar
	Bookmarks   jars.BookmarksJar
	History     *PageStack
	lastRequest *http.Request
	attributes  AttributeMap
	refresh     *time.Timer
}

// NewBrowser creates and returns a *Browser type.
func NewBrowser() (*Browser, error) {
	cookies, err := cookiejar.New(nil)
	if err != nil {
		return nil, err
	}

	return &Browser{
		UserAgent: DefaultUserAgent,
		Cookies: cookies,
		Bookmarks: jars.NewMemoryBookmarks(),
		History:   NewPageStack(),
		attributes: AttributeMap{
			SendRefererAttribute:         DefaultSendRefererAttribute,
			MetaRefreshHandlingAttribute: DefaultMetaRefreshHandlingAttribute,
			FollowRedirectsAttribute:     DefaultFollowRedirectsAttribute,
		},
	}, nil
}

// Get requests the given URL using the GET method.
func (b *Browser) Get(u string) error {
	return b.sendGet(u, nil)
}

// GetForm appends the data values to the given URL and sends a GET request.
func (b *Browser) GetForm(u string, data url.Values) error {
	ul, err := url.Parse(u)
	if err != nil {
		return err
	}
	ul.RawQuery = data.Encode()

	return b.Get(ul.String())
}

// GetBookmark calls Get() with the URL for the bookmark with the given name.
func (b *Browser) GetBookmark(name string) error {
	url, err := b.Bookmarks.Read(name)
	if err != nil {
		return err
	}
	return b.Get(url)
}

// Post requests the given URL using the POST method.
func (b *Browser) Post(u string, bodyType string, body io.Reader) error {
	return b.sendPost(u, bodyType, body, nil)
}

// PostForm requests the given URL using the POST method with the given data.
func (b *Browser) PostForm(u string, data url.Values) error {
	return b.Post(u, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
}

// Bookmark saves the page URL in the bookmarks with the given name.
func (b *Browser) Bookmark(name string) error {
	return b.Bookmarks.Save(name, b.ResolveUrl(b.Page.Url()).String())
}

// FollowLink finds an anchor tag within the current document matching the expr,
// and calls Get() using the anchor href attribute value.
//
// The expr can be any valid goquery expression, and the "a" tag is implied. The
// method can be called using only ":contains('foo')" and the expr is automatically
// converted to "a:contains('foo')". A complete expression can still be used, for
// instance "p.title a.foo".
func (b *Browser) FollowLink(expr string) error {
	sel := b.Page.doc.Find(prefixSelection(expr, "a"))
	if sel.Length() == 0 {
		return errors.NewElementNotFound(
			"Anchor not found matching expr '%s'.", expr)
	}
	if !sel.Is("a") {
		return errors.NewElementNotFound(
			"Expr '%s' does not match an anchor tag.", expr)
	}

	href, ok := sel.Attr("href")
	if !ok {
		return errors.NewLinkNotFound("No link found matching expr %s.", expr)
	}
	hurl, err := url.Parse(href)
	if err != nil {
		return err
	}
	hurl = b.ResolveUrl(hurl)

	return b.sendGet(hurl.String(), b.Page)
}

// Links returns an array of every anchor tag href value found in the current page.
func (b *Browser) Links() []string {
	sel := b.Page.doc.Find("a")
	links := make([]string, 0, sel.Length())
	sel.Each(func(_ int, s *goquery.Selection) {
		href, ok := s.Attr("href")
		if ok {
			links = append(links, href)
		}
	})

	return links
}

// SiteCookies returns the cookies for the current site.
func (b *Browser) SiteCookies() []*http.Cookie {
	return b.Cookies.Cookies(b.Page.Url())
}

// Form returns the form in the current page that matches the given expr.
func (b *Browser) Form(expr string) (FormElement, error) {
	sel := b.Page.doc.Find(prefixSelection(expr, "form"))
	if sel.Length() == 0 {
		return nil, errors.NewElementNotFound(
			"Form not found matching expr '%s'.", expr)
	}
	if !sel.Is("form") {
		return nil, errors.NewElementNotFound(
			"Expr '%s' does not match a form tag.", expr)
	}

	return NewForm(b, sel), nil
}

// Forms returns an array of every form in the page.
func (b *Browser) Forms() []FormElement {
	sel := b.Page.doc.Find("form")
	len := sel.Length()
	if len == 0 {
		return nil
	}

	forms := make([]FormElement, len)
	sel.Each(func(_ int, s *goquery.Selection) {
		forms = append(forms, NewForm(b, s))
	})
	return forms
}

// Back loads the previously requested page.
func (b *Browser) Back() bool {
	if b.History.Len() > 0 {
		b.Page = b.History.Pop()
		return true
	}
	return false
}

// Reload duplicates the last successful request.
func (b *Browser) Reload() error {
	if b.lastRequest != nil {
		return b.send(b.lastRequest)
	}
	return errors.NewPageNotLoaded("Cannot reload, the previous request failed.")
}

// SetAttribute sets a browser instruction attribute.
func (b *Browser) SetAttribute(a Attribute, v bool) {
	b.attributes[a] = v
}

// ResolveUrl returns an absolute URL for a possibly relative URL.
func (b *Browser) ResolveUrl(u *url.URL) *url.URL {
	return b.Url().ResolveReference(u)
}

// client creates, configures, and returns a *http.Client type.
func (b *Browser) client() *http.Client {
	client := &http.Client{}
	client.Jar = b.Cookies
	client.CheckRedirect = b.shouldRedirect
	return client
}

// request creates and returns a *http.Request type.
// Sets any headers that need to be sent with the request.
func (b *Browser) request(method, url string) (*http.Request, error) {
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		return nil, err
	}
	req.Header["User-Agent"] = []string{b.UserAgent}

	return req, nil
}

// sendGet makes an HTTP GET request for the given URL.
// When via is not nil, and AttributeSendReferer is true, the Referer header will
// be set to via's URL.
func (b *Browser) sendGet(url string, via *Page) error {
	req, err := b.request("GET", url)
	if err != nil {
		return err
	}
	if b.attributes[SendRefererAttribute] && via != nil {
		req.Header["Referer"] = []string{via.Url().String()}
	}

	return b.send(req)
}

// sendPost makes an HTTP POST request for the given URL.
// When via is not nil, and AttributeSendReferer is true, the Referer header will
// be set to via's URL.
func (b *Browser) sendPost(url string, bodyType string, body io.Reader, via *Page) error {
	req, err := b.request("POST", url)
	if err != nil {
		return err
	}
	rc, ok := body.(io.ReadCloser)
	if !ok && body != nil {
		rc = ioutil.NopCloser(body)
	}
	req.Body = rc
	req.Header["Content-Type"] = []string{bodyType}
	if b.attributes[SendRefererAttribute] && via != nil {
		req.Header["Referer"] = []string{via.Url().String()}
	}

	return b.send(req)
}

// send uses the given *http.Request to make an HTTP request.
func (b *Browser) send(req *http.Request) error {
	b.preSend()
	resp, err := b.client().Do(req)
	if err != nil {
		return err
	}
	body, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return err
	}

	b.lastRequest = req
	b.History.Push(b.Page)
	b.Page = NewPage(resp, body)
	b.postSend()

	return nil
}

// preSend sets browser state before sending a request.
func (b *Browser) preSend() {
	if b.refresh != nil {
		b.refresh.Stop()
	}
}

// postSend sets browser state after sending a request.
func (b *Browser) postSend() {
	if b.attributes[MetaRefreshHandlingAttribute] {
		sel := b.Page.doc.Find("meta[http-equiv='refresh']")
		if sel.Length() > 0 {
			attr, ok := sel.Attr("content")
			if ok {
				dur, err := time.ParseDuration(attr + "s")
				if err == nil {
					b.refresh = time.NewTimer(dur)
					go func() {
						<-b.refresh.C
						b.Reload()
					}()
				}
			}
		}
	}
}

// shouldRedirect is used as the value to http.Client.CheckRedirect.
func (b *Browser) shouldRedirect(req *http.Request, _ []*http.Request) error {
	if b.attributes[FollowRedirectsAttribute] {
		return nil
	}
	return errors.NewLocation(
		"Redirects are disabled. Cannot follow '%s'.", req.URL.String())
}

// prefixSelection prefixes a selection expr with elm when sel is prefixed with
// one of the values from exprPrefixesImplied.
func prefixSelection(sel, elm string) string {
	for _, prefix := range exprPrefixesImplied {
		if strings.HasPrefix(sel, prefix) {
			return elm + sel
		}
	}
	return sel
}

// osRelease returns the name of the OS and it's release version.
func osRelease() string {
	buf := &syscall.Utsname{}
	err := syscall.Uname(buf)
	if err != nil {
		return "0.0"
	}

	return charsToString(buf.Sysname) + "/" + charsToString(buf.Release)
}

// charsToString converts a [65]int8 byte array into a string.
func charsToString(ca [65]int8) string {
	s := make([]byte, len(ca))
	var lens int
	for ; lens < len(ca); lens++ {
		if ca[lens] == 0 {
			break
		}
		s[lens] = uint8(ca[lens])
	}
	return string(s[0:lens])
}
<commit_msg>Incremented version number<commit_after>package surf

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/headzoo/surf/errors"
	"github.com/headzoo/surf/jars"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"runtime"
	"strings"
	"syscall"
	"time"
)

const (
	// Name is used as the browser name in the default user agent.
	Name = "Surf"
	// Version is used as the version in the default user agent.
	Version = "0.4.3"
)

// Attribute represents a Browser capability.
type Attribute int

// AttributeMap represents a map of Attribute values.
type AttributeMap map[Attribute]bool

const (
	// SendRefererAttribute instructs a Browser to send the Referer header.
	SendRefererAttribute Attribute = iota
	// MetaRefreshHandlingAttribute instructs a Browser to handle the refresh meta tag.
	MetaRefreshHandlingAttribute
	// FollowRedirectsAttribute instructs a Browser to follow Location headers.
	FollowRedirectsAttribute
)

var (
	// DefaultUserAgent is the global user agent value.
	DefaultUserAgent string = fmt.Sprintf("%s/%s (%s; %s)", Name, Version, runtime.Version(), osRelease())
	// DefaultSendRefererAttribute is the global value for the AttributeSendReferer attribute.
	DefaultSendRefererAttribute bool = true
	// DefaultMetaRefreshHandlingAttribute is the global value for the AttributeHandleRefresh attribute.
	DefaultMetaRefreshHandlingAttribute bool = true
	// DefaultFollowRedirectsAttribute is the global value for the AttributeFollowRedirects attribute.
	DefaultFollowRedirectsAttribute bool = true
)

// exprPrefixesImplied are strings a selection expr may start with, and the tag is implied.
var exprPrefixesImplied = []string{":", ".", "["}

// Browsable represents an HTTP web browser.
type Browsable interface {
	Document
	Get(url string) error
	GetForm(url string, data url.Values) error
	GetBookmark(name string) error
	Post(url string, bodyType string, body io.Reader) error
	PostForm(url string, data url.Values) error
	Bookmark(name string) error
	FollowLink(expr string) error
	Links() []string
	Form(expr string) (FormElement, error)
	Forms() []FormElement
	Back() bool
	Reload() error
	SiteCookies() []*http.Cookie
	SetAttribute(a Attribute, v bool)
	ResolveUrl(u *url.URL) *url.URL
}

// Browser is the default Browser implementation.
type Browser struct {
	*Page
	UserAgent   string
	Cookies   http.CookieJar
	Bookmarks   jars.BookmarksJar
	History     *PageStack
	lastRequest *http.Request
	attributes  AttributeMap
	refresh     *time.Timer
}

// NewBrowser creates and returns a *Browser type.
func NewBrowser() (*Browser, error) {
	cookies, err := cookiejar.New(nil)
	if err != nil {
		return nil, err
	}

	return &Browser{
		UserAgent: DefaultUserAgent,
		Cookies: cookies,
		Bookmarks: jars.NewMemoryBookmarks(),
		History:   NewPageStack(),
		attributes: AttributeMap{
			SendRefererAttribute:         DefaultSendRefererAttribute,
			MetaRefreshHandlingAttribute: DefaultMetaRefreshHandlingAttribute,
			FollowRedirectsAttribute:     DefaultFollowRedirectsAttribute,
		},
	}, nil
}

// Get requests the given URL using the GET method.
func (b *Browser) Get(u string) error {
	return b.sendGet(u, nil)
}

// GetForm appends the data values to the given URL and sends a GET request.
func (b *Browser) GetForm(u string, data url.Values) error {
	ul, err := url.Parse(u)
	if err != nil {
		return err
	}
	ul.RawQuery = data.Encode()

	return b.Get(ul.String())
}

// GetBookmark calls Get() with the URL for the bookmark with the given name.
func (b *Browser) GetBookmark(name string) error {
	url, err := b.Bookmarks.Read(name)
	if err != nil {
		return err
	}
	return b.Get(url)
}

// Post requests the given URL using the POST method.
func (b *Browser) Post(u string, bodyType string, body io.Reader) error {
	return b.sendPost(u, bodyType, body, nil)
}

// PostForm requests the given URL using the POST method with the given data.
func (b *Browser) PostForm(u string, data url.Values) error {
	return b.Post(u, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
}

// Bookmark saves the page URL in the bookmarks with the given name.
func (b *Browser) Bookmark(name string) error {
	return b.Bookmarks.Save(name, b.ResolveUrl(b.Page.Url()).String())
}

// FollowLink finds an anchor tag within the current document matching the expr,
// and calls Get() using the anchor href attribute value.
//
// The expr can be any valid goquery expression, and the "a" tag is implied. The
// method can be called using only ":contains('foo')" and the expr is automatically
// converted to "a:contains('foo')". A complete expression can still be used, for
// instance "p.title a.foo".
func (b *Browser) FollowLink(expr string) error {
	sel := b.Page.doc.Find(prefixSelection(expr, "a"))
	if sel.Length() == 0 {
		return errors.NewElementNotFound(
			"Anchor not found matching expr '%s'.", expr)
	}
	if !sel.Is("a") {
		return errors.NewElementNotFound(
			"Expr '%s' does not match an anchor tag.", expr)
	}

	href, ok := sel.Attr("href")
	if !ok {
		return errors.NewLinkNotFound("No link found matching expr %s.", expr)
	}
	hurl, err := url.Parse(href)
	if err != nil {
		return err
	}
	hurl = b.ResolveUrl(hurl)

	return b.sendGet(hurl.String(), b.Page)
}

// Links returns an array of every anchor tag href value found in the current page.
func (b *Browser) Links() []string {
	sel := b.Page.doc.Find("a")
	links := make([]string, 0, sel.Length())
	sel.Each(func(_ int, s *goquery.Selection) {
		href, ok := s.Attr("href")
		if ok {
			links = append(links, href)
		}
	})

	return links
}

// SiteCookies returns the cookies for the current site.
func (b *Browser) SiteCookies() []*http.Cookie {
	return b.Cookies.Cookies(b.Page.Url())
}

// Form returns the form in the current page that matches the given expr.
func (b *Browser) Form(expr string) (FormElement, error) {
	sel := b.Page.doc.Find(prefixSelection(expr, "form"))
	if sel.Length() == 0 {
		return nil, errors.NewElementNotFound(
			"Form not found matching expr '%s'.", expr)
	}
	if !sel.Is("form") {
		return nil, errors.NewElementNotFound(
			"Expr '%s' does not match a form tag.", expr)
	}

	return NewForm(b, sel), nil
}

// Forms returns an array of every form in the page.
func (b *Browser) Forms() []FormElement {
	sel := b.Page.doc.Find("form")
	len := sel.Length()
	if len == 0 {
		return nil
	}

	forms := make([]FormElement, len)
	sel.Each(func(_ int, s *goquery.Selection) {
		forms = append(forms, NewForm(b, s))
	})
	return forms
}

// Back loads the previously requested page.
func (b *Browser) Back() bool {
	if b.History.Len() > 0 {
		b.Page = b.History.Pop()
		return true
	}
	return false
}

// Reload duplicates the last successful request.
func (b *Browser) Reload() error {
	if b.lastRequest != nil {
		return b.send(b.lastRequest)
	}
	return errors.NewPageNotLoaded("Cannot reload, the previous request failed.")
}

// SetAttribute sets a browser instruction attribute.
func (b *Browser) SetAttribute(a Attribute, v bool) {
	b.attributes[a] = v
}

// ResolveUrl returns an absolute URL for a possibly relative URL.
func (b *Browser) ResolveUrl(u *url.URL) *url.URL {
	return b.Url().ResolveReference(u)
}

// client creates, configures, and returns a *http.Client type.
func (b *Browser) client() *http.Client {
	client := &http.Client{}
	client.Jar = b.Cookies
	client.CheckRedirect = b.shouldRedirect
	return client
}

// request creates and returns a *http.Request type.
// Sets any headers that need to be sent with the request.
func (b *Browser) request(method, url string) (*http.Request, error) {
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		return nil, err
	}
	req.Header["User-Agent"] = []string{b.UserAgent}

	return req, nil
}

// sendGet makes an HTTP GET request for the given URL.
// When via is not nil, and AttributeSendReferer is true, the Referer header will
// be set to via's URL.
func (b *Browser) sendGet(url string, via *Page) error {
	req, err := b.request("GET", url)
	if err != nil {
		return err
	}
	if b.attributes[SendRefererAttribute] && via != nil {
		req.Header["Referer"] = []string{via.Url().String()}
	}

	return b.send(req)
}

// sendPost makes an HTTP POST request for the given URL.
// When via is not nil, and AttributeSendReferer is true, the Referer header will
// be set to via's URL.
func (b *Browser) sendPost(url string, bodyType string, body io.Reader, via *Page) error {
	req, err := b.request("POST", url)
	if err != nil {
		return err
	}
	rc, ok := body.(io.ReadCloser)
	if !ok && body != nil {
		rc = ioutil.NopCloser(body)
	}
	req.Body = rc
	req.Header["Content-Type"] = []string{bodyType}
	if b.attributes[SendRefererAttribute] && via != nil {
		req.Header["Referer"] = []string{via.Url().String()}
	}

	return b.send(req)
}

// send uses the given *http.Request to make an HTTP request.
func (b *Browser) send(req *http.Request) error {
	b.preSend()
	resp, err := b.client().Do(req)
	if err != nil {
		return err
	}
	body, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return err
	}

	b.lastRequest = req
	b.History.Push(b.Page)
	b.Page = NewPage(resp, body)
	b.postSend()

	return nil
}

// preSend sets browser state before sending a request.
func (b *Browser) preSend() {
	if b.refresh != nil {
		b.refresh.Stop()
	}
}

// postSend sets browser state after sending a request.
func (b *Browser) postSend() {
	if b.attributes[MetaRefreshHandlingAttribute] {
		sel := b.Page.doc.Find("meta[http-equiv='refresh']")
		if sel.Length() > 0 {
			attr, ok := sel.Attr("content")
			if ok {
				dur, err := time.ParseDuration(attr + "s")
				if err == nil {
					b.refresh = time.NewTimer(dur)
					go func() {
						<-b.refresh.C
						b.Reload()
					}()
				}
			}
		}
	}
}

// shouldRedirect is used as the value to http.Client.CheckRedirect.
func (b *Browser) shouldRedirect(req *http.Request, _ []*http.Request) error {
	if b.attributes[FollowRedirectsAttribute] {
		return nil
	}
	return errors.NewLocation(
		"Redirects are disabled. Cannot follow '%s'.", req.URL.String())
}

// prefixSelection prefixes a selection expr with elm when sel is prefixed with
// one of the values from exprPrefixesImplied.
func prefixSelection(sel, elm string) string {
	for _, prefix := range exprPrefixesImplied {
		if strings.HasPrefix(sel, prefix) {
			return elm + sel
		}
	}
	return sel
}

// osRelease returns the name of the OS and it's release version.
func osRelease() string {
	buf := &syscall.Utsname{}
	err := syscall.Uname(buf)
	if err != nil {
		return "0.0"
	}

	return charsToString(buf.Sysname) + "/" + charsToString(buf.Release)
}

// charsToString converts a [65]int8 byte array into a string.
func charsToString(ca [65]int8) string {
	s := make([]byte, len(ca))
	var lens int
	for ; lens < len(ca); lens++ {
		if ca[lens] == 0 {
			break
		}
		s[lens] = uint8(ca[lens])
	}
	return string(s[0:lens])
}
<|endoftext|>


<commit_before>package dsbldr

import (
	"io"
	"io/ioutil"
	"net/http"
)

// Builder is main type for this tool.
type Builder struct {
	BaseURL        string
	RequestHeaders map[string]string // Manually written Request Header (including auth)
	featureMap     map[string]*Feature
	dataMap        map[string][]string
	data           [][]string // Strings of Data to be read in to CSV
	request        http.Request
}

// NewBuilder creates new Builder struct
func NewBuilder(featureCount, recordCount int) *Builder {
	// Add extra row for header
	preallocatedData := make([][]string, recordCount+1)
	for i := range preallocatedData {
		preallocatedData[i] = make([]string, featureCount)
	}
	return &Builder{
		data: preallocatedData,
	}
}

func (b *Builder) addDataFeature(featureName string, values []string) error {
	// First row is table headers
	var colIndex int
	for i := range b.data[0] {
		// Find first empty column
		if b.data[0][i] == "" {
			colIndex = i
			b.data[0][i] = featureName
			break
		}
	}
	// Add all the values as well (remember that Builder.data is pre-allocated)
	// for i := 1; i < len(b.data); i++ {
	// 	// fmt.Printf("%v", values[i])
	//
	// }
	for i := 1; i < len(b.data); i++ {
		b.data[i][colIndex] = values[i-1]
	}
	return nil
}

func (b *Builder) createClientAndRequest(endpoint string, headers map[string]string) (*http.Client, *http.Request, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", b.BaseURL+endpoint, nil)
	for k, v := range headers {
		req.Header.Add(k, v)
	}
	return client, req, err
}

// Run Builder to aggregate all features and manage concurrent operations
func (b *Builder) Run() error {
	for _, feature := range b.featureMap {
		client, req, err := b.createClientAndRequest(feature.Endpoint, b.RequestHeaders)
		if err != nil {
			return err
		}
		resp, err := client.Do(req)
		parsedResponse, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		output := feature.RunFunc(string(parsedResponse))
		b.addDataFeature(feature.Name, output)
	}
	return nil
}

// Save commits the downloaded features to a file
// as specified by the Writer interface (has to implement WriteAll)
func (b *Builder) Save(writer io.Writer) error {
	// err := writer.WriteAll(b.data)
	return nil
}

// AddFeatures adds a Feature struct to the "Features" Field on Builder
func (b *Builder) AddFeatures(features ...*Feature) {
	for _, feature := range features {
		b.featureMap[feature.Name] = feature
	}
}

// GetFeature returns a feature in the detaset based on it's name
func (b *Builder) GetFeature(name string) *Feature {
	if val, ok := b.featureMap[name]; ok {
		return val
	}
	return nil
}
<commit_msg>Remove unnecessary comments<commit_after>package dsbldr

import (
	"io"
	"io/ioutil"
	"net/http"
)

// Builder is main type for this tool.
type Builder struct {
	BaseURL        string
	RequestHeaders map[string]string // Manually written Request Header (including auth)
	featureMap     map[string]*Feature
	dataMap        map[string][]string
	data           [][]string // Strings of Data to be read in to CSV
	request        http.Request
}

// NewBuilder creates new Builder struct
func NewBuilder(featureCount, recordCount int) *Builder {
	// Add extra row for header
	preallocatedData := make([][]string, recordCount+1)
	for i := range preallocatedData {
		preallocatedData[i] = make([]string, featureCount)
	}
	return &Builder{
		data: preallocatedData,
	}
}

func (b *Builder) addDataFeature(featureName string, values []string) error {
	// First row is table headers
	var colIndex int
	for i := range b.data[0] {
		// Find first empty column
		if b.data[0][i] == "" {
			colIndex = i
			b.data[0][i] = featureName
			break
		}
	}
	// Add all the values as well (remember that Builder.data is pre-allocated)
	for i := 1; i < len(b.data); i++ {
		b.data[i][colIndex] = values[i-1]
	}
	return nil
}

func (b *Builder) createClientAndRequest(endpoint string, headers map[string]string) (*http.Client, *http.Request, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", b.BaseURL+endpoint, nil)
	for k, v := range headers {
		req.Header.Add(k, v)
	}
	return client, req, err
}

// Run Builder to aggregate all features and manage concurrent operations
func (b *Builder) Run() error {
	for _, feature := range b.featureMap {
		client, req, err := b.createClientAndRequest(feature.Endpoint, b.RequestHeaders)
		if err != nil {
			return err
		}
		resp, err := client.Do(req)
		parsedResponse, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		output := feature.RunFunc(string(parsedResponse))
		b.addDataFeature(feature.Name, output)
	}
	return nil
}

// Save commits the downloaded features to a file
// as specified by the Writer interface (has to implement WriteAll)
func (b *Builder) Save(writer io.Writer) error {
	// err := writer.WriteAll(b.data)
	return nil
}

// AddFeatures adds a Feature struct to the "Features" Field on Builder
func (b *Builder) AddFeatures(features ...*Feature) {
	for _, feature := range features {
		b.featureMap[feature.Name] = feature
	}
}

// GetFeature returns a feature in the detaset based on it's name
func (b *Builder) GetFeature(name string) *Feature {
	if val, ok := b.featureMap[name]; ok {
		return val
	}
	return nil
}
<|endoftext|>


<commit_before>package qb

import (
	"fmt"
	"strings"
)

// NewBuilder generates a new builder struct
func NewBuilder(driver string) *Builder {
	return &Builder{
		query:   NewQuery(),
		adapter: NewAdapter(driver),
	}
}

// Builder is a struct that holds an active query that it is used for building common sql queries
// it has all the common functions except multiple statements & table crudders
type Builder struct {
	query   *Query
	adapter Adapter
}

// SetEscaping sets the escaping parameter of current adapter
func (b *Builder) SetEscaping(escaping bool) {
	b.adapter.SetEscaping(escaping)
}

// Adapter returns the active adapter of builder
func (b *Builder) Adapter() Adapter {
	return b.adapter
}

// Reset clears query bindings and its errors
func (b *Builder) Reset() {
	b.query = NewQuery()
	b.adapter.Reset()
}

// Query returns the active query and resets the query.
// The query clauses and returns the sql and bindings
func (b *Builder) Query() *Query {
	query := b.query
	b.Reset()
	//fmt.Printf("\n%s\n%s\n", query.SQL(), query.Bindings())
	return query
}

// Insert generates an "insert into %s(%s)" statement
func (b *Builder) Insert(table string) *Builder {
	clause := fmt.Sprintf("INSERT INTO %s", b.adapter.Escape(table))
	b.query.AddClause(clause)
	return b
}

// Values generates "values(%s)" statement and add bindings for each value
func (b *Builder) Values(m map[string]interface{}) *Builder {
	keys := []string{}
	values := []interface{}{}
	for k, v := range m {
		keys = append(keys, b.adapter.Escape(k))
		values = append(values, v)
		b.query.AddBinding(v)
	}

	b.query.AddClause(fmt.Sprintf("(%s)", strings.Join(keys, ", ")))

	placeholders := []string{}

	for _ = range values {
		placeholders = append(placeholders, b.adapter.Placeholder())
	}
	clause := fmt.Sprintf("VALUES (%s)", strings.Join(placeholders, ", "))
	b.query.AddClause(clause)
	return b
}

// Returning generates RETURNING statement for postgres only
// NOTE: Do not use it with sqlite, mysql or other drivers
func (b *Builder) Returning(cols ...string) *Builder {
	cols = b.adapter.EscapeAll(cols)
	clause := fmt.Sprintf("RETURNING %s", strings.Join(cols, ", "))
	b.query.AddClause(clause)
	return b
}

// Update generates "update %s" statement
func (b *Builder) Update(table string) *Builder {
	clause := fmt.Sprintf("UPDATE %s", b.adapter.Escape(table))
	b.query.AddClause(clause)
	return b
}

// Set generates "set a = placeholder" statement for each key a and add bindings for map value
func (b *Builder) Set(m map[string]interface{}) *Builder {
	updates := []string{}
	for k, v := range m {
		// check if aliasing exists
		if strings.Contains(k, ".") {
			kpieces := strings.Split(k, ".")
			k = fmt.Sprintf("%s.%s", kpieces[0], b.adapter.Escape(kpieces[1]))
		} else {
			k = b.adapter.Escape(k)
		}
		updates = append(updates, fmt.Sprintf("%s = %s", k, b.adapter.Placeholder()))
		b.query.AddBinding(v)
	}
	clause := fmt.Sprintf("SET %s", strings.Join(updates, ", "))
	b.query.AddClause(clause)
	return b
}

// Delete generates "delete" statement
func (b *Builder) Delete(table string) *Builder {
	b.query.AddClause(fmt.Sprintf("DELETE FROM %s", b.adapter.Escape(table)))
	return b
}

// Select generates "select %s" statement
func (b *Builder) Select(columns ...string) *Builder {
	clause := fmt.Sprintf("SELECT %s", strings.Join(columns, ", "))
	b.query.AddClause(clause)
	return b
}

// From generates "from %s" statement for each table name
func (b *Builder) From(tables ...string) *Builder {
	tbls := []string{}
	for _, v := range tables {
		tablePieces := strings.Split(v, " ")
		v = b.adapter.Escape(tablePieces[0])
		if len(tablePieces) > 1 {
			v = fmt.Sprintf("%s %s", v, tablePieces[1])
		}
		tbls = append(tbls, v)
	}
	b.query.AddClause(fmt.Sprintf("FROM %s", strings.Join(tbls, ", ")))
	return b
}

// InnerJoin generates "inner join %s on %s" statement for each expression
func (b *Builder) InnerJoin(table string, expressions ...string) *Builder {
	tablePieces := strings.Split(table, " ")

	v := b.adapter.Escape(tablePieces[0])
	if len(tablePieces) > 1 {
		v = fmt.Sprintf("%s %s", v, tablePieces[1])
	}
	b.query.AddClause(fmt.Sprintf("INNER JOIN %s ON %s", v, strings.Join(expressions, " ")))
	return b
}

// CrossJoin generates "cross join %s" statement for table
func (b *Builder) CrossJoin(table string) *Builder {
	tablePieces := strings.Split(table, " ")

	v := b.adapter.Escape(tablePieces[0])
	if len(tablePieces) > 1 {
		v = fmt.Sprintf("%s %s", v, tablePieces[1])
	}

	b.query.AddClause(fmt.Sprintf("CROSS JOIN %s", v))
	return b
}

// LeftOuterJoin generates "left outer join %s on %s" statement for each expression
func (b *Builder) LeftOuterJoin(table string, expressions ...string) *Builder {
	tablePieces := strings.Split(table, " ")

	v := b.adapter.Escape(tablePieces[0])
	if len(tablePieces) > 1 {
		v = fmt.Sprintf("%s %s", v, tablePieces[1])
	}

	b.query.AddClause(fmt.Sprintf("LEFT OUTER JOIN %s ON %s", v, strings.Join(expressions, " ")))
	return b
}

// RightOuterJoin generates "right outer join %s on %s" statement for each expression
func (b *Builder) RightOuterJoin(table string, expressions ...string) *Builder {
	tablePieces := strings.Split(table, " ")

	v := b.adapter.Escape(tablePieces[0])
	if len(tablePieces) > 1 {
		v = fmt.Sprintf("%s %s", v, tablePieces[1])
	}
	b.query.AddClause(fmt.Sprintf("RIGHT OUTER JOIN %s ON %s", v, strings.Join(expressions, " ")))
	return b
}

// FullOuterJoin generates "full outer join %s on %s" for each expression
func (b *Builder) FullOuterJoin(table string, expressions ...string) *Builder {
	tablePieces := strings.Split(table, " ")

	v := b.adapter.Escape(tablePieces[0])
	if len(tablePieces) > 1 {
		v = fmt.Sprintf("%s %s", v, tablePieces[1])
	}
	b.query.AddClause(fmt.Sprintf("FULL OUTER JOIN %s ON %s", v, strings.Join(expressions, " ")))
	return b
}

// Where generates "where %s" for the expression and adds bindings for each value
func (b *Builder) Where(expression string, bindings ...interface{}) *Builder {
	if expression == "" {
		return b
	}
	b.query.AddClause(fmt.Sprintf("WHERE %s", expression))
	b.query.AddBinding(bindings...)
	return b
}

// OrderBy generates "order by %s" for each expression
func (b *Builder) OrderBy(expressions ...string) *Builder {
	b.query.AddClause(fmt.Sprintf("ORDER BY %s", strings.Join(expressions, ", ")))
	return b
}

// GroupBy generates "group by %s" for each column
func (b *Builder) GroupBy(columns ...string) *Builder {
	b.query.AddClause(fmt.Sprintf("GROUP BY %s", strings.Join(columns, ", ")))
	return b
}

// Having generates "having %s" for each expression
func (b *Builder) Having(expressions ...string) *Builder {
	b.query.AddClause(fmt.Sprintf("HAVING %s", strings.Join(expressions, ", ")))
	return b
}

// Limit generates limit %d offset %d for offset and count
func (b *Builder) Limit(offset int, count int) *Builder {
	b.query.AddClause(fmt.Sprintf("LIMIT %d OFFSET %d", count, offset))
	return b
}

// aggregates

// Avg function generates "avg(%s)" statement for column
func (b *Builder) Avg(column string) string {
	return fmt.Sprintf("AVG(%s)", b.adapter.Escape(column))
}

// Count function generates "count(%s)" statement for column
func (b *Builder) Count(column string) string {
	return fmt.Sprintf("COUNT(%s)", b.adapter.Escape(column))
}

// Sum function generates "sum(%s)" statement for column
func (b *Builder) Sum(column string) string {
	return fmt.Sprintf("SUM(%s)", b.adapter.Escape(column))
}

// Min function generates "min(%s)" statement for column
func (b *Builder) Min(column string) string {
	return fmt.Sprintf("MIN(%s)", b.adapter.Escape(column))
}

// Max function generates "max(%s)" statement for column
func (b *Builder) Max(column string) string {
	return fmt.Sprintf("MAX(%s)", b.adapter.Escape(column))
}

// expressions

// NotIn function generates "%s not in (%s)" for key and adds bindings for each value
func (b *Builder) NotIn(key string, values ...interface{}) string {
	b.query.AddBinding(values...)
	return fmt.Sprintf("%s NOT IN (%s)", b.adapter.Escape(key), strings.Join(b.adapter.Placeholders(values...), ","))
}

// In function generates "%s in (%s)" for key and adds bindings for each value
func (b *Builder) In(key string, values ...interface{}) string {
	b.query.AddBinding(values...)
	return fmt.Sprintf("%s IN (%s)", b.adapter.Escape(key), strings.Join(b.adapter.Placeholders(values...), ","))
}

// NotEq function generates "%s != placeholder" for key and adds binding for value
func (b *Builder) NotEq(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s != %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// Eq function generates "%s = placeholder" for key and adds binding for value
func (b *Builder) Eq(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s = %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// Gt function generates "%s > placeholder" for key and adds binding for value
func (b *Builder) Gt(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s > %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// Gte function generates "%s >= placeholder" for key and adds binding for value
func (b *Builder) Gte(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s >= %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// St function generates "%s < placeholder" for key and adds binding for value
func (b *Builder) St(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s < %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// Ste function generates "%s <= placeholder" for key and adds binding for value
func (b *Builder) Ste(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s <= %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// And function generates " AND " between any number of expressions
func (b *Builder) And(expressions ...string) string {
	if len(expressions) == 0 {
		return ""
	} else if len(expressions) == 1 {
		return expressions[0]
	}
	return fmt.Sprintf("(%s)", strings.Join(expressions, " AND "))
}

// Or function generates " OR " between any number of expressions
func (b *Builder) Or(expressions ...string) string {
	return strings.Join(expressions, " OR ")
}

// CreateTable generates generic CREATE TABLE statement
func (b *Builder) CreateTable(table string, fields []string, constraints []string) *Builder {
	b.query.AddClause(fmt.Sprintf("CREATE TABLE %s(", b.adapter.Escape(table)))

	for k, f := range fields {
		clause := fmt.Sprintf("\t%s", f)
		if len(fields)-1 > k || len(constraints) > 0 {
			clause += ","
		}
		b.query.AddClause(clause)
	}

	for k, c := range constraints {
		constraint := fmt.Sprintf("\t%s", c)
		if len(constraints)-1 > k {
			constraint += ","
		}
		b.query.AddClause(fmt.Sprintf("%s", constraint))
	}

	b.query.AddClause(")")
	return b
}

// AlterTable generates generic ALTER TABLE statement
func (b *Builder) AlterTable(table string) *Builder {
	b.query.AddClause(fmt.Sprintf("ALTER TABLE %s", table))
	return b
}

// DropTable generates generic DROP TABLE statement
func (b *Builder) DropTable(table string) *Builder {
	b.query.AddClause(fmt.Sprintf("DROP TABLE %s", b.adapter.Escape(table)))
	return b
}

// Add generates generic ADD COLUMN statement
func (b *Builder) Add(colName string, colType string) *Builder {
	b.query.AddClause(fmt.Sprintf("ADD %s %s", colName, colType))
	return b
}

// Drop generates generic DROP COLUMN statement
func (b *Builder) Drop(colName string) *Builder {
	b.query.AddClause(fmt.Sprintf("DROP %s", colName))
	return b
}

// CreateIndex generates an index on columns
func (b *Builder) CreateIndex(indexName string, tableName string, columns ...string) *Builder {
	b.query.AddClause(fmt.Sprintf("CREATE INDEX %s ON %s (%s)", indexName, tableName, strings.Join(b.adapter.EscapeAll(columns), ",")))
	return b
}
<commit_msg>fix #2, increase test coverage in builder, remove len = 1 expression checking in builder.And()<commit_after>package qb

import (
	"fmt"
	"log"
	"os"
	"strings"
)

const (
	LDefault = iota
	// log query flag
	LQuery
	// log bindings flag
	LBindings
)

// NewBuilder generates a new builder struct
func NewBuilder(driver string) *Builder {
	return &Builder{
		query:    NewQuery(),
		adapter:  NewAdapter(driver),
		logger:   log.New(os.Stdout, "", 0),
		logFlags: LDefault,
	}
}

// Builder is a struct that holds an active query that it is used for building common sql queries
// it has all the common functions except multiple statements & table crudders
type Builder struct {
	query    *Query
	adapter  Adapter
	logger   *log.Logger
	logFlags int
}

// SetLogFlags sets the builder log flags
func (b *Builder) SetLogFlags(logFlags int) {
	b.logFlags = logFlags
}

// LogFlags returns the log flags
func (b *Builder) LogFlags() int {
	return b.logFlags
}

// SetEscaping sets the escaping parameter of current adapter
func (b *Builder) SetEscaping(escaping bool) {
	b.adapter.SetEscaping(escaping)
}

// Escaping functions returns if escaping is available
func (b *Builder) Escaping() bool {
	return b.adapter.Escaping()
}

// Adapter returns the active adapter of builder
func (b *Builder) Adapter() Adapter {
	return b.adapter
}

// Reset clears query bindings and its errors
func (b *Builder) Reset() {
	b.query = NewQuery()
	b.adapter.Reset()
}

// Query returns the active query and resets the query.
// The query clauses and returns the sql and bindings
func (b *Builder) Query() *Query {
	query := b.query
	b.Reset()
	if b.logFlags == LQuery || b.logFlags == (LQuery|LBindings) {
		b.logger.Printf("%s", query.SQL())
	}
	if b.logFlags == LBindings || b.logFlags == (LQuery|LBindings) {
		b.logger.Printf("%s", query.Bindings())
	}
	if b.logFlags != LDefault {
		b.logger.Println()
	}
	return query
}

// Insert generates an "insert into %s(%s)" statement
func (b *Builder) Insert(table string) *Builder {
	clause := fmt.Sprintf("INSERT INTO %s", b.adapter.Escape(table))
	b.query.AddClause(clause)
	return b
}

// Values generates "values(%s)" statement and add bindings for each value
func (b *Builder) Values(m map[string]interface{}) *Builder {
	keys := []string{}
	values := []interface{}{}
	for k, v := range m {
		keys = append(keys, b.adapter.Escape(k))
		values = append(values, v)
		b.query.AddBinding(v)
	}

	b.query.AddClause(fmt.Sprintf("(%s)", strings.Join(keys, ", ")))

	placeholders := []string{}

	for _ = range values {
		placeholders = append(placeholders, b.adapter.Placeholder())
	}
	clause := fmt.Sprintf("VALUES (%s)", strings.Join(placeholders, ", "))
	b.query.AddClause(clause)
	return b
}

// Returning generates RETURNING statement for postgres only
// NOTE: Do not use it with sqlite, mysql or other drivers
func (b *Builder) Returning(cols ...string) *Builder {
	cols = b.adapter.EscapeAll(cols)
	clause := fmt.Sprintf("RETURNING %s", strings.Join(cols, ", "))
	b.query.AddClause(clause)
	return b
}

// Update generates "update %s" statement
func (b *Builder) Update(table string) *Builder {
	clause := fmt.Sprintf("UPDATE %s", b.adapter.Escape(table))
	b.query.AddClause(clause)
	return b
}

// Set generates "set a = placeholder" statement for each key a and add bindings for map value
func (b *Builder) Set(m map[string]interface{}) *Builder {
	updates := []string{}
	for k, v := range m {
		// check if aliasing exists
		if strings.Contains(k, ".") {
			kpieces := strings.Split(k, ".")
			k = fmt.Sprintf("%s.%s", kpieces[0], b.adapter.Escape(kpieces[1]))
		} else {
			k = b.adapter.Escape(k)
		}
		updates = append(updates, fmt.Sprintf("%s = %s", k, b.adapter.Placeholder()))
		b.query.AddBinding(v)
	}
	clause := fmt.Sprintf("SET %s", strings.Join(updates, ", "))
	b.query.AddClause(clause)
	return b
}

// Delete generates "delete" statement
func (b *Builder) Delete(table string) *Builder {
	b.query.AddClause(fmt.Sprintf("DELETE FROM %s", b.adapter.Escape(table)))
	return b
}

// Select generates "select %s" statement
func (b *Builder) Select(columns ...string) *Builder {
	clause := fmt.Sprintf("SELECT %s", strings.Join(columns, ", "))
	b.query.AddClause(clause)
	return b
}

// From generates "from %s" statement for each table name
func (b *Builder) From(tables ...string) *Builder {
	tbls := []string{}
	for _, v := range tables {
		tablePieces := strings.Split(v, " ")
		v = b.adapter.Escape(tablePieces[0])
		if len(tablePieces) > 1 {
			v = fmt.Sprintf("%s %s", v, tablePieces[1])
		}
		tbls = append(tbls, v)
	}
	b.query.AddClause(fmt.Sprintf("FROM %s", strings.Join(tbls, ", ")))
	return b
}

// InnerJoin generates "inner join %s on %s" statement for each expression
func (b *Builder) InnerJoin(table string, expressions ...string) *Builder {
	tablePieces := strings.Split(table, " ")

	v := b.adapter.Escape(tablePieces[0])
	if len(tablePieces) > 1 {
		v = fmt.Sprintf("%s %s", v, tablePieces[1])
	}
	b.query.AddClause(fmt.Sprintf("INNER JOIN %s ON %s", v, strings.Join(expressions, " ")))
	return b
}

// CrossJoin generates "cross join %s" statement for table
func (b *Builder) CrossJoin(table string) *Builder {
	tablePieces := strings.Split(table, " ")

	v := b.adapter.Escape(tablePieces[0])
	if len(tablePieces) > 1 {
		v = fmt.Sprintf("%s %s", v, tablePieces[1])
	}

	b.query.AddClause(fmt.Sprintf("CROSS JOIN %s", v))
	return b
}

// LeftOuterJoin generates "left outer join %s on %s" statement for each expression
func (b *Builder) LeftOuterJoin(table string, expressions ...string) *Builder {
	tablePieces := strings.Split(table, " ")

	v := b.adapter.Escape(tablePieces[0])
	if len(tablePieces) > 1 {
		v = fmt.Sprintf("%s %s", v, tablePieces[1])
	}

	b.query.AddClause(fmt.Sprintf("LEFT OUTER JOIN %s ON %s", v, strings.Join(expressions, " ")))
	return b
}

// RightOuterJoin generates "right outer join %s on %s" statement for each expression
func (b *Builder) RightOuterJoin(table string, expressions ...string) *Builder {
	tablePieces := strings.Split(table, " ")

	v := b.adapter.Escape(tablePieces[0])
	if len(tablePieces) > 1 {
		v = fmt.Sprintf("%s %s", v, tablePieces[1])
	}
	b.query.AddClause(fmt.Sprintf("RIGHT OUTER JOIN %s ON %s", v, strings.Join(expressions, " ")))
	return b
}

// FullOuterJoin generates "full outer join %s on %s" for each expression
func (b *Builder) FullOuterJoin(table string, expressions ...string) *Builder {
	tablePieces := strings.Split(table, " ")

	v := b.adapter.Escape(tablePieces[0])
	if len(tablePieces) > 1 {
		v = fmt.Sprintf("%s %s", v, tablePieces[1])
	}
	b.query.AddClause(fmt.Sprintf("FULL OUTER JOIN %s ON %s", v, strings.Join(expressions, " ")))
	return b
}

// Where generates "where %s" for the expression and adds bindings for each value
func (b *Builder) Where(expression string, bindings ...interface{}) *Builder {
	if expression == "" {
		return b
	}
	b.query.AddClause(fmt.Sprintf("WHERE %s", expression))
	b.query.AddBinding(bindings...)
	return b
}

// OrderBy generates "order by %s" for each expression
func (b *Builder) OrderBy(expressions ...string) *Builder {
	b.query.AddClause(fmt.Sprintf("ORDER BY %s", strings.Join(expressions, ", ")))
	return b
}

// GroupBy generates "group by %s" for each column
func (b *Builder) GroupBy(columns ...string) *Builder {
	b.query.AddClause(fmt.Sprintf("GROUP BY %s", strings.Join(columns, ", ")))
	return b
}

// Having generates "having %s" for each expression
func (b *Builder) Having(expressions ...string) *Builder {
	b.query.AddClause(fmt.Sprintf("HAVING %s", strings.Join(expressions, ", ")))
	return b
}

// Limit generates limit %d offset %d for offset and count
func (b *Builder) Limit(offset int, count int) *Builder {
	b.query.AddClause(fmt.Sprintf("LIMIT %d OFFSET %d", count, offset))
	return b
}

// aggregates

// Avg function generates "avg(%s)" statement for column
func (b *Builder) Avg(column string) string {
	return fmt.Sprintf("AVG(%s)", b.adapter.Escape(column))
}

// Count function generates "count(%s)" statement for column
func (b *Builder) Count(column string) string {
	return fmt.Sprintf("COUNT(%s)", b.adapter.Escape(column))
}

// Sum function generates "sum(%s)" statement for column
func (b *Builder) Sum(column string) string {
	return fmt.Sprintf("SUM(%s)", b.adapter.Escape(column))
}

// Min function generates "min(%s)" statement for column
func (b *Builder) Min(column string) string {
	return fmt.Sprintf("MIN(%s)", b.adapter.Escape(column))
}

// Max function generates "max(%s)" statement for column
func (b *Builder) Max(column string) string {
	return fmt.Sprintf("MAX(%s)", b.adapter.Escape(column))
}

// expressions

// NotIn function generates "%s not in (%s)" for key and adds bindings for each value
func (b *Builder) NotIn(key string, values ...interface{}) string {
	b.query.AddBinding(values...)
	return fmt.Sprintf("%s NOT IN (%s)", b.adapter.Escape(key), strings.Join(b.adapter.Placeholders(values...), ","))
}

// In function generates "%s in (%s)" for key and adds bindings for each value
func (b *Builder) In(key string, values ...interface{}) string {
	b.query.AddBinding(values...)
	return fmt.Sprintf("%s IN (%s)", b.adapter.Escape(key), strings.Join(b.adapter.Placeholders(values...), ","))
}

// NotEq function generates "%s != placeholder" for key and adds binding for value
func (b *Builder) NotEq(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s != %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// Eq function generates "%s = placeholder" for key and adds binding for value
func (b *Builder) Eq(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s = %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// Gt function generates "%s > placeholder" for key and adds binding for value
func (b *Builder) Gt(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s > %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// Gte function generates "%s >= placeholder" for key and adds binding for value
func (b *Builder) Gte(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s >= %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// St function generates "%s < placeholder" for key and adds binding for value
func (b *Builder) St(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s < %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// Ste function generates "%s <= placeholder" for key and adds binding for value
func (b *Builder) Ste(key string, value interface{}) string {
	b.query.AddBinding(value)
	return fmt.Sprintf("%s <= %s", b.adapter.Escape(key), b.adapter.Placeholder())
}

// And function generates " AND " between any number of expressions
func (b *Builder) And(expressions ...string) string {
	if len(expressions) == 0 {
		return ""
	}
	return fmt.Sprintf("(%s)", strings.Join(expressions, " AND "))
}

// Or function generates " OR " between any number of expressions
func (b *Builder) Or(expressions ...string) string {
	return strings.Join(expressions, " OR ")
}

// CreateTable generates generic CREATE TABLE statement
func (b *Builder) CreateTable(table string, fields []string, constraints []string) *Builder {
	b.query.AddClause(fmt.Sprintf("CREATE TABLE %s(", b.adapter.Escape(table)))

	for k, f := range fields {
		clause := fmt.Sprintf("\t%s", f)
		if len(fields)-1 > k || len(constraints) > 0 {
			clause += ","
		}
		b.query.AddClause(clause)
	}

	for k, c := range constraints {
		constraint := fmt.Sprintf("\t%s", c)
		if len(constraints)-1 > k {
			constraint += ","
		}
		b.query.AddClause(fmt.Sprintf("%s", constraint))
	}

	b.query.AddClause(")")
	return b
}

// AlterTable generates generic ALTER TABLE statement
func (b *Builder) AlterTable(table string) *Builder {
	b.query.AddClause(fmt.Sprintf("ALTER TABLE %s", table))
	return b
}

// DropTable generates generic DROP TABLE statement
func (b *Builder) DropTable(table string) *Builder {
	b.query.AddClause(fmt.Sprintf("DROP TABLE %s", b.adapter.Escape(table)))
	return b
}

// Add generates generic ADD COLUMN statement
func (b *Builder) Add(colName string, colType string) *Builder {
	b.query.AddClause(fmt.Sprintf("ADD %s %s", colName, colType))
	return b
}

// Drop generates generic DROP COLUMN statement
func (b *Builder) Drop(colName string) *Builder {
	b.query.AddClause(fmt.Sprintf("DROP %s", colName))
	return b
}

// CreateIndex generates an index on columns
func (b *Builder) CreateIndex(indexName string, tableName string, columns ...string) *Builder {
	b.query.AddClause(fmt.Sprintf("CREATE INDEX %s ON %s(%s)", indexName, tableName, strings.Join(b.adapter.EscapeAll(columns), ",")))
	return b
}
<|endoftext|>


<commit_before>package ca

// Country: SE
// Organization: NBusy
// Organizational Unit: NBusy Certificate authority
// Common Name: NBusy Root CA
//
// func TestGenCert(t *testing.T) {
// 	// keyLength := 0 // used for internal test cert generation
// 	keyLength := 512
//
// 	caCert, caKey, clientCert, clientKey, err := genTestCertPair(keyLength)
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	if keyLength == 0 {
// 		fmt.Println("CA cert:")
// 		fmt.Println(string(caCert))
// 		fmt.Println(string(caKey))
// 		fmt.Println("Client cert:")
// 		fmt.Println(string(clientCert))
// 		fmt.Println(string(clientKey))
// 	}
// }
//
// func genTestCertPair(keyLength int) (caCert, caKey, clientCert, clientKey []byte, err error) {
// 	// CA certificate
// 	caCert, caKey, err = genCert("127.0.0.1", 0, nil, nil, keyLength, "127.0.0.1", "devastator")
//
// 	if err != nil {
// 		err = fmt.Errorf("Failed to generate CA certificate or key: %v", err)
// 		return
// 	}
// 	if caCert == nil || caKey == nil {
// 		err = fmt.Errorf("Generated empty CA certificate or key")
// 		return
// 	}
//
// 	tlsCert, err := tls.X509KeyPair(caCert, caKey)
//
// 	if err != nil {
// 		err = fmt.Errorf("Generated invalid CA certificate or key: %v", err)
// 		return
// 	}
// 	if &tlsCert == nil {
// 		err = fmt.Errorf("Generated invalid CA certificate or key")
// 		return
// 	}
//
// 	// client certificate
// 	pub, err := x509.ParseCertificate(tlsCert.Certificate[0])
// 	if err != nil {
// 		err = fmt.Errorf("Failed to parse x509 certificate of CA cert to sign client-cert: %v", err)
// 		return
// 	}
//
// 	clientCert, clientKey, err = genCert("client.127.0.0.1", 0, pub, tlsCert.PrivateKey.(*rsa.PrivateKey), keyLength, "client.127.0.0.1", "devastator")
// 	if err != nil {
// 		err = fmt.Errorf("Failed to generate client-certificate or key: %v", err)
// 		return
// 	}
//
// 	tlsCert2, err := tls.X509KeyPair(clientCert, clientKey)
//
// 	if err != nil {
// 		err = fmt.Errorf("Generated invalid client-certificate or key: %v", err)
// 		return
// 	}
// 	if &tlsCert2 == nil {
// 		err = fmt.Errorf("Generated invalid client-certificate or key")
// 		return
// 	}
//
// 	return
// }
<commit_msg>add chain test<commit_after>package ca

import (
	"crypto/x509/pkix"
	"testing"
	"time"
)

func TestGenCertChain(t *testing.T) {
	caCert, caKey, err := GenCA(pkix.Name{
		Country:            []string{"SE"},
		Organization:       []string{"FooBar"},
		OrganizationalUnit: []string{"FooBar Certificate Authority"},
		CommonName:         "FooBar Root CA",
	}, time.Hour, 512)

	if caCert == nil || caKey == nil || err != nil {
		t.Fatal(err)
	}
}

// func TestGenCert(t *testing.T) {
// 	// keyLength := 0 // used for internal test cert generation
// 	keyLength := 512
//
// 	caCert, caKey, clientCert, clientKey, err := genTestCertPair(keyLength)
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	if keyLength == 0 {
// 		fmt.Println("CA cert:")
// 		fmt.Println(string(caCert))
// 		fmt.Println(string(caKey))
// 		fmt.Println("Client cert:")
// 		fmt.Println(string(clientCert))
// 		fmt.Println(string(clientKey))
// 	}
// }
//
// func genTestCertPair(keyLength int) (caCert, caKey, clientCert, clientKey []byte, err error) {
// 	// CA certificate
// 	caCert, caKey, err = genCert("127.0.0.1", 0, nil, nil, keyLength, "127.0.0.1", "devastator")
//
// 	if err != nil {
// 		err = fmt.Errorf("Failed to generate CA certificate or key: %v", err)
// 		return
// 	}
// 	if caCert == nil || caKey == nil {
// 		err = fmt.Errorf("Generated empty CA certificate or key")
// 		return
// 	}
//
// 	tlsCert, err := tls.X509KeyPair(caCert, caKey)
//
// 	if err != nil {
// 		err = fmt.Errorf("Generated invalid CA certificate or key: %v", err)
// 		return
// 	}
// 	if &tlsCert == nil {
// 		err = fmt.Errorf("Generated invalid CA certificate or key")
// 		return
// 	}
//
// 	// client certificate
// 	pub, err := x509.ParseCertificate(tlsCert.Certificate[0])
// 	if err != nil {
// 		err = fmt.Errorf("Failed to parse x509 certificate of CA cert to sign client-cert: %v", err)
// 		return
// 	}
//
// 	clientCert, clientKey, err = genCert("client.127.0.0.1", 0, pub, tlsCert.PrivateKey.(*rsa.PrivateKey), keyLength, "client.127.0.0.1", "devastator")
// 	if err != nil {
// 		err = fmt.Errorf("Failed to generate client-certificate or key: %v", err)
// 		return
// 	}
//
// 	tlsCert2, err := tls.X509KeyPair(clientCert, clientKey)
//
// 	if err != nil {
// 		err = fmt.Errorf("Generated invalid client-certificate or key: %v", err)
// 		return
// 	}
// 	if &tlsCert2 == nil {
// 		err = fmt.Errorf("Generated invalid client-certificate or key")
// 		return
// 	}
//
// 	return
// }
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/gorilla/mux"
	"github.com/vmihailenco/redis/v2"
)

var client *redis.Client

type Measurement struct {
	Id                string  `json:"id"`
	CheckId           string  `json:"check_id"`
	Location          string  `json:"location"`
	Url               string  `json:"url"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIp           string  `json:"local_ip,omitempty"`
	PrimaryIp         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HttpStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
}

func (m *Measurement) Record() {
	s, _ := json.Marshal(m)
	z := redis.Z{Score: float64(m.T), Member: string(s)}
	err := client.ZAdd(GetRedisKey(m.CheckId), z)
	if err != nil {
		log.Fatal("Error while record measuremnt %s: %v\n", m.Id, err)
	}
}

func TrimMeasurements(check_id string, seconds int64) {
	now := time.Now()
	epoch := now.Unix() - seconds
	err := client.ZRemRangeByScore(GetRedisKey(check_id), "-inf", strconv.FormatInt(epoch, 10))
	if err != nil {
		log.Fatal("Error while trimming check_id %s: %v\n", check_id, err)
	}
}

func GetRedisKey(check_id string) string {
	return "measurements:" + check_id
}

func GetLatestMeasurements(check_id string) []Measurement {
	now := time.Now()
	epoch := now.Unix() - 60

	vals, err := client.ZRevRangeByScore(GetRedisKey(check_id), redis.ZRangeByScore{
		Min: strconv.FormatInt(epoch, 10),
		Max: "+inf",
	}).Result()

	if err != nil {
		panic(err)
	}

	measurements := make([]Measurement, 0, 100)

	for _, v := range vals {
		var m Measurement
		json.Unmarshal([]byte(v), &m)
		measurements = append(measurements, m)
	}

	return measurements
}

func GetenvWithDefault(key string, def string) string {
	try := os.Getenv(key)

	if try == "" {
		return def
	}

	return try
}

func RedirectToChecksHandler(res http.ResponseWriter, req *http.Request) {
	checks_url := GetenvWithDefault("CHECKS_URL", "https://s3.amazonaws.com/canary-public-data/data.json")
	http.Redirect(res, req, checks_url, http.StatusFound)
}

func GetMeasurementsHandler(res http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	check_id := vars["check_id"]

	s, _ := json.MarshalIndent(GetLatestMeasurements(check_id), "", "  ")

	fmt.Fprintf(res, string(s))
}

func PostMeasurementsHandler(res http.ResponseWriter, req *http.Request) {
	decoder := json.NewDecoder(req.Body)
	measurements := make([]Measurement, 0, 100)

	err := decoder.Decode(&measurements)
	if err != nil {
		panic(err)
	}

	for _, m := range measurements {
		m.Record()
		TrimMeasurements(m.CheckId, 60*60)
	}

	log.Printf("fn=post_measurements count=%d\n", len(measurements))
}

func ConnectToRedis() {
	u, err := url.Parse(GetenvWithDefault("REDIS_URL", "redis://localhost:6379"))
	if err != nil {
		panic(err)
	}

	client = redis.NewTCPClient(&redis.Options{
		Addr:     u.Host,
		Password: "", // no password set
		DB:       0,  // use default DB
	})
}

func main() {
	ConnectToRedis()

	r := mux.NewRouter()

	r.HandleFunc("/checks", RedirectToChecksHandler).Methods("GET")
	r.HandleFunc("/checks/{check_id}/measurements", GetMeasurementsHandler).Methods("GET")
	r.HandleFunc("/measurements", PostMeasurementsHandler).Methods("POST")
	http.Handle("/", r)

	port := GetenvWithDefault("PORT", "5000")
	log.Printf("fn=main listening=true port=%s\n", port)

	err := http.ListenAndServe(":"+port, nil)
	if err != nil {
		panic(err)
	}
}
<commit_msg>proper render the error<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/gorilla/mux"
	"github.com/vmihailenco/redis/v2"
)

var client *redis.Client

type Measurement struct {
	Id                string  `json:"id"`
	CheckId           string  `json:"check_id"`
	Location          string  `json:"location"`
	Url               string  `json:"url"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIp           string  `json:"local_ip,omitempty"`
	PrimaryIp         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HttpStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
}

func (m *Measurement) Record() {
	s, _ := json.Marshal(m)
	z := redis.Z{Score: float64(m.T), Member: string(s)}
	err := client.ZAdd(GetRedisKey(m.CheckId), z)
	if err != nil {
		log.Fatalf("Error while recording measuremnt %s: %v\n", m.Id, err)
	}
}

func TrimMeasurements(check_id string, seconds int64) {
	now := time.Now()
	epoch := now.Unix() - seconds
	err := client.ZRemRangeByScore(GetRedisKey(check_id), "-inf", strconv.FormatInt(epoch, 10))
	if err != nil {
		log.Fatalf("Error while trimming check_id %s: %v\n", check_id, err)
	}
}

func GetRedisKey(check_id string) string {
	return "measurements:" + check_id
}

func GetLatestMeasurements(check_id string) []Measurement {
	now := time.Now()
	epoch := now.Unix() - 60

	vals, err := client.ZRevRangeByScore(GetRedisKey(check_id), redis.ZRangeByScore{
		Min: strconv.FormatInt(epoch, 10),
		Max: "+inf",
	}).Result()

	if err != nil {
		panic(err)
	}

	measurements := make([]Measurement, 0, 100)

	for _, v := range vals {
		var m Measurement
		json.Unmarshal([]byte(v), &m)
		measurements = append(measurements, m)
	}

	return measurements
}

func GetenvWithDefault(key string, def string) string {
	try := os.Getenv(key)

	if try == "" {
		return def
	}

	return try
}

func RedirectToChecksHandler(res http.ResponseWriter, req *http.Request) {
	checks_url := GetenvWithDefault("CHECKS_URL", "https://s3.amazonaws.com/canary-public-data/data.json")
	http.Redirect(res, req, checks_url, http.StatusFound)
}

func GetMeasurementsHandler(res http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	check_id := vars["check_id"]

	s, _ := json.MarshalIndent(GetLatestMeasurements(check_id), "", "  ")

	fmt.Fprintf(res, string(s))
}

func PostMeasurementsHandler(res http.ResponseWriter, req *http.Request) {
	decoder := json.NewDecoder(req.Body)
	measurements := make([]Measurement, 0, 100)

	err := decoder.Decode(&measurements)
	if err != nil {
		panic(err)
	}

	for _, m := range measurements {
		m.Record()
		TrimMeasurements(m.CheckId, 60*60)
	}

	log.Printf("fn=post_measurements count=%d\n", len(measurements))
}

func ConnectToRedis() {
	u, err := url.Parse(GetenvWithDefault("REDIS_URL", "redis://localhost:6379"))
	if err != nil {
		panic(err)
	}

	client = redis.NewTCPClient(&redis.Options{
		Addr:     u.Host,
		Password: "", // no password set
		DB:       0,  // use default DB
	})
}

func main() {
	ConnectToRedis()

	r := mux.NewRouter()

	r.HandleFunc("/checks", RedirectToChecksHandler).Methods("GET")
	r.HandleFunc("/checks/{check_id}/measurements", GetMeasurementsHandler).Methods("GET")
	r.HandleFunc("/measurements", PostMeasurementsHandler).Methods("POST")
	http.Handle("/", r)

	port := GetenvWithDefault("PORT", "5000")
	log.Printf("fn=main listening=true port=%s\n", port)

	err := http.ListenAndServe(":"+port, nil)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
)

var port = flag.String("p", "8888", "Port to listen on")
var storage = flag.String("s", "/var/lib/shelf", "Path to store files")

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage:	%v -p <port> -s <storage-path>\n\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func errorResponse(w http.ResponseWriter, e error) {
	w.WriteHeader(http.StatusInternalServerError)
	w.Write([]byte(e.Error()))
	log.Println("error:", e.Error())
}

func main() {
	flag.Parse()

	var storagepath string
	if flag.NArg() == 1 {
		// deprecated: passing storage path as argument
		storagepath = flag.Arg(0)
	} else {
		storagepath = *storage
	}
	os.MkdirAll(storagepath, 0755)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		filepath := storagepath + r.RequestURI
		switch r.Method {
		case "GET":
			file, err := os.Open(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			defer file.Close()
			_, err = io.Copy(w, file)
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("GET", r.RequestURI)
		case "PUT":
			os.MkdirAll(path.Dir(filepath), 0755)
			file, err := os.Create(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			defer file.Close()
			_, err = io.Copy(file, r.Body)
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("PUT", r.RequestURI)
		case "DELETE":
			err := os.RemoveAll(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("DELETE", r.RequestURI)
		default:
			w.WriteHeader(http.StatusMethodNotAllowed)
		}
	})
	log.Println("Shelf serving files on " + *port + " from " + storagepath)
	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<commit_msg>blobstore: Use http.ServeContent to serve files<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
)

var port = flag.String("p", "8888", "Port to listen on")
var storage = flag.String("s", "/var/lib/shelf", "Path to store files")

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage:	%v -p <port> -s <storage-path>\n\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func errorResponse(w http.ResponseWriter, e error) {
	w.WriteHeader(http.StatusInternalServerError)
	w.Write([]byte(e.Error()))
	log.Println("error:", e.Error())
}

func main() {
	flag.Parse()

	var storagepath string
	if flag.NArg() == 1 {
		// deprecated: passing storage path as argument
		storagepath = flag.Arg(0)
	} else {
		storagepath = *storage
	}
	os.MkdirAll(storagepath, 0755)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		filepath := storagepath + r.RequestURI
		switch r.Method {
		case "GET":
			file, err := os.Open(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			defer file.Close()
			fi, err := file.Stat()
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("GET", r.RequestURI)
			http.ServeContent(w, r, filepath, fi.ModTime(), file)
		case "PUT":
			os.MkdirAll(path.Dir(filepath), 0755)
			file, err := os.Create(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			defer file.Close()
			_, err = io.Copy(file, r.Body)
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("PUT", r.RequestURI)
		case "DELETE":
			err := os.RemoveAll(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("DELETE", r.RequestURI)
		default:
			w.WriteHeader(http.StatusMethodNotAllowed)
		}
	})
	log.Println("Shelf serving files on " + *port + " from " + storagepath)
	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/evolsnow/robot/conn"
	"github.com/go-telegram-bot-api/telegram-bot-api"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"sync"
	"time"
)

//zmz.tv needs to login before downloading
var zmzClient http.Client

func (rb *Robot) Start(update tgbotapi.Update) string {
	user := update.Message.Chat.UserName
	go conn.SetUserChatId(user, update.Message.Chat.ID)
	return "welcome: " + user
}

func (rb *Robot) Help(update tgbotapi.Update) string {
	helpMsg := `
/alarm - set a reminder
/evolve	- self evolution of me
/movie - find movie download links
/show - find American show download links
/trans - translate words between english and chinese
/help - show this help message
`
	return helpMsg
}

func (rb *Robot) Evolve(update tgbotapi.Update) {
	if update.Message.Chat.FirstName != "Evol" || update.Message.Chat.LastName != "Gan" {
		rb.bot.Send(tgbotapi.NewMessage(update.Message.Chat.ID, "sorry, unauthorized"))
		return
	}
	<-saidGoodBye
	close(saidGoodBye)
	cmd := exec.Command("bash", "/root/evolve_"+rb.nickName)
	cmd.Start()
	os.Exit(1)
}

func (rb *Robot) Translate(update tgbotapi.Update) string {
	var info string
	if string(update.Message.Text[0]) == "/" {
		raw := strings.SplitAfterN(update.Message.Text, " ", 2)
		if len(raw) < 2 {
			return "what do you want to translate, try '/trans cat'?"
		} else {
			info = "翻译" + raw[1]
		}
	} else {
		info = update.Message.Text
	}

	return qinAI(info)

}
func (rb *Robot) Talk(update tgbotapi.Update) string {
	info := update.Message.Text
	if strings.Contains(info, rb.name) {
		if strings.Contains(info, "闭嘴") || strings.Contains(info, "别说话") {
			rb.shutUp = true
		} else if rb.shutUp && strings.Contains(info, "说话") {
			rb.shutUp = false
			return fmt.Sprintf("%s终于可以说话啦", rb.nickName)
		}
		info = strings.Replace(info, fmt.Sprintf("@%s", rb.name), "", -1)
	}

	if rb.shutUp {
		return ""
	}
	log.Printf(info)

	if rb.nickName == "samaritan" {
		if chinese(info) {
			return tlAI(info)
		} else {
			return mitAI(info)
		}
	} else { //jarvis use another AI
		return qinAI(info)
	}
	//	return response
}

func (rb *Robot) SetReminder(update tgbotapi.Update, step int) string {
	user := update.Message.Chat.UserName
	switch step {
	case 0:
		//known issue of go, you can not just assign update.Message.Chat.ID to userTask[user].ChatId
		tmpTask := userTask[user]
		tmpTask.ChatId = update.Message.Chat.ID
		tmpTask.Owner = update.Message.Chat.UserName
		userTask[user] = tmpTask

		tmpAction := userAction[user]
		tmpAction.ActionStep++
		userAction[user] = tmpAction
		return "Ok, what should I remind you to do?"
	case 1:
		//save thing
		tmpTask := userTask[user]
		tmpTask.Desc = update.Message.Text
		userTask[user] = tmpTask

		tmpAction := userAction[user]
		tmpAction.ActionStep++
		userAction[user] = tmpAction
		return "When or how much time after?\n" +
			"You can type:\n" +
			"'*2/14 11:30*' means 11:30 at 2/14 \n" + //first format
			"'*11:30*' means  11:30 today\n" + //second format
			"'*5m10s*' means 5 minutes 10 seconds later" //third format
	case 2:
		defer delete(userAction, user)
		//save time duration
		text := update.Message.Text
		text = strings.Replace(text, "：", ":", -1)
		firstFormat := "1/02 15:04"
		secondFormat := "15:04"
		thirdFormat := "15:04:05"
		var showTime string
		var scheduledTime time.Time
		var nowTime = time.Now()
		var du time.Duration
		var err error
		if strings.Contains(text, ":") {
			scheduledTime, err = time.Parse(firstFormat, text)
			nowTime, _ = time.Parse(firstFormat, nowTime.Format(firstFormat))
			showTime = scheduledTime.Format(firstFormat)
			if err != nil { //try to parse with first format
				scheduledTime, err = time.Parse(secondFormat, text)
				nowTime, _ = time.Parse(secondFormat, nowTime.Format(secondFormat))
				showTime = scheduledTime.Format(secondFormat)
				if err != nil {
					return "wrong format, try '2/14 11:30' or '11:30'?"
				}
			}
			du = scheduledTime.Sub(nowTime)
		} else {

			du, err = time.ParseDuration(text)
			scheduledTime = nowTime.Add(du)
			showTime = scheduledTime.Format(thirdFormat)
			if err != nil {
				return "wrong format, try '1h2m3s'?"
			}
		}
		//		tmpTask := userTask[user]
		//		tmpTask.When = scheduledTime
		//		userTask[user] = tmpTask
		go func(rb *Robot, ts Task) {
			timer := time.NewTimer(du)
			rawMsg := fmt.Sprintf("Hi %s, maybe it's time to:\n*%s*", ts.Owner, ts.Desc)
			msg := tgbotapi.NewMessage(ts.ChatId, rawMsg)
			msg.ParseMode = "markdown"
			<-timer.C
			_, err := rb.bot.Send(msg)
			if err != nil {
				rb.bot.Send(tgbotapi.NewMessage(conn.GetUserChatId(ts.Owner), rawMsg))
			}
			delete(userTask, user)
		}(rb, userTask[user])

		//		delete(userAction, user)
		return fmt.Sprintf("Ok, I will remind you that\n*%s* - *%s*", showTime, userTask[user].Desc)
	}
	return ""
}

func (rb *Robot) DownloadMovie(update tgbotapi.Update, step int, results chan string) (ret string) {
	user := update.Message.Chat.UserName
	switch step {
	case 0:
		//known issue of go, you can not just assign update.Message.Chat.ID to userTask[user].ChatId
		tmpAction := userAction[user]
		tmpAction.ActionStep++
		userAction[user] = tmpAction
		ret = "Ok, which movie do you want to download?"
	case 1:
		defer func() {
			delete(userAction, user)
			results <- "done"
		}()
		results <- "Searching movie..."
		movie := update.Message.Text
		var wg sync.WaitGroup
		wg.Add(2)
		go getMovieFromZMZ(movie, results, &wg)
		go getMovieFromLBL(movie, results, &wg)
		wg.Wait()
	}
	return
}

func (rb *Robot) DownloadShow(update tgbotapi.Update, step int, results chan string) (ret string) {
	user := update.Message.Chat.UserName
	switch step {
	case 0:
		//known issue of go, you can not just assign update.Message.Chat.ID to userTask[user].ChatId
		tmpAction := userAction[user]
		tmpAction.ActionStep++
		userAction[user] = tmpAction
		ret = "Ok, which American show do you want to download?"
	case 1:
		defer func() {
			delete(userAction, user)
			results <- "done"
		}()
		results <- "Searching American show..."
		info := strings.Split(update.Message.Text, " ")
		GetShowFromZMZ(info[0], info[1], info[2], results)
	}
	return
}

func getMovieFromLBL(movie string, results chan string, wg *sync.WaitGroup) {
	defer wg.Done()
	var id string
	resp, _ := http.Get("http://www.lbldy.com/search/" + movie)
	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	re, _ := regexp.Compile("
")
	//find first match case
	firstId := re.FindSubmatch(body)
	if len(firstId) == 0 {
		results <- fmt.Sprintf("No results for *%s* from LBL", movie)
		return
	} else {
		id = string(firstId[1])
		resp, _ = http.Get("http://www.lbldy.com/movie/" + id + ".html")
		defer resp.Body.Close()
		re, _ = regexp.Compile(`<p><a href="(.*?)"( target="_blank">|>)(.*?)</a></p>`)
		body, _ := ioutil.ReadAll(resp.Body)
		//go does not support (?!) regex
		body = []byte(strings.Replace(string(body), `<a href="/xunlei/"`, "", -1))
		downloads := re.FindAllSubmatch(body, -1)
		if len(downloads) == 0 {
			results <- fmt.Sprintf("No results for *%s* from LBL", movie)
			return
		} else {
			ret := "Results from LBL:\n\n"
			for i := range downloads {
				ret += fmt.Sprintf("*%s*\n```%s```\n\n", string(downloads[i][3]), string(downloads[i][1]))
				if i%5 == 0 && i > 0 {
					results <- ret
					ret = fmt.Sprintf("*LBL Part %d*\n\n", i/5+1)
				}
			}
			results <- ret
		}
	}
}

func getMovieFromZMZ(movie string, results chan string, wg *sync.WaitGroup) {
	defer wg.Done()
	loginZMZ()
	if downloads := getZMZResource(movie); downloads == nil {
		results <- fmt.Sprintf("No results for *%s* from ZMZ", movie)
		return
	} else {
		ret := "Results from ZMZ:\n\n"
		for i := range downloads {
			name := string(downloads[i][1])
			size := string(downloads[i][2])
			link := string(downloads[i][3])
			ret += fmt.Sprintf("*%s*(%s)\n```%s```\n\n", name, size, link)
			if i%5 == 0 && i > 0 {
				results <- ret
				ret = fmt.Sprintf("*ZMZ Part %d*\n\n", i/5+1)
			}
		}
		results <- ret
	}
	return
}

func GetShowFromZMZ(show, s, e string, results chan string) {
	loginZMZ()
	downloads := getZMZResource(show)
	if downloads == nil {
		results <- fmt.Sprintf("No results for *%s* from ZMZ", show)
		return
	}
	//second parse
	re, _ := regexp.Compile(fmt.Sprintf(".*?season=\"%s\" episode=\"%s\">.*?", s, e))
	results <- "Results from ZMZ:\n\n"
	count := 0
	for i := range downloads {
		if re.Find(downloads[i][0]) != nil {
			name := string(downloads[i][1])
			size := string(downloads[i][2])
			link := string(downloads[i][3])
			results <- fmt.Sprintf("*ZMZ %s*(%s)\n```%s```\n\n", name, size, link)
			count++
		}
		if count == 0 {
			results <- fmt.Sprintf("No results found for *S%sE%s*", s, e)

		}
	}
	return
}

func getZMZResource(name string) [][][]byte {
	id := getZMZResourceId(name)
	if id == "" {
		return nil
	}
	resourceURL := "http://www.zimuzu.tv/resource/list/" + id
	resp, _ := zmzClient.Get(resourceURL)
	defer resp.Body.Close()
	//1.name 2.size 3.link
	re, _ := regexp.Compile(`<li class="clearfix".*?<input type="checkbox"><a title="(.*?)".*?<font class="f3">(.*?)</font>.*?<a href="(.*?)" type="ed2k">`)
	body, _ := ioutil.ReadAll(resp.Body)
	body = []byte(strings.Replace(string(body), "\n", "", -1))
	//	tmp := (strings.Replace(string(body), "
\n", "", -1))
	//	body = []byte(strings.Replace(tmp, "
\n", "", -1))
	downloads := re.FindAllSubmatch(body, -1)
	if len(downloads) == 0 {
		return nil
	}
	return downloads
}

func getZMZResourceId(name string) (id string) {
	queryURL := fmt.Sprintf("http://www.zimuzu.tv/search?keyword=%s&type=resource", name)
	re, _ := regexp.Compile(`<div class="t f14"><a href="/resource/(.*?)"><strong class="list_title">`)
	resp, _ := zmzClient.Get(queryURL)
	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	//find first match case
	firstId := re.FindSubmatch(body)
	if len(firstId) == 0 {
		return
	} else {
		log.Println(id)
		id = string(firstId[1])
		return
	}
}

func loginZMZ() {
	gCookieJar, _ := cookiejar.New(nil)
	zmzURL := "http://www.zimuzu.tv/User/Login/ajaxLogin"
	zmzClient = http.Client{
		Jar: gCookieJar,
	}
	zmzClient.PostForm(zmzURL, url.Values{"account": {"evol4snow"}, "password": {"104545"}, "remember": {"0"}})
}
<commit_msg>no results<commit_after>package main

import (
	"fmt"
	"github.com/evolsnow/robot/conn"
	"github.com/go-telegram-bot-api/telegram-bot-api"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"sync"
	"time"
)

//zmz.tv needs to login before downloading
var zmzClient http.Client

func (rb *Robot) Start(update tgbotapi.Update) string {
	user := update.Message.Chat.UserName
	go conn.SetUserChatId(user, update.Message.Chat.ID)
	return "welcome: " + user
}

func (rb *Robot) Help(update tgbotapi.Update) string {
	helpMsg := `
/alarm - set a reminder
/evolve	- self evolution of me
/movie - find movie download links
/show - find American show download links
/trans - translate words between english and chinese
/help - show this help message
`
	return helpMsg
}

func (rb *Robot) Evolve(update tgbotapi.Update) {
	if update.Message.Chat.FirstName != "Evol" || update.Message.Chat.LastName != "Gan" {
		rb.bot.Send(tgbotapi.NewMessage(update.Message.Chat.ID, "sorry, unauthorized"))
		return
	}
	<-saidGoodBye
	close(saidGoodBye)
	cmd := exec.Command("bash", "/root/evolve_"+rb.nickName)
	cmd.Start()
	os.Exit(1)
}

func (rb *Robot) Translate(update tgbotapi.Update) string {
	var info string
	if string(update.Message.Text[0]) == "/" {
		raw := strings.SplitAfterN(update.Message.Text, " ", 2)
		if len(raw) < 2 {
			return "what do you want to translate, try '/trans cat'?"
		} else {
			info = "翻译" + raw[1]
		}
	} else {
		info = update.Message.Text
	}

	return qinAI(info)

}
func (rb *Robot) Talk(update tgbotapi.Update) string {
	info := update.Message.Text
	if strings.Contains(info, rb.name) {
		if strings.Contains(info, "闭嘴") || strings.Contains(info, "别说话") {
			rb.shutUp = true
		} else if rb.shutUp && strings.Contains(info, "说话") {
			rb.shutUp = false
			return fmt.Sprintf("%s终于可以说话啦", rb.nickName)
		}
		info = strings.Replace(info, fmt.Sprintf("@%s", rb.name), "", -1)
	}

	if rb.shutUp {
		return ""
	}
	log.Printf(info)

	if rb.nickName == "samaritan" {
		if chinese(info) {
			return tlAI(info)
		} else {
			return mitAI(info)
		}
	} else { //jarvis use another AI
		return qinAI(info)
	}
	//	return response
}

func (rb *Robot) SetReminder(update tgbotapi.Update, step int) string {
	user := update.Message.Chat.UserName
	switch step {
	case 0:
		//known issue of go, you can not just assign update.Message.Chat.ID to userTask[user].ChatId
		tmpTask := userTask[user]
		tmpTask.ChatId = update.Message.Chat.ID
		tmpTask.Owner = update.Message.Chat.UserName
		userTask[user] = tmpTask

		tmpAction := userAction[user]
		tmpAction.ActionStep++
		userAction[user] = tmpAction
		return "Ok, what should I remind you to do?"
	case 1:
		//save thing
		tmpTask := userTask[user]
		tmpTask.Desc = update.Message.Text
		userTask[user] = tmpTask

		tmpAction := userAction[user]
		tmpAction.ActionStep++
		userAction[user] = tmpAction
		return "When or how much time after?\n" +
			"You can type:\n" +
			"'*2/14 11:30*' means 11:30 at 2/14 \n" + //first format
			"'*11:30*' means  11:30 today\n" + //second format
			"'*5m10s*' means 5 minutes 10 seconds later" //third format
	case 2:
		defer delete(userAction, user)
		//save time duration
		text := update.Message.Text
		text = strings.Replace(text, "：", ":", -1)
		firstFormat := "1/02 15:04"
		secondFormat := "15:04"
		thirdFormat := "15:04:05"
		var showTime string
		var scheduledTime time.Time
		var nowTime = time.Now()
		var du time.Duration
		var err error
		if strings.Contains(text, ":") {
			scheduledTime, err = time.Parse(firstFormat, text)
			nowTime, _ = time.Parse(firstFormat, nowTime.Format(firstFormat))
			showTime = scheduledTime.Format(firstFormat)
			if err != nil { //try to parse with first format
				scheduledTime, err = time.Parse(secondFormat, text)
				nowTime, _ = time.Parse(secondFormat, nowTime.Format(secondFormat))
				showTime = scheduledTime.Format(secondFormat)
				if err != nil {
					return "wrong format, try '2/14 11:30' or '11:30'?"
				}
			}
			du = scheduledTime.Sub(nowTime)
		} else {

			du, err = time.ParseDuration(text)
			scheduledTime = nowTime.Add(du)
			showTime = scheduledTime.Format(thirdFormat)
			if err != nil {
				return "wrong format, try '1h2m3s'?"
			}
		}
		//		tmpTask := userTask[user]
		//		tmpTask.When = scheduledTime
		//		userTask[user] = tmpTask
		go func(rb *Robot, ts Task) {
			timer := time.NewTimer(du)
			rawMsg := fmt.Sprintf("Hi %s, maybe it's time to:\n*%s*", ts.Owner, ts.Desc)
			msg := tgbotapi.NewMessage(ts.ChatId, rawMsg)
			msg.ParseMode = "markdown"
			<-timer.C
			_, err := rb.bot.Send(msg)
			if err != nil {
				rb.bot.Send(tgbotapi.NewMessage(conn.GetUserChatId(ts.Owner), rawMsg))
			}
			delete(userTask, user)
		}(rb, userTask[user])

		//		delete(userAction, user)
		return fmt.Sprintf("Ok, I will remind you that\n*%s* - *%s*", showTime, userTask[user].Desc)
	}
	return ""
}

func (rb *Robot) DownloadMovie(update tgbotapi.Update, step int, results chan string) (ret string) {
	user := update.Message.Chat.UserName
	switch step {
	case 0:
		//known issue of go, you can not just assign update.Message.Chat.ID to userTask[user].ChatId
		tmpAction := userAction[user]
		tmpAction.ActionStep++
		userAction[user] = tmpAction
		ret = "Ok, which movie do you want to download?"
	case 1:
		defer func() {
			delete(userAction, user)
			results <- "done"
		}()
		results <- "Searching movie..."
		movie := update.Message.Text
		var wg sync.WaitGroup
		wg.Add(2)
		go getMovieFromZMZ(movie, results, &wg)
		go getMovieFromLBL(movie, results, &wg)
		wg.Wait()
	}
	return
}

func (rb *Robot) DownloadShow(update tgbotapi.Update, step int, results chan string) (ret string) {
	user := update.Message.Chat.UserName
	switch step {
	case 0:
		//known issue of go, you can not just assign update.Message.Chat.ID to userTask[user].ChatId
		tmpAction := userAction[user]
		tmpAction.ActionStep++
		userAction[user] = tmpAction
		ret = "Ok, which American show do you want to download?"
	case 1:
		defer func() {
			delete(userAction, user)
			results <- "done"
		}()
		results <- "Searching American show..."
		info := strings.Split(update.Message.Text, " ")
		GetShowFromZMZ(info[0], info[1], info[2], results)
	}
	return
}

func getMovieFromLBL(movie string, results chan string, wg *sync.WaitGroup) {
	defer wg.Done()
	var id string
	resp, _ := http.Get("http://www.lbldy.com/search/" + movie)
	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	re, _ := regexp.Compile("
")
	//find first match case
	firstId := re.FindSubmatch(body)
	if len(firstId) == 0 {
		results <- fmt.Sprintf("No results for *%s* from LBL", movie)
		return
	} else {
		id = string(firstId[1])
		resp, _ = http.Get("http://www.lbldy.com/movie/" + id + ".html")
		defer resp.Body.Close()
		re, _ = regexp.Compile(`<p><a href="(.*?)"( target="_blank">|>)(.*?)</a></p>`)
		body, _ := ioutil.ReadAll(resp.Body)
		//go does not support (?!) regex
		body = []byte(strings.Replace(string(body), `<a href="/xunlei/"`, "", -1))
		downloads := re.FindAllSubmatch(body, -1)
		if len(downloads) == 0 {
			results <- fmt.Sprintf("No results for *%s* from LBL", movie)
			return
		} else {
			ret := "Results from LBL:\n\n"
			for i := range downloads {
				ret += fmt.Sprintf("*%s*\n```%s```\n\n", string(downloads[i][3]), string(downloads[i][1]))
				if i%5 == 0 && i > 0 {
					results <- ret
					ret = fmt.Sprintf("*LBL Part %d*\n\n", i/5+1)
				}
			}
			results <- ret
		}
	}
}

func getMovieFromZMZ(movie string, results chan string, wg *sync.WaitGroup) {
	defer wg.Done()
	loginZMZ()
	if downloads := getZMZResource(movie); downloads == nil {
		results <- fmt.Sprintf("No results for *%s* from ZMZ", movie)
		return
	} else {
		ret := "Results from ZMZ:\n\n"
		for i := range downloads {
			name := string(downloads[i][1])
			size := string(downloads[i][2])
			link := string(downloads[i][3])
			ret += fmt.Sprintf("*%s*(%s)\n```%s```\n\n", name, size, link)
			if i%5 == 0 && i > 0 {
				results <- ret
				ret = fmt.Sprintf("*ZMZ Part %d*\n\n", i/5+1)
			}
		}
		results <- ret
	}
	return
}

func GetShowFromZMZ(show, s, e string, results chan string) {
	loginZMZ()
	downloads := getZMZResource(show)
	if downloads == nil {
		results <- fmt.Sprintf("No results for *%s* from ZMZ", show)
		return
	}
	//second parse
	re, _ := regexp.Compile(fmt.Sprintf(".*?season=\"%s\" episode=\"%s\">.*?", s, e))
	results <- "Results from ZMZ:\n\n"
	count := 0
	for i := range downloads {
		if re.Find(downloads[i][0]) != nil {
			name := string(downloads[i][1])
			size := string(downloads[i][2])
			link := string(downloads[i][3])
			results <- fmt.Sprintf("*ZMZ %s*(%s)\n```%s```\n\n", name, size, link)
			count++
		}

	}
	if count == 0 {
		results <- fmt.Sprintf("No results found for *S%sE%s*", s, e)

	}
	return
}

func getZMZResource(name string) [][][]byte {
	id := getZMZResourceId(name)
	if id == "" {
		return nil
	}
	resourceURL := "http://www.zimuzu.tv/resource/list/" + id
	resp, _ := zmzClient.Get(resourceURL)
	defer resp.Body.Close()
	//1.name 2.size 3.link
	re, _ := regexp.Compile(`<li class="clearfix".*?<input type="checkbox"><a title="(.*?)".*?<font class="f3">(.*?)</font>.*?<a href="(.*?)" type="ed2k">`)
	body, _ := ioutil.ReadAll(resp.Body)
	body = []byte(strings.Replace(string(body), "\n", "", -1))
	//	tmp := (strings.Replace(string(body), "
\n", "", -1))
	//	body = []byte(strings.Replace(tmp, "
\n", "", -1))
	downloads := re.FindAllSubmatch(body, -1)
	if len(downloads) == 0 {
		return nil
	}
	return downloads
}

func getZMZResourceId(name string) (id string) {
	queryURL := fmt.Sprintf("http://www.zimuzu.tv/search?keyword=%s&type=resource", name)
	re, _ := regexp.Compile(`<div class="t f14"><a href="/resource/(.*?)"><strong class="list_title">`)
	resp, _ := zmzClient.Get(queryURL)
	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	//find first match case
	firstId := re.FindSubmatch(body)
	if len(firstId) == 0 {
		return
	} else {
		log.Println(id)
		id = string(firstId[1])
		return
	}
}

func loginZMZ() {
	gCookieJar, _ := cookiejar.New(nil)
	zmzURL := "http://www.zimuzu.tv/User/Login/ajaxLogin"
	zmzClient = http.Client{
		Jar: gCookieJar,
	}
	zmzClient.PostForm(zmzURL, url.Values{"account": {"evol4snow"}, "password": {"104545"}, "remember": {"0"}})
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/nlopes/slack"
	"gopkg.in/redis.v3"

	messages "github.com/foxelbox/foxbukkitslacklink/messages"
)

func (s *SlackLink) handleSlackMessage(msg *slack.MessageEvent) {
	info := s.slack.GetInfo()

	if msg.UserId == "" || msg.UserId == info.User.Id {
		return
	}

	if strings.HasPrefix(msg.Text, ".") {
		// Always handle commands, regardless of channel
		s.forwardSlackMessageToChatLink(msg)
		return
	}

	channel := info.GetChannelById(msg.ChannelId)

	if channel == nil {
		// We don't know about this channel.
		return
	}

	switch channel.Name {
	case "minecraft":
	case "minecraft-ops":
		msg.Text = "#" + msg.Text
	default:
		log.Printf("Ignored message from %s", channel.Name)
		return
	}

	s.forwardSlackMessageToChatLink(msg)
}

func (s *SlackLink) forwardSlackMessageToChatLink(msg *slack.MessageEvent) {
	if strings.HasPrefix(msg.Text, ".") {
		msg.Text = "/" + msg.Text[1:]
	}

	minecraftAccount := s.getMinecraftFromSlack(msg.UserId)
	if minecraftAccount == nil {
		// They aren't associated with an account. Ignore.
		return
	}

	cmi := &ChatMessageIn{
		Server:  "Slack",
		Context: uuid.NewRandom(),
		Type:    messages.MessageType_TEXT,

		From: minecraftAccount,

		Timestamp: parseSlackTimestamp(msg.Timestamp),

		Contents: msg.Text,
	}

	s.addContextAssociation(cmi.Context, msg.ChannelId)

	log.Printf("Outgoing message: %#v", cmi)
	s.chatLinkOut <- CMIToProtoCMI(cmi)
}

func (s *SlackLink) handlePresenceChange(ev *slack.PresenceChangeEvent) {
	mcID, err := s.redis.HGet("slacklinks:slack-to-mc", ev.UserId).Result()
	if err == redis.Nil {
		return
	} else if err != nil {
		panic(err)
	}

	if ev.Presence == "active" {
		s.redis.SAdd("playersOnline:Slack", mcID)
	} else {
		s.redis.SRem("playersOnline:Slack", mcID)
	}
}

func (s *SlackLink) handleSlackMessages() {
	defer s.wg.Done()

	for msg := range s.slackMessages {
		switch data := msg.Data.(type) {
		case *slack.MessageEvent:
			s.handleSlackMessage(data)
		case *slack.PresenceChangeEvent:
			s.handlePresenceChange(data)
		case slack.HelloEvent:
			s.refreshPresenceInfo()
		case *slack.SlackWSError:
			panic(data)
		default:
			log.Printf("Unhandled message: %T", data)
		}
	}
}

func (s *SlackLink) refreshPresenceInfo() {
	users := s.slack.GetInfo().Users

	slackToMC, err := s.redis.HGetAllMap("slacklinks:slack-to-mc").Result()
	if err != nil {
		panic(err)
	}

	mcIDs := make([]string, 0, len(slackToMC))
	for _, user := range users {
		mcID, ok := slackToMC[user.Id]
		if !ok {
			continue
		}

		if user.Presence != "active" {
			continue
		}

		mcIDs = append(mcIDs, mcID)
	}

	s.redis.Del("playersOnline:Slack")
	s.redis.SAdd("playersOnline:Slack", mcIDs...)
}

func (s *SlackLink) receiveSlackMessages() {
	defer s.wg.Done()

	rtm, err := s.slack.StartRTM("", "https://slack.com")
	if err != nil {
		panic(err)
	}

	rtm.SetUserAsActive()

	rtm.HandleIncomingEvents(s.slackMessages)
}

func (s *SlackLink) sendSlackMessages() {
	defer s.wg.Done()

	for msg := range s.slackOut {
		params := slack.NewPostMessageParameters()

		params.Markdown = !msg.DisableMarkdown

		if msg.As != nil {
			params.AsUser = false
			params.Username = msg.As.Name
			params.IconURL = "https://minotar.net/avatar/" + url.QueryEscape(strings.ToLower(msg.As.Name)) + "/48.png"
		} else {
			params.AsUser = true
		}

		s.slack.PostMessage(msg.To, msg.Message, params)
	}
}

type SlackMessage struct {
	To      string
	Message string

	As              *MinecraftPlayer
	DisableMarkdown bool
}

func parseSlackTimestamp(ts string) time.Time {
	f, _ := strconv.ParseFloat(ts, 64)

	tsInt := int64(f)
	leftover := f - float64(tsInt)
	leftoverNsec := int64((float64(time.Second) * leftover) / float64(time.Nanosecond))

	return time.Unix(tsInt, leftoverNsec)
}
<commit_msg>Remove debug prints<commit_after>package main

import (
	"log"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/nlopes/slack"
	"gopkg.in/redis.v3"

	messages "github.com/foxelbox/foxbukkitslacklink/messages"
)

func (s *SlackLink) handleSlackMessage(msg *slack.MessageEvent) {
	info := s.slack.GetInfo()

	if msg.UserId == "" || msg.UserId == info.User.Id {
		return
	}

	if strings.HasPrefix(msg.Text, ".") {
		// Always handle commands, regardless of channel
		s.forwardSlackMessageToChatLink(msg)
		return
	}

	channel := info.GetChannelById(msg.ChannelId)

	if channel == nil {
		// We don't know about this channel.
		return
	}

	switch channel.Name {
	case "minecraft":
	case "minecraft-ops":
		msg.Text = "#" + msg.Text
	default:
		return
	}

	s.forwardSlackMessageToChatLink(msg)
}

func (s *SlackLink) forwardSlackMessageToChatLink(msg *slack.MessageEvent) {
	if strings.HasPrefix(msg.Text, ".") {
		msg.Text = "/" + msg.Text[1:]
	}

	minecraftAccount := s.getMinecraftFromSlack(msg.UserId)
	if minecraftAccount == nil {
		// They aren't associated with an account. Ignore.
		return
	}

	cmi := &ChatMessageIn{
		Server:  "Slack",
		Context: uuid.NewRandom(),
		Type:    messages.MessageType_TEXT,

		From: minecraftAccount,

		Timestamp: parseSlackTimestamp(msg.Timestamp),

		Contents: msg.Text,
	}

	s.addContextAssociation(cmi.Context, msg.ChannelId)

	s.chatLinkOut <- CMIToProtoCMI(cmi)
}

func (s *SlackLink) handlePresenceChange(ev *slack.PresenceChangeEvent) {
	mcID, err := s.redis.HGet("slacklinks:slack-to-mc", ev.UserId).Result()
	if err == redis.Nil {
		return
	} else if err != nil {
		panic(err)
	}

	if ev.Presence == "active" {
		s.redis.SAdd("playersOnline:Slack", mcID)
	} else {
		s.redis.SRem("playersOnline:Slack", mcID)
	}
}

func (s *SlackLink) handleSlackMessages() {
	defer s.wg.Done()

	for msg := range s.slackMessages {
		switch data := msg.Data.(type) {
		case *slack.MessageEvent:
			s.handleSlackMessage(data)
		case *slack.PresenceChangeEvent:
			s.handlePresenceChange(data)
		case slack.HelloEvent:
			s.refreshPresenceInfo()
		case *slack.SlackWSError:
			panic(data)
		default:
			log.Printf("Unhandled message: %T", data)
		}
	}
}

func (s *SlackLink) refreshPresenceInfo() {
	users := s.slack.GetInfo().Users

	slackToMC, err := s.redis.HGetAllMap("slacklinks:slack-to-mc").Result()
	if err != nil {
		panic(err)
	}

	mcIDs := make([]string, 0, len(slackToMC))
	for _, user := range users {
		mcID, ok := slackToMC[user.Id]
		if !ok {
			continue
		}

		if user.Presence != "active" {
			continue
		}

		mcIDs = append(mcIDs, mcID)
	}

	s.redis.Del("playersOnline:Slack")
	s.redis.SAdd("playersOnline:Slack", mcIDs...)
}

func (s *SlackLink) receiveSlackMessages() {
	defer s.wg.Done()

	rtm, err := s.slack.StartRTM("", "https://slack.com")
	if err != nil {
		panic(err)
	}

	rtm.SetUserAsActive()

	rtm.HandleIncomingEvents(s.slackMessages)
}

func (s *SlackLink) sendSlackMessages() {
	defer s.wg.Done()

	for msg := range s.slackOut {
		params := slack.NewPostMessageParameters()

		params.Markdown = !msg.DisableMarkdown

		if msg.As != nil {
			params.AsUser = false
			params.Username = msg.As.Name
			params.IconURL = "https://minotar.net/avatar/" + url.QueryEscape(strings.ToLower(msg.As.Name)) + "/48.png"
		} else {
			params.AsUser = true
		}

		s.slack.PostMessage(msg.To, msg.Message, params)
	}
}

type SlackMessage struct {
	To      string
	Message string

	As              *MinecraftPlayer
	DisableMarkdown bool
}

func parseSlackTimestamp(ts string) time.Time {
	f, _ := strconv.ParseFloat(ts, 64)

	tsInt := int64(f)
	leftover := f - float64(tsInt)
	leftoverNsec := int64((float64(time.Second) * leftover) / float64(time.Nanosecond))

	return time.Unix(tsInt, leftoverNsec)
}
<|endoftext|>


<commit_before>package rdb

// #include <stdlib.h>
import "C"

import "unsafe"

// Slice is used as a wrapper for non-copy values
type Slice struct {
	data  *C.char
	size  C.size_t
	freed bool
}

func NewSlice(data *C.char, size C.size_t) *Slice {
	return &Slice{data, size, false}
}

func (self *Slice) Data() []byte {
	return charToByte(self.data, self.size)
}

func (self *Slice) Size() int {
	return int(self.size)
}

func (self *Slice) Free() {
	if !self.freed {
		C.free(unsafe.Pointer(self.data))
		self.freed = true
	}
}
<commit_msg>used rocksdb_free to dealocate malloc()ed memory (see c.h of rocksdb)<commit_after>package rdb

// #include <stdlib.h>
// #include "rocksdb/c.h"
import "C"

// Slice is used as a wrapper for non-copy values
type Slice struct {
	data  *C.char
	size  C.size_t
	freed bool
}

func NewSlice(data *C.char, size C.size_t) *Slice {
	return &Slice{data, size, false}
}

func (self *Slice) Data() []byte {
	return charToByte(self.data, self.size)
}

func (self *Slice) Size() int {
	return int(self.size)
}

func (self *Slice) Free() {
	if !self.freed {
		C.rocksdb_free(self.data)
		self.freed = true
	}
}
<|endoftext|>