<commit_before>// Provides functions to query web interfaces.
package query

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/Islandstone/yr"
)

const (
	geoURI    = "http://api.geonames.org/search?username=%s&q=%s&maxRows=1&type=json&orderby=relevance"
	yrURI     = "http://www.yr.no/place/%s/%s/%s/forecast.xml"
	yrNorURI  = "http://www.yr.no/place/%s/%s/%s/%s/forecast.xml"
	geoErrMsg = "Unable to find %s"
)

var placesLookup = map[string][]string{
	"oslo":     {"Norway", "Oslo", "Oslo", "Oslo"},
	"sandvika": {"Norway", "Akershus", "Bærum", "Sandvika"},
}

// GoogleData is used to parse the response from Google.
type GoogleData struct {
	ResponseData   *ResponseData
	ResponseStatus float64
}

// ResponseData is a substruct of GoogleData.
type ResponseData struct {
	Results []*Result
	Cursor  *Cursor
}

// Result is a substruct of GoogleData.
type Result struct {
	GsearchResultClass string
	UnescapedUrl       string
	Url                string
	VisibleUrl         string
	CacheUrl           string
	Title              string
	TitleNoFormatting  string
	Content            string
}

// Cursor is a substruct of GoogleData.
type Cursor struct {
	ResultCount    string
	MoreResultsUrl string
}

// WolframData is used to parse the response from WolframAlpha.
type WolframData struct {
	XMLName     xml.Name `xml:"queryresult"`
	Success     bool     `xml:"success,attr"`
	ParseTiming float64  `xml:"parsetiming,attr"`
	Numpods     int      `xml:"numpods,attr"`
	Pods        []*Pod   `xml:"pod"`
	DidYouMeans []string `xml:"didyoumeans>didyoumean"`
}

// Pod is a substruct of WolframData.
type Pod struct {
	Title      string   `xml:"title,attr"`
	Id         string   `xml:"id,attr"`
	Primary    bool     `xml:"primary,attr"`
	Numsubpods int      `xml:"numsubpods,attr"`
	PlainTexts []string `xml:"subpod>plaintext"`
}

// Config is the configuration for this thing.
type Config struct {
	WolframId    string `toml:"wolfram_id"`
	GeonamesId   string `toml:"geonames_id"`
	GoogleApiKey string `toml:"google_api_key"`
}

// NewConfig loads the config file.
func NewConfig(file string) *Config {
	var conf Config
	_, err := toml.DecodeFile(file, &conf)
	if err != nil {
		return nil
	}
	return &conf
}

var (
	rgxTitle = regexp.MustCompile(`<meta name="title" content="(.*?)"`)
	rgxUrl   = regexp.MustCompile(
		`(http|https):\/\/(www\.)?(youtube.com|youtu.be)\/\S+`)
	rgxDur = regexp.MustCompile(
		`<meta itemprop="duration" content="PT([0-9]+M[0-9]+S)"`)
	rgxTags = regexp.MustCompile(`<[^>]*>`)
)

const (
	googleURI  = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=%s"
	shortenURI = "https://www.googleapis.com/urlshortener/v1/url?fields=id,longUrl&key=%s"
	wolframURI = "http://api.wolframalpha.com/v2/query?format=plaintext&input=%s&appid=%s"
)

// Google performs a query and returns a formatted result.
func Google(query string, conf *Config) (output string, err error) {
	var resp *http.Response
	resp, err = http.Get(fmt.Sprintf(googleURI, url.QueryEscape(query)))
	if err != nil {
		return
	}

	defer resp.Body.Close()
	var jsonObj GoogleData
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&jsonObj)
	if err != nil {
		return "", err
	}

	if jsonObj.ResponseData == nil || int(jsonObj.ResponseStatus) != 200 || len(jsonObj.ResponseData.Results) == 0 {
		return fmt.Sprintf("\x02Google: No results found."), err
	}

	result := jsonObj.ResponseData.Results[0]
	url := rgxTags.ReplaceAllString(result.UnescapedUrl, "")
	var short string

	if len(conf.GoogleApiKey) > 0 {
		short, _ = GetShortUrl(jsonObj.ResponseData.Cursor.MoreResultsUrl, conf)
		if len(short) > 0 {
			short = fmt.Sprintf(" [%s]", short)
		}
	}

	output = fmt.Sprintf(
		"\x02Google (\x02%s results\x02)%s:\x02 %s - %s",
		jsonObj.ResponseData.Cursor.ResultCount,
		short,
		url,
		html.UnescapeString(rgxTags.ReplaceAllString(result.Content, "")),
	)

	return output, nil
}

type URLShortenResponse struct {
	ID      string `json:"id"`
	LongURL string `json:"longUrl"`
}

type URLShortenQuery struct {
	LongURL string `json:"longUrl"`
}

// GetShortUrl takes a long url and returns a shorter url from the Google API
func GetShortUrl(longURL string, conf *Config) (short string, err error) {
	var resp *http.Response
	var body []byte

	urlQuery := URLShortenQuery{LongURL: longURL}
	if body, err = json.Marshal(urlQuery); err != nil {
		return "", fmt.Errorf("failed to marshal url query: %v", err)
	}

	resp, err = http.Post(
		fmt.Sprintf(shortenURI, conf.GoogleApiKey),
		"application/json",
		bytes.NewReader(body),
	)

	if err != nil {
		return
	}
	defer resp.Body.Close()

	var jsonObj URLShortenResponse
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&jsonObj)
	if err != nil {
		return "", err
	}

	return jsonObj.ID, nil
}

// Wolfram performs a query and returns a formatted result.
func Wolfram(query string, conf *Config) (output string, err error) {
	var resp *http.Response
	requestUri := fmt.Sprintf(wolframURI, url.QueryEscape(query), conf.WolframId)
	resp, err = http.Get(requestUri)
	if err != nil {
		return
	}

	defer resp.Body.Close()
	var xmlObj WolframData
	decoder := xml.NewDecoder(resp.Body)
	err = decoder.Decode(&xmlObj)
	if err != nil {
		return
	}

	// Handle cases of no results.
	if !xmlObj.Success {
		if len(xmlObj.DidYouMeans) > 0 {
			output = fmt.Sprintf("\x02Wolfram (\x02%.2fms\x02):\x02 Did you mean: %s",
				xmlObj.ParseTiming,
				xmlObj.DidYouMeans[0],
			)
		} else {
			output = fmt.Sprintf("\x02Wolfram (\x02%.2fms\x02):\x02 No results found.",
				xmlObj.ParseTiming)
		}
		return
	}

	// If there was no primary response fallback to link.
	if len(xmlObj.Pods) < 2 || len(xmlObj.Pods[1].PlainTexts[0]) == 0 {
		output = fmt.Sprintf(
			"\x02Wolfram (\x02%.2fms\x02):\x02 %s \x02=>\x02 http://www.wolframalpha.com/input/?i=%s",
			xmlObj.ParseTiming,
			xmlObj.Pods[0].PlainTexts[0],
			url.QueryEscape(query),
		)
		return
	}

	output = fmt.Sprintf(
		"\x02Wolfram (\x02%.2fms\x02):\x02 %s \x02=>\x02 %s",
		xmlObj.ParseTiming,
		xmlObj.Pods[0].PlainTexts[0],
		xmlObj.Pods[1].PlainTexts[0],
	)

	return
}

// YouTube will check to see if a message contains a YouTube uri, if so it will
// format a string with the title in it.
func YouTube(msg string) (output string, err error) {
	link := rgxUrl.FindString(msg)
	if len(link) == 0 {
		return
	}

	var resp *http.Response
	var body []byte

	resp, err = http.Get(link)
	if err != nil {
		return
	}

	defer resp.Body.Close()
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	var buf = bytes.NewBufferString("\x02YouTube")
	if duration := rgxDur.FindSubmatch(body); duration != nil {
		dur, e := time.ParseDuration(string(bytes.ToLower(duration[1])))
		if e == nil {
			buf.WriteString(" (\x02")
			buf.WriteString(dur.String())
			buf.WriteString("\x02)")
		}
	}

	if title := rgxTitle.FindSubmatch(body); title != nil {
		buf.WriteString(":\x02 ")
		buf.WriteString(html.UnescapeString(string(title[1])))
		output = buf.String()
	}

	return
}

type geonameplace struct {
	CountryName string
	AdminName1  string
	Name        string
}

type geonamesdata struct {
	Geonames []geonameplace
}

type geoErr struct {
	query string
}

func (g geoErr) Error() string {
	return fmt.Sprintf(geoErrMsg, g.query)
}

func getLocation(query string, conf *Config) (country, state, city string, err error) {
	resp, err := http.Get(fmt.Sprintf(geoURI, conf.GeonamesId, url.QueryEscape(query)))

	if err != nil {
		return
	}

	defer resp.Body.Close()

	r := json.NewDecoder(resp.Body)

	var data geonamesdata
	err = r.Decode(&data)

	if err != nil {
		return
	}

	if len(data.Geonames) == 0 {
		err = geoErr{query}
		return
	}

	country = data.Geonames[0].CountryName
	state = data.Geonames[0].AdminName1
	city = data.Geonames[0].Name

	return
}

func Weather(query string, conf *Config) (output string, err error) {
	var data *yr.WeatherData
	var URL, city, country, state string

	if subURI, ok := placesLookup[strings.ToLower(query)]; ok {
		country = subURI[0]
		state = subURI[1]
		county := subURI[2]
		city = subURI[3]

		URL = fmt.Sprintf(yrNorURI, country, state, county, city)
	} else {
		country, state, city, err = getLocation(query, conf)

		if err != nil {
			if e, ok := err.(geoErr); ok {
				return fmt.Sprintf("\x02Weather (\x02YR.no\x02):\x02 %v", e), nil
			}
			return "", err
		}

		URL = fmt.Sprintf(yrURI, country, state, city)
	}

	data, err = yr.LoadFromURL(URL)

	if err != nil {
		return
	}

	output = fmt.Sprintf(
		"\x02Weather (\x02YR.no\x02):\x02 %s, %s \x02=>\x02 %s, %d \u00B0C",
		city,
		country,
		data.Current().Symbol.Name,
		data.Current().Temperature.Value,
	)

	return
}
<commit_msg>Fix output a bit more.<commit_after>// Provides functions to query web interfaces.
package query

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/Islandstone/yr"
)

const (
	geoURI    = "http://api.geonames.org/search?username=%s&q=%s&maxRows=1&type=json&orderby=relevance"
	yrURI     = "http://www.yr.no/place/%s/%s/%s/forecast.xml"
	yrNorURI  = "http://www.yr.no/place/%s/%s/%s/%s/forecast.xml"
	geoErrMsg = "Unable to find %s"
)

var placesLookup = map[string][]string{
	"oslo":     {"Norway", "Oslo", "Oslo", "Oslo"},
	"sandvika": {"Norway", "Akershus", "Bærum", "Sandvika"},
}

// GoogleData is used to parse the response from Google.
type GoogleData struct {
	ResponseData   *ResponseData
	ResponseStatus float64
}

// ResponseData is a substruct of GoogleData.
type ResponseData struct {
	Results []*Result
	Cursor  *Cursor
}

// Result is a substruct of GoogleData.
type Result struct {
	GsearchResultClass string
	UnescapedUrl       string
	Url                string
	VisibleUrl         string
	CacheUrl           string
	Title              string
	TitleNoFormatting  string
	Content            string
}

// Cursor is a substruct of GoogleData.
type Cursor struct {
	ResultCount    string
	MoreResultsUrl string
}

// WolframData is used to parse the response from WolframAlpha.
type WolframData struct {
	XMLName     xml.Name `xml:"queryresult"`
	Success     bool     `xml:"success,attr"`
	ParseTiming float64  `xml:"parsetiming,attr"`
	Numpods     int      `xml:"numpods,attr"`
	Pods        []*Pod   `xml:"pod"`
	DidYouMeans []string `xml:"didyoumeans>didyoumean"`
}

// Pod is a substruct of WolframData.
type Pod struct {
	Title      string   `xml:"title,attr"`
	Id         string   `xml:"id,attr"`
	Primary    bool     `xml:"primary,attr"`
	Numsubpods int      `xml:"numsubpods,attr"`
	PlainTexts []string `xml:"subpod>plaintext"`
}

// Config is the configuration for this thing.
type Config struct {
	WolframId    string `toml:"wolfram_id"`
	GeonamesId   string `toml:"geonames_id"`
	GoogleApiKey string `toml:"google_api_key"`
}

// NewConfig loads the config file.
func NewConfig(file string) *Config {
	var conf Config
	_, err := toml.DecodeFile(file, &conf)
	if err != nil {
		return nil
	}
	return &conf
}

var (
	rgxTitle = regexp.MustCompile(`<meta name="title" content="(.*?)"`)
	rgxUrl   = regexp.MustCompile(
		`(http|https):\/\/(www\.)?(youtube.com|youtu.be)\/\S+`)
	rgxDur = regexp.MustCompile(
		`<meta itemprop="duration" content="PT([0-9]+M[0-9]+S)"`)
	rgxTags = regexp.MustCompile(`<[^>]*>`)
)

const (
	googleURI  = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=%s"
	shortenURI = "https://www.googleapis.com/urlshortener/v1/url?fields=id,longUrl&key=%s"
	wolframURI = "http://api.wolframalpha.com/v2/query?format=plaintext&input=%s&appid=%s"
)

// Google performs a query and returns a formatted result.
func Google(query string, conf *Config) (output string, err error) {
	var resp *http.Response
	resp, err = http.Get(fmt.Sprintf(googleURI, url.QueryEscape(query)))
	if err != nil {
		return
	}

	defer resp.Body.Close()
	var jsonObj GoogleData
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&jsonObj)
	if err != nil {
		return "", err
	}

	if jsonObj.ResponseData == nil || int(jsonObj.ResponseStatus) != 200 || len(jsonObj.ResponseData.Results) == 0 {
		return fmt.Sprintf("\x02Google: No results found."), err
	}

	result := jsonObj.ResponseData.Results[0]
	url := rgxTags.ReplaceAllString(result.UnescapedUrl, "")
	var short string

	if len(conf.GoogleApiKey) > 0 {
		short, _ = GetShortUrl(jsonObj.ResponseData.Cursor.MoreResultsUrl, conf)
		if len(short) > 0 {
			short = fmt.Sprintf(" %s", short)
		}
	}

	output = fmt.Sprintf(
		"\x02Google (\x02%s results%s\x02):\x02 %s - %s",
		jsonObj.ResponseData.Cursor.ResultCount,
		short,
		url,
		html.UnescapeString(rgxTags.ReplaceAllString(result.Content, "")),
	)

	return output, nil
}

type URLShortenResponse struct {
	ID      string `json:"id"`
	LongURL string `json:"longUrl"`
}

type URLShortenQuery struct {
	LongURL string `json:"longUrl"`
}

// GetShortUrl takes a long url and returns a shorter url from the Google API
func GetShortUrl(longURL string, conf *Config) (short string, err error) {
	var resp *http.Response
	var body []byte

	urlQuery := URLShortenQuery{LongURL: longURL}
	if body, err = json.Marshal(urlQuery); err != nil {
		return "", fmt.Errorf("failed to marshal url query: %v", err)
	}

	resp, err = http.Post(
		fmt.Sprintf(shortenURI, conf.GoogleApiKey),
		"application/json",
		bytes.NewReader(body),
	)

	if err != nil {
		return
	}
	defer resp.Body.Close()

	var jsonObj URLShortenResponse
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&jsonObj)
	if err != nil {
		return "", err
	}

	return jsonObj.ID, nil
}

// Wolfram performs a query and returns a formatted result.
func Wolfram(query string, conf *Config) (output string, err error) {
	var resp *http.Response
	requestUri := fmt.Sprintf(wolframURI, url.QueryEscape(query), conf.WolframId)
	resp, err = http.Get(requestUri)
	if err != nil {
		return
	}

	defer resp.Body.Close()
	var xmlObj WolframData
	decoder := xml.NewDecoder(resp.Body)
	err = decoder.Decode(&xmlObj)
	if err != nil {
		return
	}

	// Handle cases of no results.
	if !xmlObj.Success {
		if len(xmlObj.DidYouMeans) > 0 {
			output = fmt.Sprintf("\x02Wolfram (\x02%.2fms\x02):\x02 Did you mean: %s",
				xmlObj.ParseTiming,
				xmlObj.DidYouMeans[0],
			)
		} else {
			output = fmt.Sprintf("\x02Wolfram (\x02%.2fms\x02):\x02 No results found.",
				xmlObj.ParseTiming)
		}
		return
	}

	// If there was no primary response fallback to link.
	if len(xmlObj.Pods) < 2 || len(xmlObj.Pods[1].PlainTexts[0]) == 0 {
		output = fmt.Sprintf(
			"\x02Wolfram (\x02%.2fms\x02):\x02 %s \x02=>\x02 http://www.wolframalpha.com/input/?i=%s",
			xmlObj.ParseTiming,
			xmlObj.Pods[0].PlainTexts[0],
			url.QueryEscape(query),
		)
		return
	}

	output = fmt.Sprintf(
		"\x02Wolfram (\x02%.2fms\x02):\x02 %s \x02=>\x02 %s",
		xmlObj.ParseTiming,
		xmlObj.Pods[0].PlainTexts[0],
		xmlObj.Pods[1].PlainTexts[0],
	)

	return
}

// YouTube will check to see if a message contains a YouTube uri, if so it will
// format a string with the title in it.
func YouTube(msg string) (output string, err error) {
	link := rgxUrl.FindString(msg)
	if len(link) == 0 {
		return
	}

	var resp *http.Response
	var body []byte

	resp, err = http.Get(link)
	if err != nil {
		return
	}

	defer resp.Body.Close()
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	var buf = bytes.NewBufferString("\x02YouTube")
	if duration := rgxDur.FindSubmatch(body); duration != nil {
		dur, e := time.ParseDuration(string(bytes.ToLower(duration[1])))
		if e == nil {
			buf.WriteString(" (\x02")
			buf.WriteString(dur.String())
			buf.WriteString("\x02)")
		}
	}

	if title := rgxTitle.FindSubmatch(body); title != nil {
		buf.WriteString(":\x02 ")
		buf.WriteString(html.UnescapeString(string(title[1])))
		output = buf.String()
	}

	return
}

type geonameplace struct {
	CountryName string
	AdminName1  string
	Name        string
}

type geonamesdata struct {
	Geonames []geonameplace
}

type geoErr struct {
	query string
}

func (g geoErr) Error() string {
	return fmt.Sprintf(geoErrMsg, g.query)
}

func getLocation(query string, conf *Config) (country, state, city string, err error) {
	resp, err := http.Get(fmt.Sprintf(geoURI, conf.GeonamesId, url.QueryEscape(query)))

	if err != nil {
		return
	}

	defer resp.Body.Close()

	r := json.NewDecoder(resp.Body)

	var data geonamesdata
	err = r.Decode(&data)

	if err != nil {
		return
	}

	if len(data.Geonames) == 0 {
		err = geoErr{query}
		return
	}

	country = data.Geonames[0].CountryName
	state = data.Geonames[0].AdminName1
	city = data.Geonames[0].Name

	return
}

func Weather(query string, conf *Config) (output string, err error) {
	var data *yr.WeatherData
	var URL, city, country, state string

	if subURI, ok := placesLookup[strings.ToLower(query)]; ok {
		country = subURI[0]
		state = subURI[1]
		county := subURI[2]
		city = subURI[3]

		URL = fmt.Sprintf(yrNorURI, country, state, county, city)
	} else {
		country, state, city, err = getLocation(query, conf)

		if err != nil {
			if e, ok := err.(geoErr); ok {
				return fmt.Sprintf("\x02Weather (\x02YR.no\x02):\x02 %v", e), nil
			}
			return "", err
		}

		URL = fmt.Sprintf(yrURI, country, state, city)
	}

	data, err = yr.LoadFromURL(URL)

	if err != nil {
		return
	}

	output = fmt.Sprintf(
		"\x02Weather (\x02YR.no\x02):\x02 %s, %s \x02=>\x02 %s, %d \u00B0C",
		city,
		country,
		data.Current().Symbol.Name,
		data.Current().Temperature.Value,
	)

	return
}
<|endoftext|>


<commit_before>package sqlread

type Query struct {
	Columns []string
	Table   string
	Outfile *string
}

type QueryTree struct {
	Queries     []Query
	ShowTables  uint
	ShowColumns []string
	Quit        bool
}

type QueryParser struct {
	Tree QueryTree
}

func NewQueryParser() *QueryParser {
	return &QueryParser{
		Tree: QueryTree{
			Queries:     []Query{},
			ShowColumns: []string{},
			ShowTables:  0,
			Quit:        false,
		},
	}
}

func (q *QueryParser) ParseStart(p *Parser) parseState {
	for {
		c, ok := p.scan()
		if !ok {
			break
		}

		if isOfAny(c, TIntpSelect) {
			return q.parseSelect
		}

		if isOfAny(c, TIntpShowTables) {
			return q.parseShowTables
		}

		if isOfAny(c, TIntpShowColumns) {
			return q.parseShowColumns
		}

		if isOfAny(c, TIntpQuit) {
			return q.parseQuit
		}

		p.errorUnexpectedLex(c, TIntpSelect, TIntpShowTables, TIntpQuit)
		return nil
	}

	return nil
}

func (q *QueryParser) parseQuit(p *Parser) parseState {
	x, ok := p.scan()
	if !ok {
		p.errorUnexpectedEOF()
		return nil
	}
	if !isOfAny(x, TSemi) {
		p.errorUnexpectedLex(x, TSemi)
		return nil
	}

	q.Tree.Quit = true

	return q.ParseStart
}

func (q *QueryParser) parseShowTables(p *Parser) parseState {
	x, ok := p.scan()
	if !ok {
		p.errorUnexpectedEOF()
		return nil
	}
	if !isOfAny(x, TSemi) {
		p.errorUnexpectedLex(x, TSemi)
		return nil
	}

	q.Tree.ShowTables++

	return q.ParseStart
}

func (q *QueryParser) parseShowColumns(p *Parser) parseState {
	x, ok := p.scan()
	if !ok {
		p.errorUnexpectedEOF()
		return nil
	}
	if !isOfAny(x, TIntpFrom) {
		p.errorUnexpectedLex(x, TIntpFrom)
		return nil
	}

	y, ok := p.scan()
	if !ok {
		p.errorUnexpectedEOF()
		return nil
	}
	if !isOfAny(y, TIdentifier) {
		p.errorUnexpectedLex(x, TIdentifier)
		return nil
	}

	q.Tree.ShowColumns = append(q.Tree.ShowColumns, identValue(y))

	return q.ParseStart
}

func (q *QueryParser) parseSelect(p *Parser) parseState {
	qry := &Query{
		Columns: make([]string, 0),
	}

	for {
		i, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}

		if isOfAny(i, TIdentifier, TIntpStar) {
			value := i.Val
			if i.Type == TIdentifier {
				value = identValue(i)
			}

			qry.Columns = append(qry.Columns, value)

			j, ok := p.scan()
			if !ok {
				p.errorUnexpectedEOF()
				return nil
			}

			if j.Type == TComma {
				continue
			}

			if j.Type == TIntpFrom {
				return q.parseSelectFromBuilder(qry)
			}

			p.errorUnexpectedLex(i, TComma, TIntpFrom)
			return nil
		}

		p.errorUnexpectedLex(i, TIdentifier, TIntpStar)
		return nil
	}
}

func (q *QueryParser) parseSelectFromBuilder(qry *Query) parseState {
	return func(p *Parser) parseState {
		c, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}

		if c.Type != TIdentifier {
			p.errorUnexpectedLex(c, TIdentifier)
			return nil
		}

		qry.Table = identValue(c)

		d, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}

		if d.Type == TSemi {
			q.Tree.Queries = append(q.Tree.Queries, *qry)
			return nil
		}

		if d.Type == TIntpIntoOutfile {
			return q.parseSelectIntoOutfileBuilder(qry)
		}

		p.errorUnexpectedLex(d, TSemi, TIntpIntoOutfile)
		return nil
	}
}

func (q *QueryParser) parseSelectIntoOutfileBuilder(qry *Query) parseState {
	return func(p *Parser) parseState {
		c, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}

		if c.Type != TString {
			p.errorUnexpectedLex(c, TString)
			return nil
		}

		s, err := stringValue(c)
		if err != nil {
			p.err = err
			return nil
		}

		qry.Outfile = &s

		d, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}
		if d.Type == TSemi {
			q.Tree.Queries = append(q.Tree.Queries, *qry)
			return nil
		}

		p.errorUnexpectedLex(d, TSemi)
		return nil
	}
}

func identValue(c LexItem) string {
	// remove backticks - if/when we implement backtickless
	// identifiers this will need to be better handled.
	return c.Val[1 : len(c.Val)-1]
}
<commit_msg>Fixed UnexpectedLex returning wrong lex<commit_after>package sqlread

type Query struct {
	Columns []string
	Table   string
	Outfile *string
}

type QueryTree struct {
	Queries     []Query
	ShowTables  uint
	ShowColumns []string
	Quit        bool
}

type QueryParser struct {
	Tree QueryTree
}

func NewQueryParser() *QueryParser {
	return &QueryParser{
		Tree: QueryTree{
			Queries:     []Query{},
			ShowColumns: []string{},
			ShowTables:  0,
			Quit:        false,
		},
	}
}

func (q *QueryParser) ParseStart(p *Parser) parseState {
	for {
		c, ok := p.scan()
		if !ok {
			break
		}

		if isOfAny(c, TIntpSelect) {
			return q.parseSelect
		}

		if isOfAny(c, TIntpShowTables) {
			return q.parseShowTables
		}

		if isOfAny(c, TIntpShowColumns) {
			return q.parseShowColumns
		}

		if isOfAny(c, TIntpQuit) {
			return q.parseQuit
		}

		p.errorUnexpectedLex(c, TIntpSelect, TIntpShowTables, TIntpQuit)
		return nil
	}

	return nil
}

func (q *QueryParser) parseQuit(p *Parser) parseState {
	x, ok := p.scan()
	if !ok {
		p.errorUnexpectedEOF()
		return nil
	}
	if !isOfAny(x, TSemi) {
		p.errorUnexpectedLex(x, TSemi)
		return nil
	}

	q.Tree.Quit = true

	return q.ParseStart
}

func (q *QueryParser) parseShowTables(p *Parser) parseState {
	x, ok := p.scan()
	if !ok {
		p.errorUnexpectedEOF()
		return nil
	}
	if !isOfAny(x, TSemi) {
		p.errorUnexpectedLex(x, TSemi)
		return nil
	}

	q.Tree.ShowTables++

	return q.ParseStart
}

func (q *QueryParser) parseShowColumns(p *Parser) parseState {
	x, ok := p.scan()
	if !ok {
		p.errorUnexpectedEOF()
		return nil
	}
	if !isOfAny(x, TIntpFrom) {
		p.errorUnexpectedLex(x, TIntpFrom)
		return nil
	}

	y, ok := p.scan()
	if !ok {
		p.errorUnexpectedEOF()
		return nil
	}
	if !isOfAny(y, TIdentifier) {
		p.errorUnexpectedLex(y, TIdentifier)
		return nil
	}

	q.Tree.ShowColumns = append(q.Tree.ShowColumns, identValue(y))

	return q.ParseStart
}

func (q *QueryParser) parseSelect(p *Parser) parseState {
	qry := &Query{
		Columns: make([]string, 0),
	}

	for {
		i, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}

		if isOfAny(i, TIdentifier, TIntpStar) {
			value := i.Val
			if i.Type == TIdentifier {
				value = identValue(i)
			}

			qry.Columns = append(qry.Columns, value)

			j, ok := p.scan()
			if !ok {
				p.errorUnexpectedEOF()
				return nil
			}

			if j.Type == TComma {
				continue
			}

			if j.Type == TIntpFrom {
				return q.parseSelectFromBuilder(qry)
			}

			p.errorUnexpectedLex(i, TComma, TIntpFrom)
			return nil
		}

		p.errorUnexpectedLex(i, TIdentifier, TIntpStar)
		return nil
	}
}

func (q *QueryParser) parseSelectFromBuilder(qry *Query) parseState {
	return func(p *Parser) parseState {
		c, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}

		if c.Type != TIdentifier {
			p.errorUnexpectedLex(c, TIdentifier)
			return nil
		}

		qry.Table = identValue(c)

		d, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}

		if d.Type == TSemi {
			q.Tree.Queries = append(q.Tree.Queries, *qry)
			return nil
		}

		if d.Type == TIntpIntoOutfile {
			return q.parseSelectIntoOutfileBuilder(qry)
		}

		p.errorUnexpectedLex(d, TSemi, TIntpIntoOutfile)
		return nil
	}
}

func (q *QueryParser) parseSelectIntoOutfileBuilder(qry *Query) parseState {
	return func(p *Parser) parseState {
		c, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}

		if c.Type != TString {
			p.errorUnexpectedLex(c, TString)
			return nil
		}

		s, err := stringValue(c)
		if err != nil {
			p.err = err
			return nil
		}

		qry.Outfile = &s

		d, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}
		if d.Type == TSemi {
			q.Tree.Queries = append(q.Tree.Queries, *qry)
			return nil
		}

		p.errorUnexpectedLex(d, TSemi)
		return nil
	}
}

func identValue(c LexItem) string {
	// remove backticks - if/when we implement backtickless
	// identifiers this will need to be better handled.
	return c.Val[1 : len(c.Val)-1]
}
<|endoftext|>


<commit_before>package pop

// Query is the main value that is used to build up a query
// to be executed against the `Connection`.
type Query struct {
	RawSQL                  *clause
	limitResults            int
	whereClauses            clauses
	orderClauses            clauses
	fromClauses             fromClauses
	belongsToThroughClauses belongsToThroughClauses
	joinClauses             joinClauses
	Paginator               *Paginator
	Connection              *Connection
}

// RawQuery will override the query building feature of Pop and will use
// whatever query you want to execute against the `Connection`. You can continue
// to use the `?` argument syntax.
//
//	c.RawQuery("select * from foo where id = ?", 1)
func (c *Connection) RawQuery(stmt string, args ...interface{}) *Query {
	return Q(c).RawQuery(stmt, args...)
}

// RawQuery will override the query building feature of Pop and will use
// whatever query you want to execute against the `Connection`. You can continue
// to use the `?` argument syntax.
//
//	q.RawQuery("select * from foo where id = ?", 1)
func (q *Query) RawQuery(stmt string, args ...interface{}) *Query {
	q.RawSQL = &clause{stmt, args}
	return q
}

func (q *Query) Join(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"JOIN", table, as, on, args})
	return q
}

func (q *Query) LeftJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"LEFT JOIN", table, as, on, args})
	return q
}

func (q *Query) RightJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"RIGHT JOIN", table, as, on, args})
	return q
}

func (q *Query) LeftOuterJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"LEFT OUTER JOIN", table, as, on, args})
	return q
}

func (q *Query) RightOuterJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"RIGHT OUTER JOIN", table, as, on, args})
	return q
}

func (q *Query) LeftInnerJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"LEFT INNER JOIN", table, as, on, args})
	return q
}

func (q *Query) RightInnerJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"RIGHT INNER JOIN", table, as, on, args})
	return q
}

// Where will append a where clause to the query. You may use `?` in place of
// arguments.
//
// 	c.Where("id = ?", 1)
func (c *Connection) Where(stmt string, args ...interface{}) *Query {
	return Q(c).Where(stmt, args...)
}

// Where will append a where clause to the query. You may use `?` in place of
// arguments.
//
// 	q.Where("id = ?", 1)
func (q *Query) Where(stmt string, args ...interface{}) *Query {
	q.whereClauses = append(q.whereClauses, clause{stmt, args})
	return q
}

// Order will append an order clause to the query.
//
// 	c.Order("name desc")
func (c *Connection) Order(stmt string) *Query {
	return Q(c).Order(stmt)
}

// 	q.Order("name desc")
func (q *Query) Order(stmt string) *Query {
	q.orderClauses = append(q.orderClauses, clause{stmt, []interface{}{}})
	return q
}

// Limit will add a limit clause to the query.
func (c *Connection) Limit(limit int) *Query {
	return Q(c).Limit(limit)
}

// Limit will add a limit clause to the query.
func (q *Query) Limit(limit int) *Query {
	q.limitResults = limit
	return q
}

// Q will create a new "empty" query from the current connection.
func Q(c *Connection) *Query {
	return &Query{
		RawSQL:     &clause{},
		Connection: c,
	}
}

// ToSQL will generate SQL and the appropriate arguments for that SQL
// from the `Model` passed in.
func (q Query) ToSQL(model *Model, addColumns ...string) (string, []interface{}) {
	sb := q.toSQLBuilder(model, addColumns...)
	return sb.String(), sb.Args()
}

// ToSQLBuilder returns a new `SQLBuilder` that can be used to generate SQL,
// get arguments, and more.
func (q Query) toSQLBuilder(model *Model, addColumns ...string) *sqlBuilder {
	return newSQLBuilder(q, model, addColumns...)
}
<commit_msg>realign code<commit_after>package pop

// Query is the main value that is used to build up a query
// to be executed against the `Connection`.
type Query struct {
	RawSQL                  *clause
	limitResults            int
	whereClauses            clauses
	orderClauses            clauses
	fromClauses             fromClauses
	belongsToThroughClauses belongsToThroughClauses
	joinClauses             joinClauses
	Paginator               *Paginator
	Connection              *Connection
}

// RawQuery will override the query building feature of Pop and will use
// whatever query you want to execute against the `Connection`. You can continue
// to use the `?` argument syntax.
//
//	c.RawQuery("select * from foo where id = ?", 1)
func (c *Connection) RawQuery(stmt string, args ...interface{}) *Query {
	return Q(c).RawQuery(stmt, args...)
}

// RawQuery will override the query building feature of Pop and will use
// whatever query you want to execute against the `Connection`. You can continue
// to use the `?` argument syntax.
//
//	q.RawQuery("select * from foo where id = ?", 1)
func (q *Query) RawQuery(stmt string, args ...interface{}) *Query {
	q.RawSQL = &clause{stmt, args}
	return q
}

func (q *Query) Join(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"JOIN", table, as, on, args})
	return q
}

func (q *Query) LeftJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"LEFT JOIN", table, as, on, args})
	return q
}

func (q *Query) RightJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"RIGHT JOIN", table, as, on, args})
	return q
}

func (q *Query) LeftOuterJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"LEFT OUTER JOIN", table, as, on, args})
	return q
}

func (q *Query) RightOuterJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"RIGHT OUTER JOIN", table, as, on, args})
	return q
}

func (q *Query) LeftInnerJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"LEFT INNER JOIN", table, as, on, args})
	return q
}

func (q *Query) RightInnerJoin(table string, as string, on []string, args ...interface{}) *Query {
	q.joinClauses = append(q.joinClauses, joinClause{"RIGHT INNER JOIN", table, as, on, args})
	return q
}

// Where will append a where clause to the query. You may use `?` in place of
// arguments.
//
// 	c.Where("id = ?", 1)
func (c *Connection) Where(stmt string, args ...interface{}) *Query {
	return Q(c).Where(stmt, args...)
}

// Where will append a where clause to the query. You may use `?` in place of
// arguments.
//
// 	q.Where("id = ?", 1)
func (q *Query) Where(stmt string, args ...interface{}) *Query {
	q.whereClauses = append(q.whereClauses, clause{stmt, args})
	return q
}

// Order will append an order clause to the query.
//
// 	c.Order("name desc")
func (c *Connection) Order(stmt string) *Query {
	return Q(c).Order(stmt)
}

// Order will append an order clause to the query.
//
// 	q.Order("name desc")
func (q *Query) Order(stmt string) *Query {
	q.orderClauses = append(q.orderClauses, clause{stmt, []interface{}{}})
	return q
}

// Limit will add a limit clause to the query.
func (c *Connection) Limit(limit int) *Query {
	return Q(c).Limit(limit)
}

// Limit will add a limit clause to the query.
func (q *Query) Limit(limit int) *Query {
	q.limitResults = limit
	return q
}

// Q will create a new "empty" query from the current connection.
func Q(c *Connection) *Query {
	return &Query{
		RawSQL:     &clause{},
		Connection: c,
	}
}

// ToSQL will generate SQL and the appropriate arguments for that SQL
// from the `Model` passed in.
func (q Query) ToSQL(model *Model, addColumns ...string) (string, []interface{}) {
	sb := q.toSQLBuilder(model, addColumns...)
	return sb.String(), sb.Args()
}

// ToSQLBuilder returns a new `SQLBuilder` that can be used to generate SQL,
// get arguments, and more.
func (q Query) toSQLBuilder(model *Model, addColumns ...string) *sqlBuilder {
	return newSQLBuilder(q, model, addColumns...)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package picasa implements an importer for picasa.com accounts.
package picasa

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"

	"camlistore.org/pkg/blob"
	"camlistore.org/pkg/context"
	"camlistore.org/pkg/importer"
	"camlistore.org/pkg/schema"
	"camlistore.org/pkg/schema/nodeattr"
	"camlistore.org/pkg/syncutil"

	"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
	"camlistore.org/third_party/github.com/tgulacsi/picago"
)

const (
	apiURL   = "https://api.picasa.com/v2/"
	authURL  = "https://accounts.google.com/o/oauth2/auth"
	tokenURL = "https://accounts.google.com/o/oauth2/token"
	scopeURL = "https://picasaweb.google.com/data/"

	// runCompleteVersion is a cache-busting version number of the
	// importer code. It should be incremented whenever the
	// behavior of this importer is updated enough to warrant a
	// complete run.  Otherwise, if the importer runs to
	// completion, this version number is recorded on the account
	// permanode and subsequent importers can stop early.
	runCompleteVersion = "1"
)

func init() {
	importer.Register("picasa", newImporter())
}

var _ importer.ImporterSetupHTMLer = (*imp)(nil)

type imp struct {
	extendedOAuth2
}

var baseOAuthConfig = oauth.Config{
	AuthURL:  authURL,
	TokenURL: tokenURL,
	Scope:    scopeURL,

	// AccessType needs to be "offline", as the user is not here all the time;
	// ApprovalPrompt needs to be "force" to be able to get a RefreshToken
	// everytime, even for Re-logins, too.
	//
	// Source: https://developers.google.com/youtube/v3/guides/authentication#server-side-apps
	AccessType:     "offline",
	ApprovalPrompt: "force",
}

func newImporter() *imp {
	return &imp{
		newExtendedOAuth2(
			baseOAuthConfig,
			func(ctx *context.Context) (*userInfo, error) {
				u, err := picago.GetUser(ctx.HTTPClient(), "default")
				if err != nil {
					return nil, err
				}
				firstName, lastName := u.Name, ""
				i := strings.LastIndex(u.Name, " ")
				if i >= 0 {
					firstName, lastName = u.Name[:i], u.Name[i+1:]
				}
				return &userInfo{
					ID:        u.ID,
					FirstName: firstName,
					LastName:  lastName,
				}, nil
			}),
	}
}

func (*imp) AccountSetupHTML(host *importer.Host) string {
	// Picasa doesn't allow a path in the origin. Remove it.
	origin := host.ImporterBaseURL()
	if u, err := url.Parse(origin); err == nil {
		u.Path = ""
		origin = u.String()
	}

	callback := host.ImporterBaseURL() + "picasa/callback"
	return fmt.Sprintf(`

Configuring Picasa

Visit https://console.developers.google.com/
and click "Create Project".

Then under "APIs & Auth" in the left sidebar, click on "Credentials", then click the button "Create new Client ID".

Use the following settings:

Web application

Authorized JavaScript origins: %s

Authorized Redirect URI: %s

Click "Create Client ID".  Copy the "Client ID" and "Client Secret" into the boxes above.

`, origin, callback)
}

// A run is our state for a given run of the importer.
type run struct {
	*importer.RunContext
	im          *imp
	incremental bool // whether we've completed a run in the past
	photoGate   *syncutil.Gate

	mu     sync.Mutex // guards anyErr
	anyErr bool
}

func (r *run) errorf(format string, args ...interface{}) {
	log.Printf(format, args...)
	r.mu.Lock()
	defer r.mu.Unlock()
	r.anyErr = true
}

var forceFullImport, _ = strconv.ParseBool(os.Getenv("CAMLI_PICASA_FULL_IMPORT"))

func (im *imp) Run(ctx *importer.RunContext) error {
	clientId, secret, err := ctx.Credentials()
	if err != nil {
		return err
	}
	acctNode := ctx.AccountNode()
	ocfg := baseOAuthConfig
	ocfg.ClientId, ocfg.ClientSecret = clientId, secret
	token := decodeToken(acctNode.Attr(acctAttrOAuthToken))
	transport := &oauth.Transport{
		Config:    &ocfg,
		Token:     &token,
		Transport: notOAuthTransport(ctx.HTTPClient()),
	}
	ctx.Context = ctx.Context.New(context.WithHTTPClient(transport.Client()))

	root := ctx.RootNode()
	if root.Attr(nodeattr.Title) == "" {
		if err := root.SetAttr(nodeattr.Title,
			fmt.Sprintf("%s %s - Google/Picasa Photos",
				acctNode.Attr(importer.AcctAttrGivenName),
				acctNode.Attr(importer.AcctAttrFamilyName))); err != nil {
			return err
		}
	}

	r := &run{
		RunContext:  ctx,
		im:          im,
		incremental: !forceFullImport && acctNode.Attr(importer.AcctAttrCompletedVersion) == runCompleteVersion,
		photoGate:   syncutil.NewGate(3),
	}
	if err := r.importAlbums(); err != nil {
		return err
	}

	r.mu.Lock()
	anyErr := r.anyErr
	r.mu.Unlock()
	if !anyErr {
		if err := acctNode.SetAttrs(importer.AcctAttrCompletedVersion, runCompleteVersion); err != nil {
			return err
		}
	}

	return nil
}

func (r *run) importAlbums() error {
	albums, err := picago.GetAlbums(r.HTTPClient(), "default")
	if err != nil {
		return fmt.Errorf("importAlbums: error listing albums: %v", err)
	}
	albumsNode, err := r.getTopLevelNode("albums", "Albums")
	for _, album := range albums {
		if r.Context.IsCanceled() {
			return context.ErrCanceled
		}
		if err := r.importAlbum(albumsNode, album); err != nil {
			return fmt.Errorf("picasa importer: error importing album %s: %v", album, err)
		}
	}
	return nil
}

func (r *run) importAlbum(albumsNode *importer.Object, album picago.Album) (ret error) {
	if album.ID == "" {
		return errors.New("album has no ID")
	}
	albumNode, err := albumsNode.ChildPathObject(album.ID)
	if err != nil {
		return fmt.Errorf("importAlbum: error listing album: %v", err)
	}

	dateMod := schema.RFC3339FromTime(album.Updated)

	// Data reference: https://developers.google.com/picasa-web/docs/2.0/reference
	// TODO(tgulacsi): add more album info
	changes, err := albumNode.SetAttrs2(
		"picasaId", album.ID,
		nodeattr.Type, "picasaweb.google.com:album",
		nodeattr.Title, album.Title,
		nodeattr.DatePublished, schema.RFC3339FromTime(album.Published),
		nodeattr.LocationText, album.Location,
	)
	if err != nil {
		return fmt.Errorf("error setting album attributes: %v", err)
	}
	if !changes && r.incremental && albumNode.Attr(nodeattr.DateModified) == dateMod {
		return nil
	}
	defer func() {
		// Don't update DateModified on the album node until
		// we've successfully imported all the photos.
		if ret == nil {
			ret = albumNode.SetAttr(nodeattr.DateModified, dateMod)
		}
	}()

	log.Printf("Importing album %v: %v/%v (published %v, updated %v)", album.ID, album.Name, album.Title, album.Published, album.Updated)

	// TODO(bradfitz): GetPhotos does multiple HTTP requests to
	// return a slice of all photos. My "InstantUpload/Auto
	// Backup" album has 6678 photos (and growing) and this
	// currently takes like 40 seconds. Fix.
	photos, err := picago.GetPhotos(r.HTTPClient(), "default", album.ID)
	if err != nil {
		return err
	}

	log.Printf("Importing %d photos from album %q (%s)", len(photos), albumNode.Attr(nodeattr.Title),
		albumNode.PermanodeRef())

	var grp syncutil.Group
	for i := range photos {
		if r.Context.IsCanceled() {
			return context.ErrCanceled
		}
		photo := photos[i]
		r.photoGate.Start()
		grp.Go(func() error {
			defer r.photoGate.Done()
			return r.updatePhotoInAlbum(albumNode, photo)
		})
	}
	return grp.Err()
}

func (r *run) updatePhotoInAlbum(albumNode *importer.Object, photo picago.Photo) (ret error) {
	if photo.ID == "" {
		return errors.New("photo has no ID")
	}

	getMediaBytes := func() (io.ReadCloser, error) {
		log.Printf("Importing media from %v", photo.URL)
		resp, err := r.HTTPClient().Get(photo.URL)
		if err != nil {
			return nil, fmt.Errorf("importing photo %d: %v", photo.ID, err)
		}
		if resp.StatusCode != http.StatusOK {
			resp.Body.Close()
			return nil, fmt.Errorf("importing photo %d: status code = %d", photo.ID, resp.StatusCode)
		}
		return resp.Body, nil
	}

	var fileRefStr string
	idFilename := photo.ID + "-" + photo.Filename()
	photoNode, err := albumNode.ChildPathObjectOrFunc(idFilename, func() (*importer.Object, error) {
		h := blob.NewHash()
		rc, err := getMediaBytes()
		if err != nil {
			return nil, err
		}
		fileRef, err := schema.WriteFileFromReader(r.Host.Target(), photo.Filename(), io.TeeReader(rc, h))
		if err != nil {
			return nil, err
		}
		fileRefStr = fileRef.String()
		// TODO: do a search on r.Host to look for an existing
		// permanode that has that the wholeref of h, and no
		// conflicting attributes set (e.g. it's not a Flickr
		// photo with a different DatePublished or
		// whatnot). If it's just a boring file permanode,
		// then we'll re-use it
		return r.Host.NewObject()
	})
	if err != nil {
		return err
	}

	const attrMediaURL = "picasaMediaURL"
	if fileRefStr == "" {
		fileRefStr = photoNode.Attr(nodeattr.CamliContent)
		// Only re-download the source photo if its URL has changed.
		// Empirically this seems to work: cropping a photo in the
		// photos.google.com UI causes its URL to change. And it makes
		// sense, looking at the ugliness of the URLs with all their
		// encoded/signed state.
		if photoNode.Attr(attrMediaURL) != photo.URL {
			rc, err := getMediaBytes()
			if err != nil {
				return err
			}
			fileRef, err := schema.WriteFileFromReader(r.Host.Target(), photo.Filename(), rc)
			rc.Close()
			if err != nil {
				return err
			}
			fileRefStr = fileRef.String()
		}
	}

	// TODO(tgulacsi): add more attrs (comments ?)
	// for names, see http://schema.org/ImageObject and http://schema.org/CreativeWork
	attrs := []string{
		nodeattr.CamliContent, fileRefStr,
		"picasaId", photo.ID,
		nodeattr.Title, photo.Title,
		"caption", photo.Summary,
		nodeattr.Description, photo.Description,
		nodeattr.LocationText, photo.Location,
		nodeattr.DateModified, schema.RFC3339FromTime(photo.Updated),
		nodeattr.DatePublished, schema.RFC3339FromTime(photo.Published),
	}
	if photo.Latitude != 0 || photo.Longitude != 0 {
		attrs = append(attrs,
			nodeattr.Latitude, fmt.Sprintf("%f", photo.Latitude),
			nodeattr.Longitude, fmt.Sprintf("%f", photo.Longitude),
		)
	}
	if err := photoNode.SetAttrs(attrs...); err != nil {
		return err
	}
	if err := photoNode.SetAttrValues("tag", photo.Keywords); err != nil {
		return err
	}

	// Do this last, after we're sure the "camliContent" attribute
	// has been saved successfully, because this is the one that
	// causes us to do it again in the future or not.
	if err := photoNode.SetAttrs(attrMediaURL, photo.URL); err != nil {
		return err
	}
	return nil
}

func (r *run) getTopLevelNode(path string, title string) (*importer.Object, error) {
	childObject, err := r.RootNode().ChildPathObject(path)
	if err != nil {
		return nil, err
	}

	if err := childObject.SetAttr(nodeattr.Title, title); err != nil {
		return nil, err
	}
	return childObject, nil
}
<commit_msg>picasa: TODO about broken video support<commit_after>/*
Copyright 2014 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package picasa implements an importer for picasa.com accounts.
package picasa

// TODO: videos don't import correctly. it currently imports a still
// preview frame from the video, instead of the video bytes.

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"

	"camlistore.org/pkg/blob"
	"camlistore.org/pkg/context"
	"camlistore.org/pkg/importer"
	"camlistore.org/pkg/schema"
	"camlistore.org/pkg/schema/nodeattr"
	"camlistore.org/pkg/syncutil"

	"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
	"camlistore.org/third_party/github.com/tgulacsi/picago"
)

const (
	apiURL   = "https://api.picasa.com/v2/"
	authURL  = "https://accounts.google.com/o/oauth2/auth"
	tokenURL = "https://accounts.google.com/o/oauth2/token"
	scopeURL = "https://picasaweb.google.com/data/"

	// runCompleteVersion is a cache-busting version number of the
	// importer code. It should be incremented whenever the
	// behavior of this importer is updated enough to warrant a
	// complete run.  Otherwise, if the importer runs to
	// completion, this version number is recorded on the account
	// permanode and subsequent importers can stop early.
	runCompleteVersion = "1"
)

func init() {
	importer.Register("picasa", newImporter())
}

var _ importer.ImporterSetupHTMLer = (*imp)(nil)

type imp struct {
	extendedOAuth2
}

var baseOAuthConfig = oauth.Config{
	AuthURL:  authURL,
	TokenURL: tokenURL,
	Scope:    scopeURL,

	// AccessType needs to be "offline", as the user is not here all the time;
	// ApprovalPrompt needs to be "force" to be able to get a RefreshToken
	// everytime, even for Re-logins, too.
	//
	// Source: https://developers.google.com/youtube/v3/guides/authentication#server-side-apps
	AccessType:     "offline",
	ApprovalPrompt: "force",
}

func newImporter() *imp {
	return &imp{
		newExtendedOAuth2(
			baseOAuthConfig,
			func(ctx *context.Context) (*userInfo, error) {
				u, err := picago.GetUser(ctx.HTTPClient(), "default")
				if err != nil {
					return nil, err
				}
				firstName, lastName := u.Name, ""
				i := strings.LastIndex(u.Name, " ")
				if i >= 0 {
					firstName, lastName = u.Name[:i], u.Name[i+1:]
				}
				return &userInfo{
					ID:        u.ID,
					FirstName: firstName,
					LastName:  lastName,
				}, nil
			}),
	}
}

func (*imp) AccountSetupHTML(host *importer.Host) string {
	// Picasa doesn't allow a path in the origin. Remove it.
	origin := host.ImporterBaseURL()
	if u, err := url.Parse(origin); err == nil {
		u.Path = ""
		origin = u.String()
	}

	callback := host.ImporterBaseURL() + "picasa/callback"
	return fmt.Sprintf(`

Configuring Picasa

Visit https://console.developers.google.com/
and click "Create Project".

Then under "APIs & Auth" in the left sidebar, click on "Credentials", then click the button "Create new Client ID".

Use the following settings:

Web application

Authorized JavaScript origins: %s

Authorized Redirect URI: %s

Click "Create Client ID".  Copy the "Client ID" and "Client Secret" into the boxes above.

`, origin, callback)
}

// A run is our state for a given run of the importer.
type run struct {
	*importer.RunContext
	im          *imp
	incremental bool // whether we've completed a run in the past
	photoGate   *syncutil.Gate

	mu     sync.Mutex // guards anyErr
	anyErr bool
}

func (r *run) errorf(format string, args ...interface{}) {
	log.Printf(format, args...)
	r.mu.Lock()
	defer r.mu.Unlock()
	r.anyErr = true
}

var forceFullImport, _ = strconv.ParseBool(os.Getenv("CAMLI_PICASA_FULL_IMPORT"))

func (im *imp) Run(ctx *importer.RunContext) error {
	clientId, secret, err := ctx.Credentials()
	if err != nil {
		return err
	}
	acctNode := ctx.AccountNode()
	ocfg := baseOAuthConfig
	ocfg.ClientId, ocfg.ClientSecret = clientId, secret
	token := decodeToken(acctNode.Attr(acctAttrOAuthToken))
	transport := &oauth.Transport{
		Config:    &ocfg,
		Token:     &token,
		Transport: notOAuthTransport(ctx.HTTPClient()),
	}
	ctx.Context = ctx.Context.New(context.WithHTTPClient(transport.Client()))

	root := ctx.RootNode()
	if root.Attr(nodeattr.Title) == "" {
		if err := root.SetAttr(nodeattr.Title,
			fmt.Sprintf("%s %s - Google/Picasa Photos",
				acctNode.Attr(importer.AcctAttrGivenName),
				acctNode.Attr(importer.AcctAttrFamilyName))); err != nil {
			return err
		}
	}

	r := &run{
		RunContext:  ctx,
		im:          im,
		incremental: !forceFullImport && acctNode.Attr(importer.AcctAttrCompletedVersion) == runCompleteVersion,
		photoGate:   syncutil.NewGate(3),
	}
	if err := r.importAlbums(); err != nil {
		return err
	}

	r.mu.Lock()
	anyErr := r.anyErr
	r.mu.Unlock()
	if !anyErr {
		if err := acctNode.SetAttrs(importer.AcctAttrCompletedVersion, runCompleteVersion); err != nil {
			return err
		}
	}

	return nil
}

func (r *run) importAlbums() error {
	albums, err := picago.GetAlbums(r.HTTPClient(), "default")
	if err != nil {
		return fmt.Errorf("importAlbums: error listing albums: %v", err)
	}
	albumsNode, err := r.getTopLevelNode("albums", "Albums")
	for _, album := range albums {
		if r.Context.IsCanceled() {
			return context.ErrCanceled
		}
		if err := r.importAlbum(albumsNode, album); err != nil {
			return fmt.Errorf("picasa importer: error importing album %s: %v", album, err)
		}
	}
	return nil
}

func (r *run) importAlbum(albumsNode *importer.Object, album picago.Album) (ret error) {
	if album.ID == "" {
		return errors.New("album has no ID")
	}
	albumNode, err := albumsNode.ChildPathObject(album.ID)
	if err != nil {
		return fmt.Errorf("importAlbum: error listing album: %v", err)
	}

	dateMod := schema.RFC3339FromTime(album.Updated)

	// Data reference: https://developers.google.com/picasa-web/docs/2.0/reference
	// TODO(tgulacsi): add more album info
	changes, err := albumNode.SetAttrs2(
		"picasaId", album.ID,
		nodeattr.Type, "picasaweb.google.com:album",
		nodeattr.Title, album.Title,
		nodeattr.DatePublished, schema.RFC3339FromTime(album.Published),
		nodeattr.LocationText, album.Location,
	)
	if err != nil {
		return fmt.Errorf("error setting album attributes: %v", err)
	}
	if !changes && r.incremental && albumNode.Attr(nodeattr.DateModified) == dateMod {
		return nil
	}
	defer func() {
		// Don't update DateModified on the album node until
		// we've successfully imported all the photos.
		if ret == nil {
			ret = albumNode.SetAttr(nodeattr.DateModified, dateMod)
		}
	}()

	log.Printf("Importing album %v: %v/%v (published %v, updated %v)", album.ID, album.Name, album.Title, album.Published, album.Updated)

	// TODO(bradfitz): GetPhotos does multiple HTTP requests to
	// return a slice of all photos. My "InstantUpload/Auto
	// Backup" album has 6678 photos (and growing) and this
	// currently takes like 40 seconds. Fix.
	photos, err := picago.GetPhotos(r.HTTPClient(), "default", album.ID)
	if err != nil {
		return err
	}

	log.Printf("Importing %d photos from album %q (%s)", len(photos), albumNode.Attr(nodeattr.Title),
		albumNode.PermanodeRef())

	var grp syncutil.Group
	for i := range photos {
		if r.Context.IsCanceled() {
			return context.ErrCanceled
		}
		photo := photos[i]
		r.photoGate.Start()
		grp.Go(func() error {
			defer r.photoGate.Done()
			return r.updatePhotoInAlbum(albumNode, photo)
		})
	}
	return grp.Err()
}

func (r *run) updatePhotoInAlbum(albumNode *importer.Object, photo picago.Photo) (ret error) {
	if photo.ID == "" {
		return errors.New("photo has no ID")
	}

	getMediaBytes := func() (io.ReadCloser, error) {
		log.Printf("Importing media from %v", photo.URL)
		resp, err := r.HTTPClient().Get(photo.URL)
		if err != nil {
			return nil, fmt.Errorf("importing photo %d: %v", photo.ID, err)
		}
		if resp.StatusCode != http.StatusOK {
			resp.Body.Close()
			return nil, fmt.Errorf("importing photo %d: status code = %d", photo.ID, resp.StatusCode)
		}
		return resp.Body, nil
	}

	var fileRefStr string
	idFilename := photo.ID + "-" + photo.Filename()
	photoNode, err := albumNode.ChildPathObjectOrFunc(idFilename, func() (*importer.Object, error) {
		h := blob.NewHash()
		rc, err := getMediaBytes()
		if err != nil {
			return nil, err
		}
		fileRef, err := schema.WriteFileFromReader(r.Host.Target(), photo.Filename(), io.TeeReader(rc, h))
		if err != nil {
			return nil, err
		}
		fileRefStr = fileRef.String()
		// TODO: do a search on r.Host to look for an existing
		// permanode that has that the wholeref of h, and no
		// conflicting attributes set (e.g. it's not a Flickr
		// photo with a different DatePublished or
		// whatnot). If it's just a boring file permanode,
		// then we'll re-use it
		return r.Host.NewObject()
	})
	if err != nil {
		return err
	}

	const attrMediaURL = "picasaMediaURL"
	if fileRefStr == "" {
		fileRefStr = photoNode.Attr(nodeattr.CamliContent)
		// Only re-download the source photo if its URL has changed.
		// Empirically this seems to work: cropping a photo in the
		// photos.google.com UI causes its URL to change. And it makes
		// sense, looking at the ugliness of the URLs with all their
		// encoded/signed state.
		if photoNode.Attr(attrMediaURL) != photo.URL {
			rc, err := getMediaBytes()
			if err != nil {
				return err
			}
			fileRef, err := schema.WriteFileFromReader(r.Host.Target(), photo.Filename(), rc)
			rc.Close()
			if err != nil {
				return err
			}
			fileRefStr = fileRef.String()
		}
	}

	// TODO(tgulacsi): add more attrs (comments ?)
	// for names, see http://schema.org/ImageObject and http://schema.org/CreativeWork
	attrs := []string{
		nodeattr.CamliContent, fileRefStr,
		"picasaId", photo.ID,
		nodeattr.Title, photo.Title,
		"caption", photo.Summary,
		nodeattr.Description, photo.Description,
		nodeattr.LocationText, photo.Location,
		nodeattr.DateModified, schema.RFC3339FromTime(photo.Updated),
		nodeattr.DatePublished, schema.RFC3339FromTime(photo.Published),
	}
	if photo.Latitude != 0 || photo.Longitude != 0 {
		attrs = append(attrs,
			nodeattr.Latitude, fmt.Sprintf("%f", photo.Latitude),
			nodeattr.Longitude, fmt.Sprintf("%f", photo.Longitude),
		)
	}
	if err := photoNode.SetAttrs(attrs...); err != nil {
		return err
	}
	if err := photoNode.SetAttrValues("tag", photo.Keywords); err != nil {
		return err
	}

	// Do this last, after we're sure the "camliContent" attribute
	// has been saved successfully, because this is the one that
	// causes us to do it again in the future or not.
	if err := photoNode.SetAttrs(attrMediaURL, photo.URL); err != nil {
		return err
	}
	return nil
}

func (r *run) getTopLevelNode(path string, title string) (*importer.Object, error) {
	childObject, err := r.RootNode().ChildPathObject(path)
	if err != nil {
		return nil, err
	}

	if err := childObject.SetAttr(nodeattr.Title, title); err != nil {
		return nil, err
	}
	return childObject, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2012 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysql_test

import (
	"database/sql"
	"errors"
	"fmt"
	"os"
	"sync"
	"testing"

	"camlistore.org/pkg/index"
	"camlistore.org/pkg/index/indextest"
	"camlistore.org/pkg/index/mysql"
	"camlistore.org/pkg/test/testdep"

	_ "camlistore.org/third_party/github.com/ziutek/mymysql/godrv"
)

var (
	once        sync.Once
	dbAvailable bool
	rootdb      *sql.DB
)

func checkDB() {
	var err error
	if rootdb, err = sql.Open("mymysql", "mysql/root/root"); err == nil {
		var n int
		err := rootdb.QueryRow("SELECT COUNT(*) FROM user").Scan(&n)
		if err == nil {
			dbAvailable = true
		}
	}
}

func makeIndex() *index.Index {
	dbname := "camlitest_" + os.Getenv("USER")
	do(rootdb, "DROP DATABASE IF EXISTS "+dbname)
	do(rootdb, "CREATE DATABASE "+dbname)

	db, err := sql.Open("mymysql", dbname+"/root/root")
	if err != nil {
		panic("opening test database: " + err.Error())
	}
	for _, tableSql := range mysql.SQLCreateTables() {
		do(db, tableSql)
	}

	do(db, fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, mysql.SchemaVersion()))
	s, err := mysql.NewStorage("localhost", "root", "root", dbname)
	if err != nil {
		panic(err)
	}
	return index.New(s)
}

func do(db *sql.DB, sql string) {
	_, err := db.Exec(sql)
	if err == nil {
		return
	}
	panic(fmt.Sprintf("Error %v running SQL: %s", err, sql))
}

type mysqlTester struct{}

func (mysqlTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
	once.Do(checkDB)
	if !dbAvailable {
		// TODO(bradfitz): accept somehow other passwords than
		// 'root', and/or try localhost unix socket
		// connections rather than using TCP localhost?
		err := errors.New("Not running; start a MySQL daemon on the standard port (3306) with root password 'root'")
		testdep.CheckEnv(t)
		t.Fatalf("MySQL not available locally for testing: %v", err)
	}
	tfn(t, makeIndex)
}

func TestIndex_MySQL(t *testing.T) {
	mysqlTester{}.test(t, indextest.Index)
}

func TestPathsOfSignerTarget_MySQL(t *testing.T) {
	mysqlTester{}.test(t, indextest.PathsOfSignerTarget)
}

func TestFiles_MysQL(t *testing.T) {
	mysqlTester{}.test(t, indextest.Files)
}
<commit_msg>fix typo in test name<commit_after>/*
Copyright 2012 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysql_test

import (
	"database/sql"
	"errors"
	"fmt"
	"os"
	"sync"
	"testing"

	"camlistore.org/pkg/index"
	"camlistore.org/pkg/index/indextest"
	"camlistore.org/pkg/index/mysql"
	"camlistore.org/pkg/test/testdep"

	_ "camlistore.org/third_party/github.com/ziutek/mymysql/godrv"
)

var (
	once        sync.Once
	dbAvailable bool
	rootdb      *sql.DB
)

func checkDB() {
	var err error
	if rootdb, err = sql.Open("mymysql", "mysql/root/root"); err == nil {
		var n int
		err := rootdb.QueryRow("SELECT COUNT(*) FROM user").Scan(&n)
		if err == nil {
			dbAvailable = true
		}
	}
}

func makeIndex() *index.Index {
	dbname := "camlitest_" + os.Getenv("USER")
	do(rootdb, "DROP DATABASE IF EXISTS "+dbname)
	do(rootdb, "CREATE DATABASE "+dbname)

	db, err := sql.Open("mymysql", dbname+"/root/root")
	if err != nil {
		panic("opening test database: " + err.Error())
	}
	for _, tableSql := range mysql.SQLCreateTables() {
		do(db, tableSql)
	}

	do(db, fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, mysql.SchemaVersion()))
	s, err := mysql.NewStorage("localhost", "root", "root", dbname)
	if err != nil {
		panic(err)
	}
	return index.New(s)
}

func do(db *sql.DB, sql string) {
	_, err := db.Exec(sql)
	if err == nil {
		return
	}
	panic(fmt.Sprintf("Error %v running SQL: %s", err, sql))
}

type mysqlTester struct{}

func (mysqlTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
	once.Do(checkDB)
	if !dbAvailable {
		// TODO(bradfitz): accept somehow other passwords than
		// 'root', and/or try localhost unix socket
		// connections rather than using TCP localhost?
		err := errors.New("Not running; start a MySQL daemon on the standard port (3306) with root password 'root'")
		testdep.CheckEnv(t)
		t.Fatalf("MySQL not available locally for testing: %v", err)
	}
	tfn(t, makeIndex)
}

func TestIndex_MySQL(t *testing.T) {
	mysqlTester{}.test(t, indextest.Index)
}

func TestPathsOfSignerTarget_MySQL(t *testing.T) {
	mysqlTester{}.test(t, indextest.PathsOfSignerTarget)
}

func TestFiles_MySQL(t *testing.T) {
	mysqlTester{}.test(t, indextest.Files)
}
<|endoftext|>


<commit_before>package data

// CreditCardInfo contains credit card info
type CreditCardInfo struct {
	Display  string
	Patterns []uint
	Gaps     []uint
	Lengths  []uint
	Code     CreditCardCode
}

// CreditCardCode contains code type and size
type CreditCardCode struct {
	Name string
	Size uint
}

// CreditCardTypes is an array of credit card types
var CreditCardTypes = []string{"visa", "masterCreditcard", "american-express", "diners-club", "discover", "jcb", "unionpay", "maestro", "elo", "hiper", "hiperCreditcard"}

// CreditCards contains payment information
var CreditCards = map[string]CreditCardInfo{
	"visa": {
		Display:  "Visa",
		Patterns: []uint{4},
		Gaps:     []uint{4, 8, 12},
		Lengths:  []uint{16, 18, 19},
		Code: CreditCardCode{
			Name: "CVV",
			Size: 3,
		},
	},
	"masterCreditcard": {
		Display: "MasterCreditcard",
		Patterns: []uint{
			51, 55,
			2221, 2229,
			223, 229,
			23, 26,
			270, 271,
			2720,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{16},
		Code: CreditCardCode{
			Name: "CVC",
			Size: 3,
		},
	},
	"american-express": {
		Display:  "American Express",
		Patterns: []uint{34, 37},
		Gaps:     []uint{4, 10},
		Lengths:  []uint{15},
		Code: CreditCardCode{
			Name: "CID",
			Size: 4,
		},
	},
	"diners-club": {
		Display: "Diners Club",
		Patterns: []uint{
			300, 305,
			36, 38, 39,
		},
		Gaps:    []uint{4, 10},
		Lengths: []uint{14, 16, 19},
		Code: CreditCardCode{
			Name: "CVV",
			Size: 3,
		},
	},
	"discover": {
		Display: "Discover",
		Patterns: []uint{
			6011, 644, 649, 65,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{16, 19},
		Code: CreditCardCode{
			Name: "CID",
			Size: 3,
		},
	},
	"jcb": {
		Display: "JCB",
		Patterns: []uint{
			2131, 1800, 3528, 3589,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{16, 17, 18, 19},
		Code: CreditCardCode{
			Name: "CVV",
			Size: 3,
		},
	},
	"unionpay": {
		Display: "UnionPay",
		Patterns: []uint{
			620, 624, 626,
			62100, 62182,
			62184, 62187,
			62185, 62197,
			62200, 62205,
			622010, 622999,
			622018,
			622019, 622999,
			62207, 62209,
			622126, 622925,
			623, 626,
			6270, 6272, 6276,
			627700, 627779,
			627781, 627799,
			6282, 6289,
			6291, 6292,
			810,
			8110, 8131,
			8132, 8151,
			8152, 8163,
			8164, 817,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{14, 15, 16, 17, 18, 19},
		Code: CreditCardCode{
			Name: "CVN",
			Size: 3,
		},
	},
	"maestro": {
		Display: "Maestro",
		Patterns: []uint{
			493698,
			500000, 506698,
			506779, 508999,
			56, 59,
			6, 63, 67,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{12, 13, 14, 15, 16, 17, 18, 19},
		Code: CreditCardCode{
			Name: "CVC",
			Size: 3,
		},
	},
	"elo": {
		Display: "Elo",
		Patterns: []uint{
			401178, 401179,
			438935, 457631,
			457632, 431274,
			451416, 457393,
			504175, 506699,
			506778, 509000,
			509999, 627780,
			636297, 636368,
			650031, 650033,
			650035, 650051,
			650405, 650439,
			650485, 650538,
			650541, 650598,
			650700, 650718,
			650720, 650727,
			650901, 650978,
			651652, 651679,
			655000, 655019,
			655021, 65505,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{16},
		Code: CreditCardCode{
			Name: "CVE",
			Size: 3,
		},
	},
	"mir": {
		Display:  "Mir",
		Patterns: []uint{2200, 2204},
		Gaps:     []uint{4, 8, 12},
		Lengths:  []uint{16, 17, 18, 19},
		Code: CreditCardCode{
			Name: "CVP2",
			Size: 3,
		},
	},
	"hiper": {
		Display: "Hiper",
		Patterns: []uint{
			637095,
			637568,
			637599,
			637609,
			637612,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{16},
		Code: CreditCardCode{
			Name: "CVC",
			Size: 3,
		},
	},
	"hiperCreditcard": {
		Display:  "HiperCreditcard",
		Patterns: []uint{606282},
		Gaps:     []uint{4, 8, 12},
		Lengths:  []uint{16},
		Code: CreditCardCode{
			Name: "CVC",
			Size: 3,
		},
	},
}
<commit_msg>payment - fix credit card types<commit_after>package data

// CreditCardInfo contains credit card info
type CreditCardInfo struct {
	Display  string
	Patterns []uint
	Gaps     []uint
	Lengths  []uint
	Code     CreditCardCode
}

// CreditCardCode contains code type and size
type CreditCardCode struct {
	Name string
	Size uint
}

// CreditCardTypes is an array of credit card types
var CreditCardTypes = []string{"visa", "mastercard", "american-express", "diners-club", "discover", "jcb", "unionpay", "maestro", "elo", "hiper", "hipercard"}

// CreditCards contains payment information
var CreditCards = map[string]CreditCardInfo{
	"visa": {
		Display:  "Visa",
		Patterns: []uint{4},
		Gaps:     []uint{4, 8, 12},
		Lengths:  []uint{16, 18, 19},
		Code: CreditCardCode{
			Name: "CVV",
			Size: 3,
		},
	},
	"mastercard": {
		Display: "Mastercard",
		Patterns: []uint{
			51, 55,
			2221, 2229,
			223, 229,
			23, 26,
			270, 271,
			2720,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{16},
		Code: CreditCardCode{
			Name: "CVC",
			Size: 3,
		},
	},
	"american-express": {
		Display:  "American Express",
		Patterns: []uint{34, 37},
		Gaps:     []uint{4, 10},
		Lengths:  []uint{15},
		Code: CreditCardCode{
			Name: "CID",
			Size: 4,
		},
	},
	"diners-club": {
		Display: "Diners Club",
		Patterns: []uint{
			300, 305,
			36, 38, 39,
		},
		Gaps:    []uint{4, 10},
		Lengths: []uint{14, 16, 19},
		Code: CreditCardCode{
			Name: "CVV",
			Size: 3,
		},
	},
	"discover": {
		Display: "Discover",
		Patterns: []uint{
			6011, 644, 649, 65,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{16, 19},
		Code: CreditCardCode{
			Name: "CID",
			Size: 3,
		},
	},
	"jcb": {
		Display: "JCB",
		Patterns: []uint{
			2131, 1800, 3528, 3589,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{16, 17, 18, 19},
		Code: CreditCardCode{
			Name: "CVV",
			Size: 3,
		},
	},
	"unionpay": {
		Display: "UnionPay",
		Patterns: []uint{
			620, 624, 626,
			62100, 62182,
			62184, 62187,
			62185, 62197,
			62200, 62205,
			622010, 622999,
			622018,
			622019, 622999,
			62207, 62209,
			622126, 622925,
			623, 626,
			6270, 6272, 6276,
			627700, 627779,
			627781, 627799,
			6282, 6289,
			6291, 6292,
			810,
			8110, 8131,
			8132, 8151,
			8152, 8163,
			8164, 817,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{14, 15, 16, 17, 18, 19},
		Code: CreditCardCode{
			Name: "CVN",
			Size: 3,
		},
	},
	"maestro": {
		Display: "Maestro",
		Patterns: []uint{
			493698,
			500000, 506698,
			506779, 508999,
			56, 59,
			6, 63, 67,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{12, 13, 14, 15, 16, 17, 18, 19},
		Code: CreditCardCode{
			Name: "CVC",
			Size: 3,
		},
	},
	"elo": {
		Display: "Elo",
		Patterns: []uint{
			401178, 401179,
			438935, 457631,
			457632, 431274,
			451416, 457393,
			504175, 506699,
			506778, 509000,
			509999, 627780,
			636297, 636368,
			650031, 650033,
			650035, 650051,
			650405, 650439,
			650485, 650538,
			650541, 650598,
			650700, 650718,
			650720, 650727,
			650901, 650978,
			651652, 651679,
			655000, 655019,
			655021, 65505,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{16},
		Code: CreditCardCode{
			Name: "CVE",
			Size: 3,
		},
	},
	"mir": {
		Display:  "Mir",
		Patterns: []uint{2200, 2204},
		Gaps:     []uint{4, 8, 12},
		Lengths:  []uint{16, 17, 18, 19},
		Code: CreditCardCode{
			Name: "CVP2",
			Size: 3,
		},
	},
	"hiper": {
		Display: "Hiper",
		Patterns: []uint{
			637095,
			637568,
			637599,
			637609,
			637612,
		},
		Gaps:    []uint{4, 8, 12},
		Lengths: []uint{16},
		Code: CreditCardCode{
			Name: "CVC",
			Size: 3,
		},
	},
	"hipercard": {
		Display:  "Hipercard",
		Patterns: []uint{606282},
		Gaps:     []uint{4, 8, 12},
		Lengths:  []uint{16},
		Code: CreditCardCode{
			Name: "CVC",
			Size: 3,
		},
	},
}
<|endoftext|>


<commit_before>//
// Copyright 2017 Pedro Salgado
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package resque

import (
	"encoding/json"

	"github.com/go-redis/redis"
)

// Queue a job queue.
type Queue struct {
	Name  string
	Redis *redis.Client
}

// NewQueue links a job queue to a Redis instance.
func NewQueue(n string, c *redis.Client) *Queue {
	return &Queue{
		Name:  n,
		Redis: c,
	}
}

// Put places a job on the queue.
func (q *Queue) Put(j *Job) error {
	json, err := json.Marshal(j)
	if err != nil {
		return err
	}

	cmd := q.Redis.RPush(q.Name, json)
	return cmd.Err()
}

// Subscribe returns a queue subscriber.
func (q *Queue) Subscribe() (*redis.PubSub, error) {
	pubsub, err := q.Redis.Subscribe(q.Name)
	if err != nil {
		return nil, err
	}

	return pubsub, nil
}
<commit_msg>queue.go: overhaul struct.<commit_after>//
// Copyright 2017 Pedro Salgado
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package resque

import (
	"encoding/json"
	"fmt"

	"gopkg.in/redis.v5"
)

// Queue a job queue.
type Queue struct {
	redis        *redis.Client
	jobClassName string
	Name         string
}

// newQueue links a job queue to a redis instance.
func newQueue(jcn string, c *redis.Client) *Queue {
	return &Queue{
		redis:        c,
		jobClassName: jcn,
		Name:         fmt.Sprintf("resque:queue:%s", jcn),
	}
}

// Receive gets a job from the queue.
func (q Queue) Receive() (*Job, error) {
	cmd := q.redis.LPop(q.Name)
	json_str, err := cmd.Bytes()
	if err != nil {
		return nil, err
	}

	job := &Job{}
	err = json.Unmarshal(json_str, job)
	if err != nil {
		return nil, err
	}

	return job, err
}

// Send places a job on the queue.
func (q Queue) Send(args []JobArgument) error {
	json_str, err := json.Marshal(Job{Class: q.jobClassName})
	if err != nil {
		return err
	}

	return q.redis.RPush(q.Name, json_str).Err()
}
<|endoftext|>


<commit_before>package sort

// Quick sorts data in place using the Quicksort algorithm. The function returns
// two orderings, order1 and order2, such that order1[i] is the old position of
// of the new data[i], and order2[i] is the new position of of the old data[i].
//
// http://en.wikipedia.org/wiki/Quicksort
func Quick(data []float64) ([]uint, []uint) {
	n := uint(len(data))

	order := make([]uint, 2*n)

	for i := uint(0); i < n; i++ {
		order[i] = i
	}

	if n > 1 {
		quickSort(data, order, 0, int(n)-1)
	}

	for i := uint(0); i < n; i++ {
		order[n+order[i]] = i
	}

	return order[:n], order[n:]
}

func quickSort(data []float64, order []uint, left, right int) {
	i, j := left, right
	pivot := data[(left+right)/2]

	for i <= j {
		for data[i] < pivot {
			i++
		}

		for data[j] > pivot {
			j--
		}

		if i <= j {
			data[i], data[j] = data[j], data[i]
			order[i], order[j] = order[j], order[i]
			i++
			j--
		}
	}

	if left < j {
		quickSort(data, order, left, j)
	}

	if i < right {
		quickSort(data, order, i, right)
	}
}
<commit_msg>Fix a link to Wikipedia<commit_after>package sort

// Quick sorts data in place using the Quicksort algorithm. The function returns
// two orderings, order1 and order2, such that order1[i] is the old position of
// of the new data[i], and order2[i] is the new position of of the old data[i].
//
// https://en.wikipedia.org/wiki/Quicksort
func Quick(data []float64) ([]uint, []uint) {
	n := uint(len(data))

	order := make([]uint, 2*n)

	for i := uint(0); i < n; i++ {
		order[i] = i
	}

	if n > 1 {
		quickSort(data, order, 0, int(n)-1)
	}

	for i := uint(0); i < n; i++ {
		order[n+order[i]] = i
	}

	return order[:n], order[n:]
}

func quickSort(data []float64, order []uint, left, right int) {
	i, j := left, right
	pivot := data[(left+right)/2]

	for i <= j {
		for data[i] < pivot {
			i++
		}

		for data[j] > pivot {
			j--
		}

		if i <= j {
			data[i], data[j] = data[j], data[i]
			order[i], order[j] = order[j], order[i]
			i++
			j--
		}
	}

	if left < j {
		quickSort(data, order, left, j)
	}

	if i < right {
		quickSort(data, order, i, right)
	}
}
<|endoftext|>


<commit_before>//go:build freebsd && cgo
// +build freebsd,cgo

package system

import (
	"fmt"
	"unsafe"

	"golang.org/x/sys/unix"
)

// #include <unistd.h>
// #include <sys/vmmeter.h>
// #include <sys/sysctl.h>
// #include <vm/vm_param.h>
import "C"

func getMemInfo() (int64, int64, error) {
	data, err := unix.SysctlRaw("vm.vmtotal")
	if err != nil {
		return -1, -1, fmt.Errorf("can't get kernel info: %w", err)
	}
	if len(data) != C.sizeof_struct_vmtotal {
		return -1, -1, fmt.Errorf("unexpected vmtotal size %d", len(data))
	}

	total := (*C.struct_vmtotal)(unsafe.Pointer(&data[0]))

	pagesize := int64(C.sysconf(C._SC_PAGESIZE))
	npages := int64(C.sysconf(C._SC_PHYS_PAGES))
	return pagesize * npages, pagesize * int64(total.t_free), nil
}

func getSwapInfo() (int64, int64, error) {
	var (
		total int64 = 0
		used  int64 = 0
	)
	swapCount, err := unix.SysctlUint32("vm.nswapdev")
	if err != nil {
		return -1, -1, fmt.Errorf("reading vm.nswapdev: %w", err)
	}
	for i := 0; i < int(swapCount); i++ {
		data, err := unix.SysctlRaw("vm.swap_info", i)
		if err != nil {
			return -1, -1, fmt.Errorf("reading vm.swap_info.%d: %w", i, err)
		}
		if len(data) != C.sizeof_struct_xswdev {
			return -1, -1, fmt.Errorf("unexpected swap_info size %d", len(data))
		}
		xsw := (*C.struct_xswdev)(unsafe.Pointer(&data[0]))
		total += int64(xsw.xsw_nblks)
		used += int64(xsw.xsw_used)
	}
	pagesize := int64(C.sysconf(C._SC_PAGESIZE))
	return pagesize * total, pagesize * (total - used), nil
}

// ReadMemInfo retrieves memory statistics of the host system and returns a
//  MemInfo type.
func ReadMemInfo() (*MemInfo, error) {
	MemTotal, MemFree, err := getMemInfo()
	if err != nil {
		return nil, fmt.Errorf("getting memory totals %w", err)
	}
	SwapTotal, SwapFree, err := getSwapInfo()
	if err != nil {
		return nil, fmt.Errorf("getting swap totals %w", err)
	}

	if MemTotal < 0 || MemFree < 0 || SwapTotal < 0 || SwapFree < 0 {
		return nil, fmt.Errorf("getting system memory info %w", err)
	}

	meminfo := &MemInfo{}
	// Total memory is total physical memory less than memory locked by kernel
	meminfo.MemTotal = MemTotal
	meminfo.MemFree = MemFree
	meminfo.SwapTotal = SwapTotal
	meminfo.SwapFree = SwapFree

	return meminfo, nil
}
<commit_msg>ReadMemInfo(): don't wrap a `nil` error<commit_after>//go:build freebsd && cgo
// +build freebsd,cgo

package system

import (
	"fmt"
	"unsafe"

	"golang.org/x/sys/unix"
)

// #include <unistd.h>
// #include <sys/vmmeter.h>
// #include <sys/sysctl.h>
// #include <vm/vm_param.h>
import "C"

func getMemInfo() (int64, int64, error) {
	data, err := unix.SysctlRaw("vm.vmtotal")
	if err != nil {
		return -1, -1, fmt.Errorf("can't get kernel info: %w", err)
	}
	if len(data) != C.sizeof_struct_vmtotal {
		return -1, -1, fmt.Errorf("unexpected vmtotal size %d", len(data))
	}

	total := (*C.struct_vmtotal)(unsafe.Pointer(&data[0]))

	pagesize := int64(C.sysconf(C._SC_PAGESIZE))
	npages := int64(C.sysconf(C._SC_PHYS_PAGES))
	return pagesize * npages, pagesize * int64(total.t_free), nil
}

func getSwapInfo() (int64, int64, error) {
	var (
		total int64 = 0
		used  int64 = 0
	)
	swapCount, err := unix.SysctlUint32("vm.nswapdev")
	if err != nil {
		return -1, -1, fmt.Errorf("reading vm.nswapdev: %w", err)
	}
	for i := 0; i < int(swapCount); i++ {
		data, err := unix.SysctlRaw("vm.swap_info", i)
		if err != nil {
			return -1, -1, fmt.Errorf("reading vm.swap_info.%d: %w", i, err)
		}
		if len(data) != C.sizeof_struct_xswdev {
			return -1, -1, fmt.Errorf("unexpected swap_info size %d", len(data))
		}
		xsw := (*C.struct_xswdev)(unsafe.Pointer(&data[0]))
		total += int64(xsw.xsw_nblks)
		used += int64(xsw.xsw_used)
	}
	pagesize := int64(C.sysconf(C._SC_PAGESIZE))
	return pagesize * total, pagesize * (total - used), nil
}

// ReadMemInfo retrieves memory statistics of the host system and returns a
//  MemInfo type.
func ReadMemInfo() (*MemInfo, error) {
	MemTotal, MemFree, err := getMemInfo()
	if err != nil {
		return nil, fmt.Errorf("getting memory totals %w", err)
	}
	SwapTotal, SwapFree, err := getSwapInfo()
	if err != nil {
		return nil, fmt.Errorf("getting swap totals %w", err)
	}

	if MemTotal < 0 || MemFree < 0 || SwapTotal < 0 || SwapFree < 0 {
		return nil, errors.New("getting system memory info")
	}

	meminfo := &MemInfo{}
	// Total memory is total physical memory less than memory locked by kernel
	meminfo.MemTotal = MemTotal
	meminfo.MemFree = MemFree
	meminfo.SwapTotal = SwapTotal
	meminfo.SwapFree = SwapFree

	return meminfo, nil
}
<|endoftext|>


<commit_before>package exec

import (
	"archive/tar"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"time"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/globals"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/vfs"
	"github.com/cozy/cozy-stack/pkg/workers/mails"

	"github.com/spf13/afero"
)

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

type konnectorWorker struct {
	slug     string
	msg      map[string]interface{}
	man      *apps.KonnManifest
	messages []konnectorMsg
}

// konnectorResult stores the result of a konnector execution.
type konnectorResult struct {
	DocID       string    `json:"_id,omitempty"`
	DocRev      string    `json:"_rev,omitempty"`
	CreatedAt   time.Time `json:"last_execution"`
	LastSuccess time.Time `json:"last_success"`
	Account     string    `json:"account"`
	State       string    `json:"state"`
	Error       string    `json:"error"`
}

func (r *konnectorResult) ID() string         { return r.DocID }
func (r *konnectorResult) Rev() string        { return r.DocRev }
func (r *konnectorResult) DocType() string    { return consts.KonnectorResults }
func (r *konnectorResult) Clone() couchdb.Doc { c := *r; return &c }
func (r *konnectorResult) SetID(id string)    { r.DocID = id }
func (r *konnectorResult) SetRev(rev string)  { r.DocRev = rev }

const (
	konnectorMsgTypeDebug    = "debug"
	konnectorMsgTypeWarning  = "warning"
	konnectorMsgTypeError    = "error"
	konnectorMsgTypeCritical = "critical"
)

// const konnectorMsgTypeProgress string = "progress"

func (w *konnectorWorker) PrepareWorkDir(ctx *jobs.WorkerContext, i *instance.Instance) (workDir string, err error) {
	var msg map[string]interface{}
	if err = ctx.UnmarshalMessage(&msg); err != nil {
		return
	}

	slug, _ := msg["konnector"].(string)
	man, err := apps.GetKonnectorBySlug(i, slug)
	if err != nil {
		return
	}

	// TODO: disallow konnectors on state Installed to be run when we define our
	// workflow to accept permissions changes on konnectors.
	if s := man.State(); s != apps.Ready && s != apps.Installed {
		err = errors.New("Konnector is not ready")
		return
	}

	w.slug = slug
	w.msg = msg
	w.man = man

	osFS := afero.NewOsFs()
	workDir, err = afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return
	}
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := i.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return
	}

	// Create the folder in which the konnector has the right to write.
	{
		fs := i.VFS()
		folderToSave, _ := msg["folder_to_save"].(string)
		defaultFolderPath, _ := msg["default_folder_path"].(string)
		if folderToSave != "" {
			if defaultFolderPath == "" {
				name := i.Translate("Tree Administrative")
				defaultFolderPath = fmt.Sprintf("/%s/%s", name, slug)
			}
			if _, err = fs.DirByID(folderToSave); os.IsNotExist(err) {
				folderToSave = ""
			}
		}
		if folderToSave == "" {
			var dir *vfs.DirDoc
			dir, err = vfs.MkdirAll(fs, defaultFolderPath, nil)
			if err != nil {
				return
			}
			msg["folder_to_save"] = dir.ID()
		}
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, 0640)
		if err != nil {
			return
		}
		_, err = io.Copy(f, tr)
		errc := f.Close()
		if err != nil {
			return
		}
		if errc != nil {
			err = errc
			return
		}
	}

	return workDir, nil
}

func (w *konnectorWorker) PrepareCmdEnv(ctx *jobs.WorkerContext, i *instance.Instance) (cmd string, env []string, jobID string, err error) {
	jobID = fmt.Sprintf("konnector/%s/%s", w.slug, i.Domain)

	// Directly pass the job message as fields parameters
	fieldsJSON, err := json.Marshal(w.msg)
	if err != nil {
		return
	}

	paramsJSON, err := json.Marshal(w.man.Parameters)
	if err != nil {
		return
	}

	token := i.BuildKonnectorToken(w.man)

	cmd = config.GetConfig().Konnectors.Cmd
	env = []string{
		"COZY_URL=" + i.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_PARAMETERS=" + string(paramsJSON),
		"COZY_TYPE=" + w.man.Type,
		"COZY_LOCALE=" + i.Locale,
		"COZY_JOB_ID=" + jobID,
	}
	return
}

func (w *konnectorWorker) ScanOuput(ctx *jobs.WorkerContext, i *instance.Instance, line []byte) error {
	var msg konnectorMsg
	if err := json.Unmarshal(line, &msg); err != nil {
		return fmt.Errorf("Could not parse stdout as JSON: %q", string(line))
	}

	switch msg.Type {
	case konnectorMsgTypeDebug:
		ctx.Logger().Debug(msg.Message)
	case konnectorMsgTypeWarning:
		ctx.Logger().Warn(msg.Message)
	case konnectorMsgTypeError, konnectorMsgTypeCritical:
		ctx.Logger().Error(msg.Message)
	}

	w.messages = append(w.messages, msg)
	realtime.GetHub().Publish(&realtime.Event{
		Verb: realtime.EventCreate,
		Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
			"type":    msg.Type,
			"message": msg.Message,
		}},
		Domain: i.Domain,
	})
	return nil
}

func (w *konnectorWorker) Error(i *instance.Instance, err error) error {
	// For retro-compatibility, we still use "error" logs as returned error, only
	// in the case that no "critical" message are actually returned. In such
	// case, We use the last "error" log as the returned error.
	var lastErrorMessage error
	for _, msg := range w.messages {
		if msg.Type == konnectorMsgTypeCritical {
			return errors.New(msg.Message)
		}
		if msg.Type == konnectorMsgTypeError {
			lastErrorMessage = errors.New(msg.Message)
		}
	}
	if lastErrorMessage != nil {
		return lastErrorMessage
	}

	return err
}

func (w *konnectorWorker) Commit(ctx *jobs.WorkerContext, errjob error) error {
	if w.msg == nil {
		return nil
	}

	accountID, _ := w.msg["account"].(string)
	domain := ctx.Domain()

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &konnectorResult{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, w.slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	var lastSuccess time.Time
	if errjob != nil {
		if lastResult != nil {
			lastSuccess = lastResult.LastSuccess
		}
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		lastSuccess = time.Now()
		state = jobs.Done
	}

	result := &konnectorResult{
		DocID:       w.slug,
		Account:     accountID,
		CreatedAt:   time.Now(),
		LastSuccess: lastSuccess,
		State:       state,
		Error:       errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}

	if err != nil {
		return err
	}

	// if it is the first try we do not take into account an error, we bail.
	if lastResult == nil {
		return nil
	}
	// if the job has not errored, or the last one was already errored, we bail.
	if state != jobs.Errored || lastResult.State == jobs.Errored {
		return nil
	}

	konnectorURL := inst.SubDomain(consts.CollectSlug)
	konnectorURL.Fragment = "/category/all/" + w.slug
	mail := mails.Options{
		Mode:         mails.ModeNoReply,
		Subject:      inst.Translate("Error Konnector execution", domain),
		TemplateName: "konnector_error_" + inst.Locale,
		TemplateValues: map[string]string{
			"KonnectorName": w.slug,
			"KonnectorPage": konnectorURL.String(),
		},
	}
	msg, err := jobs.NewMessage(&mail)
	if err != nil {
		return err
	}
	log := logger.WithDomain(domain)
	log.Info("Konnector has failed definitively, should send mail.", mail)
	_, err = globals.GetBroker().PushJob(&jobs.JobRequest{
		Domain:     domain,
		WorkerType: "sendmail",
		Message:    msg,
	})
	return err
}
<commit_msg>Add log when creating default folder for konnector fails<commit_after>package exec

import (
	"archive/tar"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"time"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/globals"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/vfs"
	"github.com/cozy/cozy-stack/pkg/workers/mails"

	"github.com/spf13/afero"
)

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

type konnectorWorker struct {
	slug     string
	msg      map[string]interface{}
	man      *apps.KonnManifest
	messages []konnectorMsg
}

// konnectorResult stores the result of a konnector execution.
type konnectorResult struct {
	DocID       string    `json:"_id,omitempty"`
	DocRev      string    `json:"_rev,omitempty"`
	CreatedAt   time.Time `json:"last_execution"`
	LastSuccess time.Time `json:"last_success"`
	Account     string    `json:"account"`
	State       string    `json:"state"`
	Error       string    `json:"error"`
}

func (r *konnectorResult) ID() string         { return r.DocID }
func (r *konnectorResult) Rev() string        { return r.DocRev }
func (r *konnectorResult) DocType() string    { return consts.KonnectorResults }
func (r *konnectorResult) Clone() couchdb.Doc { c := *r; return &c }
func (r *konnectorResult) SetID(id string)    { r.DocID = id }
func (r *konnectorResult) SetRev(rev string)  { r.DocRev = rev }

const (
	konnectorMsgTypeDebug    = "debug"
	konnectorMsgTypeWarning  = "warning"
	konnectorMsgTypeError    = "error"
	konnectorMsgTypeCritical = "critical"
)

// const konnectorMsgTypeProgress string = "progress"

func (w *konnectorWorker) PrepareWorkDir(ctx *jobs.WorkerContext, i *instance.Instance) (workDir string, err error) {
	var msg map[string]interface{}
	if err = ctx.UnmarshalMessage(&msg); err != nil {
		return
	}

	slug, _ := msg["konnector"].(string)
	man, err := apps.GetKonnectorBySlug(i, slug)
	if err != nil {
		return
	}

	// TODO: disallow konnectors on state Installed to be run when we define our
	// workflow to accept permissions changes on konnectors.
	if s := man.State(); s != apps.Ready && s != apps.Installed {
		err = errors.New("Konnector is not ready")
		return
	}

	w.slug = slug
	w.msg = msg
	w.man = man

	osFS := afero.NewOsFs()
	workDir, err = afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return
	}
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := i.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return
	}

	// Create the folder in which the konnector has the right to write.
	{
		fs := i.VFS()
		folderToSave, _ := msg["folder_to_save"].(string)
		defaultFolderPath, _ := msg["default_folder_path"].(string)
		if folderToSave != "" {
			if defaultFolderPath == "" {
				name := i.Translate("Tree Administrative")
				defaultFolderPath = fmt.Sprintf("/%s/%s", name, slug)
			}
			if _, err = fs.DirByID(folderToSave); os.IsNotExist(err) {
				folderToSave = ""
			}
		}
		if folderToSave == "" {
			var dir *vfs.DirDoc
			dir, err = vfs.MkdirAll(fs, defaultFolderPath, nil)
			if err != nil {
				log := logger.WithDomain(i.Domain)
				log.Warnf("Can't create the default folder %s for konnector %s: %s", defaultFolderPath, slug, err)
				return
			}
			msg["folder_to_save"] = dir.ID()
		}
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, 0640)
		if err != nil {
			return
		}
		_, err = io.Copy(f, tr)
		errc := f.Close()
		if err != nil {
			return
		}
		if errc != nil {
			err = errc
			return
		}
	}

	return workDir, nil
}

func (w *konnectorWorker) PrepareCmdEnv(ctx *jobs.WorkerContext, i *instance.Instance) (cmd string, env []string, jobID string, err error) {
	jobID = fmt.Sprintf("konnector/%s/%s", w.slug, i.Domain)

	// Directly pass the job message as fields parameters
	fieldsJSON, err := json.Marshal(w.msg)
	if err != nil {
		return
	}

	paramsJSON, err := json.Marshal(w.man.Parameters)
	if err != nil {
		return
	}

	token := i.BuildKonnectorToken(w.man)

	cmd = config.GetConfig().Konnectors.Cmd
	env = []string{
		"COZY_URL=" + i.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_PARAMETERS=" + string(paramsJSON),
		"COZY_TYPE=" + w.man.Type,
		"COZY_LOCALE=" + i.Locale,
		"COZY_JOB_ID=" + jobID,
	}
	return
}

func (w *konnectorWorker) ScanOuput(ctx *jobs.WorkerContext, i *instance.Instance, line []byte) error {
	var msg konnectorMsg
	if err := json.Unmarshal(line, &msg); err != nil {
		return fmt.Errorf("Could not parse stdout as JSON: %q", string(line))
	}

	switch msg.Type {
	case konnectorMsgTypeDebug:
		ctx.Logger().Debug(msg.Message)
	case konnectorMsgTypeWarning:
		ctx.Logger().Warn(msg.Message)
	case konnectorMsgTypeError, konnectorMsgTypeCritical:
		ctx.Logger().Error(msg.Message)
	}

	w.messages = append(w.messages, msg)
	realtime.GetHub().Publish(&realtime.Event{
		Verb: realtime.EventCreate,
		Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
			"type":    msg.Type,
			"message": msg.Message,
		}},
		Domain: i.Domain,
	})
	return nil
}

func (w *konnectorWorker) Error(i *instance.Instance, err error) error {
	// For retro-compatibility, we still use "error" logs as returned error, only
	// in the case that no "critical" message are actually returned. In such
	// case, We use the last "error" log as the returned error.
	var lastErrorMessage error
	for _, msg := range w.messages {
		if msg.Type == konnectorMsgTypeCritical {
			return errors.New(msg.Message)
		}
		if msg.Type == konnectorMsgTypeError {
			lastErrorMessage = errors.New(msg.Message)
		}
	}
	if lastErrorMessage != nil {
		return lastErrorMessage
	}

	return err
}

func (w *konnectorWorker) Commit(ctx *jobs.WorkerContext, errjob error) error {
	if w.msg == nil {
		return nil
	}

	accountID, _ := w.msg["account"].(string)
	domain := ctx.Domain()

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &konnectorResult{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, w.slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	var lastSuccess time.Time
	if errjob != nil {
		if lastResult != nil {
			lastSuccess = lastResult.LastSuccess
		}
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		lastSuccess = time.Now()
		state = jobs.Done
	}

	result := &konnectorResult{
		DocID:       w.slug,
		Account:     accountID,
		CreatedAt:   time.Now(),
		LastSuccess: lastSuccess,
		State:       state,
		Error:       errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}

	if err != nil {
		return err
	}

	// if it is the first try we do not take into account an error, we bail.
	if lastResult == nil {
		return nil
	}
	// if the job has not errored, or the last one was already errored, we bail.
	if state != jobs.Errored || lastResult.State == jobs.Errored {
		return nil
	}

	konnectorURL := inst.SubDomain(consts.CollectSlug)
	konnectorURL.Fragment = "/category/all/" + w.slug
	mail := mails.Options{
		Mode:         mails.ModeNoReply,
		Subject:      inst.Translate("Error Konnector execution", domain),
		TemplateName: "konnector_error_" + inst.Locale,
		TemplateValues: map[string]string{
			"KonnectorName": w.slug,
			"KonnectorPage": konnectorURL.String(),
		},
	}
	msg, err := jobs.NewMessage(&mail)
	if err != nil {
		return err
	}
	log := logger.WithDomain(domain)
	log.Info("Konnector has failed definitively, should send mail.", mail)
	_, err = globals.GetBroker().PushJob(&jobs.JobRequest{
		Domain:     domain,
		WorkerType: "sendmail",
		Message:    msg,
	})
	return err
}
<|endoftext|>