<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package driver defines interfaces to be implemented by database
// drivers as used by package sql.
//
// Most code should use package sql.
package driver

import "errors"

// A driver Value is a value that drivers must be able to handle.
// A Value is either nil or an instance of one of these types:
//
//   int64
//   float64
//   bool
//   []byte
//   string   [*] everywhere except from Rows.Next.
//   time.Time
type Value interface{}

// Driver is the interface that must be implemented by a database
// driver.
type Driver interface {
	// Open returns a new connection to the database.
	// The name is a string in a driver-specific format.
	//
	// Open may return a cached connection (one previously
	// closed), but doing so is unnecessary; the sql package
	// maintains a pool of idle connections for efficient re-use.
	//
	// The returned connection is only used by one goroutine at a
	// time.
	Open(name string) (Conn, error)
}

// ErrSkip may be returned by some optional interfaces' methods to
// indicate at runtime that the fast path is unavailable and the sql
// package should continue as if the optional interface was not
// implemented. ErrSkip is only supported where explicitly
// documented.
var ErrSkip = errors.New("driver: skip fast-path; continue as if unimplemented")

// ErrBadConn should be returned by a driver to signal to the sql
// package that a driver.Conn is in a bad state (such as the server
// having earlier closed the connection) and the sql package should
// retry on a new connection.
//
// To prevent duplicate operations, ErrBadConn should NOT be returned
// if there's a possibility that the database server might have
// performed the operation. Even if the server sends back an error,
// you shouldn't return ErrBadConn.
var ErrBadConn = errors.New("driver: bad connection")

// Execer is an optional interface that may be implemented by a Conn.
//
// If a Conn does not implement Execer, the sql package's DB.Exec will
// first prepare a query, execute the statement, and then close the
// statement.
//
// Exec may return ErrSkip.
type Execer interface {
	Exec(query string, args []Value) (Result, error)
}

// Queryer is an optional interface that may be implemented by a Conn.
//
// If a Conn does not implement Queryer, the sql package's DB.Query will
// first prepare a query, execute the statement, and then close the
// statement.
//
// Query may return ErrSkip.
type Queryer interface {
	Query(query string, args []Value) (Rows, error)
}

// Conn is a connection to a database. It is not used concurrently
// by multiple goroutines.
//
// Conn is assumed to be stateful.
type Conn interface {
	// Prepare returns a prepared statement, bound to this connection.
	Prepare(query string) (Stmt, error)

	// Close invalidates and potentially stops any current
	// prepared statements and transactions, marking this
	// connection as no longer in use.
	//
	// Because the sql package maintains a free pool of
	// connections and only calls Close when there's a surplus of
	// idle connections, it shouldn't be necessary for drivers to
	// do their own connection caching.
	Close() error

	// Begin starts and returns a new transaction.
	Begin() (Tx, error)
}

// Result is the result of a query execution.
type Result interface {
	// LastInsertId returns the database's auto-generated ID
	// after, for example, an INSERT into a table with primary
	// key.
	LastInsertId() (int64, error)

	// RowsAffected returns the number of rows affected by the
	// query.
	RowsAffected() (int64, error)
}

// Stmt is a prepared statement. It is bound to a Conn and not
// used by multiple goroutines concurrently.
type Stmt interface {
	// Close closes the statement.
	//
	// As of Go 1.1, a Stmt will not be closed if it's in use
	// by any queries.
	Close() error

	// NumInput returns the number of placeholder parameters.
	//
	// If NumInput returns >= 0, the sql package will sanity check
	// argument counts from callers and return errors to the caller
	// before the statement's Exec or Query methods are called.
	//
	// NumInput may also return -1, if the driver doesn't know
	// its number of placeholders. In that case, the sql package
	// will not sanity check Exec or Query argument counts.
	NumInput() int

	// Exec executes a query that doesn't return rows, such
	// as an INSERT or UPDATE.
	Exec(args []Value) (Result, error)

	// Exec executes a query that may return rows, such as a
	// SELECT.
	Query(args []Value) (Rows, error)
}

// ColumnConverter may be optionally implemented by Stmt if the
// the statement is aware of its own columns' types and can
// convert from any type to a driver Value.
type ColumnConverter interface {
	// ColumnConverter returns a ValueConverter for the provided
	// column index.  If the type of a specific column isn't known
	// or shouldn't be handled specially, DefaultValueConverter
	// can be returned.
	ColumnConverter(idx int) ValueConverter
}

// Rows is an iterator over an executed query's results.
type Rows interface {
	// Columns returns the names of the columns. The number of
	// columns of the result is inferred from the length of the
	// slice.  If a particular column name isn't known, an empty
	// string should be returned for that entry.
	Columns() []string

	// Close closes the rows iterator.
	Close() error

	// Next is called to populate the next row of data into
	// the provided slice. The provided slice will be the same
	// size as the Columns() are wide.
	//
	// The dest slice may be populated only with
	// a driver Value type, but excluding string.
	// All string values must be converted to []byte.
	//
	// Next should return io.EOF when there are no more rows.
	Next(dest []Value) error
}

// Tx is a transaction.
type Tx interface {
	Commit() error
	Rollback() error
}

// RowsAffected implements Result for an INSERT or UPDATE operation
// which mutates a number of rows.
type RowsAffected int64

var _ Result = RowsAffected(0)

func (RowsAffected) LastInsertId() (int64, error) {
	return 0, errors.New("no LastInsertId available")
}

func (v RowsAffected) RowsAffected() (int64, error) {
	return int64(v), nil
}

// ResultNoRows is a pre-defined Result for drivers to return when a DDL
// command (such as a CREATE TABLE) succeeds. It returns an error for both
// LastInsertId and RowsAffected.
var ResultNoRows noRows

type noRows struct{}

var _ Result = noRows{}

func (noRows) LastInsertId() (int64, error) {
	return 0, errors.New("no LastInsertId available after DDL statement")
}

func (noRows) RowsAffected() (int64, error) {
	return 0, errors.New("no RowsAffected available after DDL statement")
}
<commit_msg>database/sql/driver: try to unstutter Value docs<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package driver defines interfaces to be implemented by database
// drivers as used by package sql.
//
// Most code should use package sql.
package driver

import "errors"

// Value is a value that drivers must be able to handle.
// It is either nil or an instance of one of these types:
//
//   int64
//   float64
//   bool
//   []byte
//   string   [*] everywhere except from Rows.Next.
//   time.Time
type Value interface{}

// Driver is the interface that must be implemented by a database
// driver.
type Driver interface {
	// Open returns a new connection to the database.
	// The name is a string in a driver-specific format.
	//
	// Open may return a cached connection (one previously
	// closed), but doing so is unnecessary; the sql package
	// maintains a pool of idle connections for efficient re-use.
	//
	// The returned connection is only used by one goroutine at a
	// time.
	Open(name string) (Conn, error)
}

// ErrSkip may be returned by some optional interfaces' methods to
// indicate at runtime that the fast path is unavailable and the sql
// package should continue as if the optional interface was not
// implemented. ErrSkip is only supported where explicitly
// documented.
var ErrSkip = errors.New("driver: skip fast-path; continue as if unimplemented")

// ErrBadConn should be returned by a driver to signal to the sql
// package that a driver.Conn is in a bad state (such as the server
// having earlier closed the connection) and the sql package should
// retry on a new connection.
//
// To prevent duplicate operations, ErrBadConn should NOT be returned
// if there's a possibility that the database server might have
// performed the operation. Even if the server sends back an error,
// you shouldn't return ErrBadConn.
var ErrBadConn = errors.New("driver: bad connection")

// Execer is an optional interface that may be implemented by a Conn.
//
// If a Conn does not implement Execer, the sql package's DB.Exec will
// first prepare a query, execute the statement, and then close the
// statement.
//
// Exec may return ErrSkip.
type Execer interface {
	Exec(query string, args []Value) (Result, error)
}

// Queryer is an optional interface that may be implemented by a Conn.
//
// If a Conn does not implement Queryer, the sql package's DB.Query will
// first prepare a query, execute the statement, and then close the
// statement.
//
// Query may return ErrSkip.
type Queryer interface {
	Query(query string, args []Value) (Rows, error)
}

// Conn is a connection to a database. It is not used concurrently
// by multiple goroutines.
//
// Conn is assumed to be stateful.
type Conn interface {
	// Prepare returns a prepared statement, bound to this connection.
	Prepare(query string) (Stmt, error)

	// Close invalidates and potentially stops any current
	// prepared statements and transactions, marking this
	// connection as no longer in use.
	//
	// Because the sql package maintains a free pool of
	// connections and only calls Close when there's a surplus of
	// idle connections, it shouldn't be necessary for drivers to
	// do their own connection caching.
	Close() error

	// Begin starts and returns a new transaction.
	Begin() (Tx, error)
}

// Result is the result of a query execution.
type Result interface {
	// LastInsertId returns the database's auto-generated ID
	// after, for example, an INSERT into a table with primary
	// key.
	LastInsertId() (int64, error)

	// RowsAffected returns the number of rows affected by the
	// query.
	RowsAffected() (int64, error)
}

// Stmt is a prepared statement. It is bound to a Conn and not
// used by multiple goroutines concurrently.
type Stmt interface {
	// Close closes the statement.
	//
	// As of Go 1.1, a Stmt will not be closed if it's in use
	// by any queries.
	Close() error

	// NumInput returns the number of placeholder parameters.
	//
	// If NumInput returns >= 0, the sql package will sanity check
	// argument counts from callers and return errors to the caller
	// before the statement's Exec or Query methods are called.
	//
	// NumInput may also return -1, if the driver doesn't know
	// its number of placeholders. In that case, the sql package
	// will not sanity check Exec or Query argument counts.
	NumInput() int

	// Exec executes a query that doesn't return rows, such
	// as an INSERT or UPDATE.
	Exec(args []Value) (Result, error)

	// Exec executes a query that may return rows, such as a
	// SELECT.
	Query(args []Value) (Rows, error)
}

// ColumnConverter may be optionally implemented by Stmt if the
// the statement is aware of its own columns' types and can
// convert from any type to a driver Value.
type ColumnConverter interface {
	// ColumnConverter returns a ValueConverter for the provided
	// column index.  If the type of a specific column isn't known
	// or shouldn't be handled specially, DefaultValueConverter
	// can be returned.
	ColumnConverter(idx int) ValueConverter
}

// Rows is an iterator over an executed query's results.
type Rows interface {
	// Columns returns the names of the columns. The number of
	// columns of the result is inferred from the length of the
	// slice.  If a particular column name isn't known, an empty
	// string should be returned for that entry.
	Columns() []string

	// Close closes the rows iterator.
	Close() error

	// Next is called to populate the next row of data into
	// the provided slice. The provided slice will be the same
	// size as the Columns() are wide.
	//
	// The dest slice may be populated only with
	// a driver Value type, but excluding string.
	// All string values must be converted to []byte.
	//
	// Next should return io.EOF when there are no more rows.
	Next(dest []Value) error
}

// Tx is a transaction.
type Tx interface {
	Commit() error
	Rollback() error
}

// RowsAffected implements Result for an INSERT or UPDATE operation
// which mutates a number of rows.
type RowsAffected int64

var _ Result = RowsAffected(0)

func (RowsAffected) LastInsertId() (int64, error) {
	return 0, errors.New("no LastInsertId available")
}

func (v RowsAffected) RowsAffected() (int64, error) {
	return int64(v), nil
}

// ResultNoRows is a pre-defined Result for drivers to return when a DDL
// command (such as a CREATE TABLE) succeeds. It returns an error for both
// LastInsertId and RowsAffected.
var ResultNoRows noRows

type noRows struct{}

var _ Result = noRows{}

func (noRows) LastInsertId() (int64, error) {
	return 0, errors.New("no LastInsertId available after DDL statement")
}

func (noRows) RowsAffected() (int64, error) {
	return 0, errors.New("no RowsAffected available after DDL statement")
}
<|endoftext|>


<commit_before>// Package bundle manages translations for multiple languages.
package bundle

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"sync"
	"unicode"

	"github.com/nicksnyder/go-i18n/i18n/language"
	"github.com/nicksnyder/go-i18n/i18n/translation"
	toml "github.com/pelletier/go-toml"
	"gopkg.in/yaml.v2"
)

// TranslateFunc is a copy of i18n.TranslateFunc to avoid a circular dependency.
type TranslateFunc func(translationID string, args ...interface{}) string

// Bundle stores the translations for multiple languages.
type Bundle struct {
	// The primary translations for a language tag and translation id.
	translations map[string]map[string]translation.Translation

	// Translations that can be used when an exact language match is not possible.
	fallbackTranslations map[string]map[string]translation.Translation

	sync.RWMutex
}

// New returns an empty bundle.
func New() *Bundle {
	return &Bundle{
		translations:         make(map[string]map[string]translation.Translation),
		fallbackTranslations: make(map[string]map[string]translation.Translation),
	}
}

// MustLoadTranslationFile is similar to LoadTranslationFile
// except it panics if an error happens.
func (b *Bundle) MustLoadTranslationFile(filename string) {
	if err := b.LoadTranslationFile(filename); err != nil {
		panic(err)
	}
}

// LoadTranslationFile loads the translations from filename into memory.
//
// The language that the translations are associated with is parsed from the filename (e.g. en-US.json).
//
// Generally you should load translation files once during your program's initialization.
func (b *Bundle) LoadTranslationFile(filename string) error {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}
	return b.ParseTranslationFileBytes(filename, buf)
}

// ParseTranslationFileBytes is similar to LoadTranslationFile except it parses the bytes in buf.
//
// It is useful for parsing translation files embedded with go-bindata.
func (b *Bundle) ParseTranslationFileBytes(filename string, buf []byte) error {
	basename := filepath.Base(filename)
	langs := language.Parse(basename)
	switch l := len(langs); {
	case l == 0:
		return fmt.Errorf("no language found in %q", basename)
	case l > 1:
		return fmt.Errorf("multiple languages found in filename %q: %v; expected one", basename, langs)
	}
	translations, err := parseTranslations(filename, buf)
	if err != nil {
		return err
	}
	b.AddTranslation(langs[0], translations...)
	return nil
}

func parseTranslations(filename string, buf []byte) ([]translation.Translation, error) {
	if len(buf) == 0 {
		return []translation.Translation{}, nil
	}

	ext := filepath.Ext(filename)

	// `github.com/pelletier/go-toml` has an Unmarshal function,
	// that can't unmarshal to maps, so we should parse TOML format separately.
	if ext == ".toml" {
		tree, err := toml.LoadReader(bytes.NewReader(buf))
		if err != nil {
			return nil, err
		}

		m := make(map[string]map[string]interface{})
		for k, v := range tree.ToMap() {
			m[k] = v.(map[string]interface{})
		}

		return parseFlatFormat(m)
	}

	// Then parse other formats.
	if isStandardFormat(ext, buf) {
		var standardFormat []map[string]interface{}
		if err := unmarshal(ext, buf, &standardFormat); err != nil {
			return nil, fmt.Errorf("failed to unmarshal %v: %v", filename, err)
		}
		return parseStandardFormat(standardFormat)
	} else {
		var flatFormat map[string]map[string]interface{}
		if err := unmarshal(ext, buf, &flatFormat); err != nil {
			return nil, fmt.Errorf("failed to unmarshal %v: %v", filename, err)
		}
		return parseFlatFormat(flatFormat)
	}
}

func isStandardFormat(ext string, buf []byte) bool {
	buf = deleteLeadingComments(ext, buf)
	firstRune := rune(buf[0])
	if (ext == ".json" && firstRune == '[') || (ext == ".yaml" && firstRune == '-') {
		return true
	}
	return false
}

// deleteLeadingComments deletes leading newlines and comments in buf.
// It only works for ext == ".yaml".
func deleteLeadingComments(ext string, buf []byte) []byte {
	if ext != ".yaml" {
		return buf
	}

	for {
		buf = bytes.TrimLeftFunc(buf, unicode.IsSpace)
		if buf[0] == '#' {
			buf = deleteLine(buf)
		} else {
			break
		}
	}

	return buf
}

func deleteLine(buf []byte) []byte {
	index := bytes.IndexRune(buf, '\n')
	if index == -1 { // If there is only one line without newline ...
		return nil // ... delete it and return nothing.
	}
	if index == len(buf)-1 { // If there is only one line with newline ...
		return nil // ... do the same as above.
	}
	return buf[index+1:]
}

// unmarshal finds an appropriate unmarshal function for ext
// (extension of filename) and unmarshals buf to out. out must be a pointer.
func unmarshal(ext string, buf []byte, out interface{}) error {
	switch ext {
	case ".json":
		return json.Unmarshal(buf, out)
	case ".yaml":
		return yaml.Unmarshal(buf, out)
	}

	return fmt.Errorf("unsupported file extension %v", ext)
}

func parseStandardFormat(data []map[string]interface{}) ([]translation.Translation, error) {
	translations := make([]translation.Translation, 0, len(data))
	for i, translationData := range data {
		t, err := translation.NewTranslation(translationData)
		if err != nil {
			return nil, fmt.Errorf("unable to parse translation #%d because %s\n%v", i, err, translationData)
		}
		translations = append(translations, t)
	}
	return translations, nil
}

// parseFlatFormat just converts data from flat format to standard format
// and passes it to parseStandardFormat.
//
// Flat format logic:
// key of data must be a string and data[key] must be always map[string]interface{},
// but if there is only "other" key in it then it is non-plural, else plural.
func parseFlatFormat(data map[string]map[string]interface{}) ([]translation.Translation, error) {
	var standardFormatData []map[string]interface{}
	for id, translationData := range data {
		dataObject := make(map[string]interface{})
		dataObject["id"] = id
		if len(translationData) == 1 { // non-plural form
			_, otherExists := translationData["other"]
			if otherExists {
				dataObject["translation"] = translationData["other"]
			}
		} else { // plural form
			dataObject["translation"] = translationData
		}

		standardFormatData = append(standardFormatData, dataObject)
	}

	return parseStandardFormat(standardFormatData)
}

// AddTranslation adds translations for a language.
//
// It is useful if your translations are in a format not supported by LoadTranslationFile.
func (b *Bundle) AddTranslation(lang *language.Language, translations ...translation.Translation) {
	b.Lock()
	defer b.Unlock()
	if b.translations[lang.Tag] == nil {
		b.translations[lang.Tag] = make(map[string]translation.Translation, len(translations))
	}
	currentTranslations := b.translations[lang.Tag]
	for _, newTranslation := range translations {
		if currentTranslation := currentTranslations[newTranslation.ID()]; currentTranslation != nil {
			currentTranslations[newTranslation.ID()] = currentTranslation.Merge(newTranslation)
		} else {
			currentTranslations[newTranslation.ID()] = newTranslation
		}
	}

	// lang can provide translations for less specific language tags.
	for _, tag := range lang.MatchingTags() {
		b.fallbackTranslations[tag] = currentTranslations
	}
}

// Translations returns all translations in the bundle.
func (b *Bundle) Translations() map[string]map[string]translation.Translation {
	t := make(map[string]map[string]translation.Translation)
	b.RLock()
	for tag, translations := range b.translations {
		t[tag] = make(map[string]translation.Translation)
		for id, translation := range translations {
			t[tag][id] = translation
		}
	}
	b.RUnlock()
	return t
}

// LanguageTags returns the tags of all languages that that have been added.
func (b *Bundle) LanguageTags() []string {
	var tags []string
	b.RLock()
	for k := range b.translations {
		tags = append(tags, k)
	}
	b.RUnlock()
	return tags
}

// LanguageTranslationIDs returns the ids of all translations that have been added for a given language.
func (b *Bundle) LanguageTranslationIDs(languageTag string) []string {
	var ids []string
	b.RLock()
	for id := range b.translations[languageTag] {
		ids = append(ids, id)
	}
	b.RUnlock()
	return ids
}

// MustTfunc is similar to Tfunc except it panics if an error happens.
func (b *Bundle) MustTfunc(pref string, prefs ...string) TranslateFunc {
	tfunc, err := b.Tfunc(pref, prefs...)
	if err != nil {
		panic(err)
	}
	return tfunc
}

// MustTfuncAndLanguage is similar to TfuncAndLanguage except it panics if an error happens.
func (b *Bundle) MustTfuncAndLanguage(pref string, prefs ...string) (TranslateFunc, *language.Language) {
	tfunc, language, err := b.TfuncAndLanguage(pref, prefs...)
	if err != nil {
		panic(err)
	}
	return tfunc, language
}

// Tfunc is similar to TfuncAndLanguage except is doesn't return the Language.
func (b *Bundle) Tfunc(pref string, prefs ...string) (TranslateFunc, error) {
	tfunc, _, err := b.TfuncAndLanguage(pref, prefs...)
	return tfunc, err
}

// TfuncAndLanguage returns a TranslateFunc for the first Language that
// has a non-zero number of translations in the bundle.
//
// The returned Language matches the the first language preference that could be satisfied,
// but this may not strictly match the language of the translations used to satisfy that preference.
//
// For example, the user may request "zh". If there are no translations for "zh" but there are translations
// for "zh-cn", then the translations for "zh-cn" will be used but the returned Language will be "zh".
//
// It can parse languages from Accept-Language headers (RFC 2616),
// but it assumes weights are monotonically decreasing.
func (b *Bundle) TfuncAndLanguage(pref string, prefs ...string) (TranslateFunc, *language.Language, error) {
	lang := b.supportedLanguage(pref, prefs...)
	var err error
	if lang == nil {
		err = fmt.Errorf("no supported languages found %#v", append(prefs, pref))
	}
	return func(translationID string, args ...interface{}) string {
		return b.translate(lang, translationID, args...)
	}, lang, err
}

// supportedLanguage returns the first language which
// has a non-zero number of translations in the bundle.
func (b *Bundle) supportedLanguage(pref string, prefs ...string) *language.Language {
	lang := b.translatedLanguage(pref)
	if lang == nil {
		for _, pref := range prefs {
			lang = b.translatedLanguage(pref)
			if lang != nil {
				break
			}
		}
	}
	return lang
}

func (b *Bundle) translatedLanguage(src string) *language.Language {
	langs := language.Parse(src)
	b.RLock()
	defer b.RUnlock()
	for _, lang := range langs {
		if len(b.translations[lang.Tag]) > 0 ||
			len(b.fallbackTranslations[lang.Tag]) > 0 {
			return lang
		}
	}
	return nil
}

func (b *Bundle) translate(lang *language.Language, translationID string, args ...interface{}) string {
	if lang == nil {
		return translationID
	}

	translation := b.translation(lang, translationID)
	if translation == nil {
		return translationID
	}

	var data interface{}
	var count interface{}
	if argc := len(args); argc > 0 {
		if isNumber(args[0]) {
			count = args[0]
			if argc > 1 {
				data = args[1]
			}
		} else {
			data = args[0]
		}
	}

	if count != nil {
		if data == nil {
			data = map[string]interface{}{"Count": count}
		} else {
			dataMap := toMap(data)
			dataMap["Count"] = count
			data = dataMap
		}
	} else {
		dataMap := toMap(data)
		if c, ok := dataMap["Count"]; ok {
			count = c
		}
	}

	p, _ := lang.Plural(count)
	template := translation.Template(p)
	if template == nil {
		return translationID
	}

	s := template.Execute(data)
	if s == "" {
		return translationID
	}
	return s
}

func (b *Bundle) translation(lang *language.Language, translationID string) translation.Translation {
	b.RLock()
	defer b.RUnlock()
	translations := b.translations[lang.Tag]
	if translations == nil {
		translations = b.fallbackTranslations[lang.Tag]
		if translations == nil {
			return nil
		}
	}
	return translations[translationID]
}

func isNumber(n interface{}) bool {
	switch n.(type) {
	case int, int8, int16, int32, int64, string:
		return true
	}
	return false
}

func toMap(input interface{}) map[string]interface{} {
	if data, ok := input.(map[string]interface{}); ok {
		return data
	}
	v := reflect.ValueOf(input)
	switch v.Kind() {
	case reflect.Ptr:
		return toMap(v.Elem().Interface())
	case reflect.Struct:
		return structToMap(v)
	default:
		return nil
	}
}

// Converts the top level of a struct to a map[string]interface{}.
// Code inspired by github.com/fatih/structs.
func structToMap(v reflect.Value) map[string]interface{} {
	out := make(map[string]interface{})
	t := v.Type()
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		if field.PkgPath != "" {
			// unexported field. skip.
			continue
		}
		out[field.Name] = v.FieldByName(field.Name).Interface()
	}
	return out
}
<commit_msg>bundle: Fix typos (#69)<commit_after>// Package bundle manages translations for multiple languages.
package bundle

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"sync"
	"unicode"

	"github.com/nicksnyder/go-i18n/i18n/language"
	"github.com/nicksnyder/go-i18n/i18n/translation"
	toml "github.com/pelletier/go-toml"
	"gopkg.in/yaml.v2"
)

// TranslateFunc is a copy of i18n.TranslateFunc to avoid a circular dependency.
type TranslateFunc func(translationID string, args ...interface{}) string

// Bundle stores the translations for multiple languages.
type Bundle struct {
	// The primary translations for a language tag and translation id.
	translations map[string]map[string]translation.Translation

	// Translations that can be used when an exact language match is not possible.
	fallbackTranslations map[string]map[string]translation.Translation

	sync.RWMutex
}

// New returns an empty bundle.
func New() *Bundle {
	return &Bundle{
		translations:         make(map[string]map[string]translation.Translation),
		fallbackTranslations: make(map[string]map[string]translation.Translation),
	}
}

// MustLoadTranslationFile is similar to LoadTranslationFile
// except it panics if an error happens.
func (b *Bundle) MustLoadTranslationFile(filename string) {
	if err := b.LoadTranslationFile(filename); err != nil {
		panic(err)
	}
}

// LoadTranslationFile loads the translations from filename into memory.
//
// The language that the translations are associated with is parsed from the filename (e.g. en-US.json).
//
// Generally you should load translation files once during your program's initialization.
func (b *Bundle) LoadTranslationFile(filename string) error {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}
	return b.ParseTranslationFileBytes(filename, buf)
}

// ParseTranslationFileBytes is similar to LoadTranslationFile except it parses the bytes in buf.
//
// It is useful for parsing translation files embedded with go-bindata.
func (b *Bundle) ParseTranslationFileBytes(filename string, buf []byte) error {
	basename := filepath.Base(filename)
	langs := language.Parse(basename)
	switch l := len(langs); {
	case l == 0:
		return fmt.Errorf("no language found in %q", basename)
	case l > 1:
		return fmt.Errorf("multiple languages found in filename %q: %v; expected one", basename, langs)
	}
	translations, err := parseTranslations(filename, buf)
	if err != nil {
		return err
	}
	b.AddTranslation(langs[0], translations...)
	return nil
}

func parseTranslations(filename string, buf []byte) ([]translation.Translation, error) {
	if len(buf) == 0 {
		return []translation.Translation{}, nil
	}

	ext := filepath.Ext(filename)

	// `github.com/pelletier/go-toml` lacks an Unmarshal function,
	// so we should parse TOML separately.
	if ext == ".toml" {
		tree, err := toml.LoadReader(bytes.NewReader(buf))
		if err != nil {
			return nil, err
		}

		m := make(map[string]map[string]interface{})
		for k, v := range tree.ToMap() {
			m[k] = v.(map[string]interface{})
		}

		return parseFlatFormat(m)
	}

	// Then parse other formats.
	if isStandardFormat(ext, buf) {
		var standardFormat []map[string]interface{}
		if err := unmarshal(ext, buf, &standardFormat); err != nil {
			return nil, fmt.Errorf("failed to unmarshal %v: %v", filename, err)
		}
		return parseStandardFormat(standardFormat)
	} else {
		var flatFormat map[string]map[string]interface{}
		if err := unmarshal(ext, buf, &flatFormat); err != nil {
			return nil, fmt.Errorf("failed to unmarshal %v: %v", filename, err)
		}
		return parseFlatFormat(flatFormat)
	}
}

func isStandardFormat(ext string, buf []byte) bool {
	buf = deleteLeadingComments(ext, buf)
	firstRune := rune(buf[0])
	if (ext == ".json" && firstRune == '[') || (ext == ".yaml" && firstRune == '-') {
		return true
	}
	return false
}

// deleteLeadingComments deletes leading newlines and comments in buf.
// It only works for ext == ".yaml".
func deleteLeadingComments(ext string, buf []byte) []byte {
	if ext != ".yaml" {
		return buf
	}

	for {
		buf = bytes.TrimLeftFunc(buf, unicode.IsSpace)
		if buf[0] == '#' {
			buf = deleteLine(buf)
		} else {
			break
		}
	}

	return buf
}

func deleteLine(buf []byte) []byte {
	index := bytes.IndexRune(buf, '\n')
	if index == -1 { // If there is only one line without newline ...
		return nil // ... delete it and return nothing.
	}
	if index == len(buf)-1 { // If there is only one line with newline ...
		return nil // ... do the same as above.
	}
	return buf[index+1:]
}

// unmarshal finds an appropriate unmarshal function for ext
// (extension of filename) and unmarshals buf to out. out must be a pointer.
func unmarshal(ext string, buf []byte, out interface{}) error {
	switch ext {
	case ".json":
		return json.Unmarshal(buf, out)
	case ".yaml":
		return yaml.Unmarshal(buf, out)
	}

	return fmt.Errorf("unsupported file extension %v", ext)
}

func parseStandardFormat(data []map[string]interface{}) ([]translation.Translation, error) {
	translations := make([]translation.Translation, 0, len(data))
	for i, translationData := range data {
		t, err := translation.NewTranslation(translationData)
		if err != nil {
			return nil, fmt.Errorf("unable to parse translation #%d because %s\n%v", i, err, translationData)
		}
		translations = append(translations, t)
	}
	return translations, nil
}

// parseFlatFormat just converts data from flat format to standard format
// and passes it to parseStandardFormat.
//
// Flat format logic:
// key of data must be a string and data[key] must be always map[string]interface{},
// but if there is only "other" key in it then it is non-plural, else plural.
func parseFlatFormat(data map[string]map[string]interface{}) ([]translation.Translation, error) {
	var standardFormatData []map[string]interface{}
	for id, translationData := range data {
		dataObject := make(map[string]interface{})
		dataObject["id"] = id
		if len(translationData) == 1 { // non-plural form
			_, otherExists := translationData["other"]
			if otherExists {
				dataObject["translation"] = translationData["other"]
			}
		} else { // plural form
			dataObject["translation"] = translationData
		}

		standardFormatData = append(standardFormatData, dataObject)
	}

	return parseStandardFormat(standardFormatData)
}

// AddTranslation adds translations for a language.
//
// It is useful if your translations are in a format not supported by LoadTranslationFile.
func (b *Bundle) AddTranslation(lang *language.Language, translations ...translation.Translation) {
	b.Lock()
	defer b.Unlock()
	if b.translations[lang.Tag] == nil {
		b.translations[lang.Tag] = make(map[string]translation.Translation, len(translations))
	}
	currentTranslations := b.translations[lang.Tag]
	for _, newTranslation := range translations {
		if currentTranslation := currentTranslations[newTranslation.ID()]; currentTranslation != nil {
			currentTranslations[newTranslation.ID()] = currentTranslation.Merge(newTranslation)
		} else {
			currentTranslations[newTranslation.ID()] = newTranslation
		}
	}

	// lang can provide translations for less specific language tags.
	for _, tag := range lang.MatchingTags() {
		b.fallbackTranslations[tag] = currentTranslations
	}
}

// Translations returns all translations in the bundle.
func (b *Bundle) Translations() map[string]map[string]translation.Translation {
	t := make(map[string]map[string]translation.Translation)
	b.RLock()
	for tag, translations := range b.translations {
		t[tag] = make(map[string]translation.Translation)
		for id, translation := range translations {
			t[tag][id] = translation
		}
	}
	b.RUnlock()
	return t
}

// LanguageTags returns the tags of all languages that that have been added.
func (b *Bundle) LanguageTags() []string {
	var tags []string
	b.RLock()
	for k := range b.translations {
		tags = append(tags, k)
	}
	b.RUnlock()
	return tags
}

// LanguageTranslationIDs returns the ids of all translations that have been added for a given language.
func (b *Bundle) LanguageTranslationIDs(languageTag string) []string {
	var ids []string
	b.RLock()
	for id := range b.translations[languageTag] {
		ids = append(ids, id)
	}
	b.RUnlock()
	return ids
}

// MustTfunc is similar to Tfunc except it panics if an error happens.
func (b *Bundle) MustTfunc(pref string, prefs ...string) TranslateFunc {
	tfunc, err := b.Tfunc(pref, prefs...)
	if err != nil {
		panic(err)
	}
	return tfunc
}

// MustTfuncAndLanguage is similar to TfuncAndLanguage except it panics if an error happens.
func (b *Bundle) MustTfuncAndLanguage(pref string, prefs ...string) (TranslateFunc, *language.Language) {
	tfunc, language, err := b.TfuncAndLanguage(pref, prefs...)
	if err != nil {
		panic(err)
	}
	return tfunc, language
}

// Tfunc is similar to TfuncAndLanguage except is doesn't return the Language.
func (b *Bundle) Tfunc(pref string, prefs ...string) (TranslateFunc, error) {
	tfunc, _, err := b.TfuncAndLanguage(pref, prefs...)
	return tfunc, err
}

// TfuncAndLanguage returns a TranslateFunc for the first Language that
// has a non-zero number of translations in the bundle.
//
// The returned Language matches the the first language preference that could be satisfied,
// but this may not strictly match the language of the translations used to satisfy that preference.
//
// For example, the user may request "zh". If there are no translations for "zh" but there are translations
// for "zh-cn", then the translations for "zh-cn" will be used but the returned Language will be "zh".
//
// It can parse languages from Accept-Language headers (RFC 2616),
// but it assumes weights are monotonically decreasing.
func (b *Bundle) TfuncAndLanguage(pref string, prefs ...string) (TranslateFunc, *language.Language, error) {
	lang := b.supportedLanguage(pref, prefs...)
	var err error
	if lang == nil {
		err = fmt.Errorf("no supported languages found %#v", append(prefs, pref))
	}
	return func(translationID string, args ...interface{}) string {
		return b.translate(lang, translationID, args...)
	}, lang, err
}

// supportedLanguage returns the first language which
// has a non-zero number of translations in the bundle.
func (b *Bundle) supportedLanguage(pref string, prefs ...string) *language.Language {
	lang := b.translatedLanguage(pref)
	if lang == nil {
		for _, pref := range prefs {
			lang = b.translatedLanguage(pref)
			if lang != nil {
				break
			}
		}
	}
	return lang
}

func (b *Bundle) translatedLanguage(src string) *language.Language {
	langs := language.Parse(src)
	b.RLock()
	defer b.RUnlock()
	for _, lang := range langs {
		if len(b.translations[lang.Tag]) > 0 ||
			len(b.fallbackTranslations[lang.Tag]) > 0 {
			return lang
		}
	}
	return nil
}

func (b *Bundle) translate(lang *language.Language, translationID string, args ...interface{}) string {
	if lang == nil {
		return translationID
	}

	translation := b.translation(lang, translationID)
	if translation == nil {
		return translationID
	}

	var data interface{}
	var count interface{}
	if argc := len(args); argc > 0 {
		if isNumber(args[0]) {
			count = args[0]
			if argc > 1 {
				data = args[1]
			}
		} else {
			data = args[0]
		}
	}

	if count != nil {
		if data == nil {
			data = map[string]interface{}{"Count": count}
		} else {
			dataMap := toMap(data)
			dataMap["Count"] = count
			data = dataMap
		}
	} else {
		dataMap := toMap(data)
		if c, ok := dataMap["Count"]; ok {
			count = c
		}
	}

	p, _ := lang.Plural(count)
	template := translation.Template(p)
	if template == nil {
		return translationID
	}

	s := template.Execute(data)
	if s == "" {
		return translationID
	}
	return s
}

func (b *Bundle) translation(lang *language.Language, translationID string) translation.Translation {
	b.RLock()
	defer b.RUnlock()
	translations := b.translations[lang.Tag]
	if translations == nil {
		translations = b.fallbackTranslations[lang.Tag]
		if translations == nil {
			return nil
		}
	}
	return translations[translationID]
}

func isNumber(n interface{}) bool {
	switch n.(type) {
	case int, int8, int16, int32, int64, string:
		return true
	}
	return false
}

func toMap(input interface{}) map[string]interface{} {
	if data, ok := input.(map[string]interface{}); ok {
		return data
	}
	v := reflect.ValueOf(input)
	switch v.Kind() {
	case reflect.Ptr:
		return toMap(v.Elem().Interface())
	case reflect.Struct:
		return structToMap(v)
	default:
		return nil
	}
}

// Converts the top level of a struct to a map[string]interface{}.
// Code inspired by github.com/fatih/structs.
func structToMap(v reflect.Value) map[string]interface{} {
	out := make(map[string]interface{})
	t := v.Type()
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		if field.PkgPath != "" {
			// unexported field. skip.
			continue
		}
		out[field.Name] = v.FieldByName(field.Name).Interface()
	}
	return out
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2015, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package install

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"testing"

	"github.com/percona/percona-agent/pct"
	. "gopkg.in/check.v1"
)

func Test(t *testing.T) { TestingT(t) }

type MainTestSuite struct {
	cmd        *exec.Cmd
	basedir    string
	bin        string
	initscript string
	username   string
}

const (
	MOCKED_PERCONA_AGENT = "github.com/percona/percona-agent/install/mock"
)

var _ = Suite(&MainTestSuite{})

func (s *MainTestSuite) SetUpSuite(t *C) {
	var err error

	// We can't/shouldn't use /usr/local/percona/ (the default basedir),
	// so use a tmpdir instead with only a bin dir inside
	s.basedir, err = ioutil.TempDir("/tmp", "percona-agent-init-test-")
	t.Assert(err, IsNil)
	err = os.Mkdir(filepath.Join(s.basedir, pct.BIN_DIR), 0777)
	t.Assert(err, IsNil)

	// Lets compile and place the mocked percona-agent on the tmp basedir
	s.bin = filepath.Join(s.basedir, pct.BIN_DIR, "percona-agent")
	cmd := exec.Command("go", "build", "-o", s.bin, MOCKED_PERCONA_AGENT)
	err = cmd.Run()
	// Failed to compile mocked percona-agent
	t.Assert(err, IsNil, Commentf("Failed to build mocked percona agent: %v", err))

	// Get current username to set test env variable
	user, erruser := user.Current()
	t.Assert(erruser, IsNil, Commentf("Failed to obtain current user: %v", err))
	s.username = user.Username

	// Copy init script to tmp basedir/bin directory
	initscript, err := filepath.Abs("./percona-agent")
	// Check if absolute path resolving succedeed
	t.Assert(err, IsNil)
	// Check if init script is there
	t.Assert(pct.FileExists(initscript), Equals, true)
	s.initscript = filepath.Join(s.basedir, pct.BIN_DIR, "init-script")
	cmd = exec.Command("cp", initscript, s.initscript)
	err = cmd.Run()
	t.Assert(err, IsNil, Commentf("Failed to copy init script to tmp dir: %v", err))

	// Set all env vars to default test values
	resetTestEnvVars(s)
}

func (s *MainTestSuite) TearDownTest(t *C) {
	// Delete any left pid file and set mocked agent start delay to 0
	resetTestEnvVars(s)
	// Kill any remaining process before deleting pidfile
	if pid, err := readPidFile(filepath.Join(s.basedir, "percona-agent.pid")); pid != "" && err == nil {
		if numPid, err := strconv.ParseInt(pid, 10, 0); err == nil {
			syscall.Kill(int(numPid), syscall.SIGTERM)
		}
	}
	// Delete if pidFile exists
	os.Remove(filepath.Join(s.basedir, "percona-agent.pid"))
}

func (s *MainTestSuite) TearDownSuite(t *C) {
	// Delete tmp
	if err := os.RemoveAll(s.basedir); err != nil {
		t.Error(err)
	}
}

func resetTestEnvVars(s *MainTestSuite) {
	// Sadly no os.Unsetenv in Go 1.3.x
	os.Setenv("PCT_TEST_START_DELAY", "")
	os.Setenv("PCT_TEST_STOP_DELAY", "")
	os.Setenv("PCT_TEST_START_TIMEOUT", "")
	os.Setenv("PCT_TEST_STOP_TIMEOUT", "")
	os.Setenv("PCT_TEST_AGENT_USER", s.username)
	os.Setenv("PCT_TEST_AGENT_DIR", s.basedir)
}

func writePidFile(filePath, pid string) error {
	flags := os.O_CREATE | os.O_EXCL | os.O_WRONLY
	file, err := os.OpenFile(filePath, flags, 0644)
	if err != nil {
		//Could not create pidfile
		return err
	}
	// Write PID to file
	if _, err := file.WriteString(pid); err != nil {
		// Could not write to stale pidfile
		return err
	}
	file.Close()
	return nil
}

func readPidFile(pidFilePath string) (pid string, err error) {
	if bytes, err := ioutil.ReadFile(pidFilePath); err != nil {
		return "", err
	} else {
		// Remove any \n
		return strings.Replace(string(bytes), "\n", "", -1), nil
	}
}

//-----------------------------------------------------------------------------

func (s *MainTestSuite) TestStatusNoAgent(t *C) {
	cmd := exec.Command(s.initscript, "status")
	output, err := cmd.Output()
	// status exit code should be 1
	t.Check(err, NotNil)
	// script should output a message
	t.Assert(string(output), Equals, "percona-agent is not running.\n")
}

func (s *MainTestSuite) TestStopNoAgent(t *C) {
	cmd := exec.Command(s.initscript, "stop")
	output, err := cmd.Output()
	// stop exit code should be 0
	t.Check(err, IsNil)
	// script should output a message
	t.Assert(string(output), Equals, "Stopping percona-agent...\npercona-agent is not running.\n")
}

func (s *MainTestSuite) TestStartStop(t *C) {
	// Start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// Start exit code should be 0
	t.Check(err, IsNil)
	// Script should output a message
	t.Assert(string(output), Equals, "Starting percona-agent...\nWaiting for percona-agent to start...\nOK\n")

	// Check status
	cmd = exec.Command(s.initscript, "status")
	output, err = cmd.Output()
	// Status exit code should be 0
	t.Check(err, IsNil)
	// Extract PID from command output
	rePID := regexp.MustCompile(`^percona-agent\ is\ running\ \((\d+)\)\.`)
	found := rePID.FindStringSubmatch(string(output))
	// Check if the command provided a PID
	var pid string
	if len(found) == 2 {
		pid = found[1]
	} else {
		t.Error("Could not get pid for mocked percona-agent")
	}

	pidbinary, err := os.Readlink(fmt.Sprintf("/proc/%v/exe", pid))
	// Check that PID actually points to our mocked percona-agent binary
	t.Assert(pidbinary, Equals, s.bin)

	// Now try to stop
	cmd = exec.Command(s.initscript, "stop")
	output, err = cmd.Output()
	// stop exit code should be 0
	t.Check(err, IsNil)
	// script should output a message
	t.Assert(string(output), Equals, "Stopping percona-agent...\nWaiting for percona-agent to exit...\nStopped percona-agent.\n")
}

func (s *MainTestSuite) TestDoubleStart(t *C) {
	// Start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)
	// Script should output a message
	t.Assert(string(output), Equals, "Starting percona-agent...\nWaiting for percona-agent to start...\nOK\n")

	// Start service again
	cmd = exec.Command(s.initscript, "start")
	output, err = cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)
	// script should output a message
	t.Assert(string(output), Equals, "Starting percona-agent...\npercona-agent is already running.\n")
}

func (s *MainTestSuite) TestWrongBin(t *C) {
	pidFilePath := filepath.Join(s.basedir, "percona-agent.pid")
	// Create pidfile with valid PID but not corresponding to a mocked percona-agent
	if err := writePidFile(pidFilePath, string(os.Getpid())); err != nil {
		t.Errorf("Could not create pidfile: %v", err)
	}

	// Now start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)
	// Script should output a message
	t.Assert(string(output), Equals, fmt.Sprintf("Starting percona-agent...\nRemoved stale pid file: %v\nWaiting for "+
		"percona-agent to start...\nOK\n", pidFilePath))
}

func (s *MainTestSuite) TestStalePIDFile(t *C) {
	// Create pidfile with non valid PID
	pidFilePath := filepath.Join(s.basedir, "percona-agent.pid")
	if err := writePidFile(pidFilePath, string(rand.Uint32())); err != nil {
		t.Errorf("Could not create pidfile: %v", err)
	}

	// Now start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)
	// script should output a message
	t.Assert(string(output), Equals, fmt.Sprintf("Starting percona-agent...\nRemoved stale pid file: %v\nWaiting for "+
		"percona-agent to start...\nOK\n", pidFilePath))
}

func (s *MainTestSuite) TestDelayedStart(t *C) {
	// Set init script timeout to 1 second
	os.Setenv("PCT_TEST_START_TIMEOUT", "1")
	// Set percona-agent start delay to 2 seconds
	os.Setenv("PCT_TEST_START_DELAY", "2")
	// Now try to start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// start exit code should be 1
	t.Check(err, NotNil)
	// path to log file, its part of the output
	perconaLogPath := filepath.Join(s.basedir, "percona-agent.log")
	// script should output message
	t.Check(string(output), Equals, fmt.Sprintf("Starting percona-agent...\nWaiting for percona-agent to start...\nFail.  "+
		"Check %v for details.\n", perconaLogPath))
}

func (s *MainTestSuite) TestDelayedStop(t *C) {
	// Set init script stop timeout to 1 second
	os.Setenv("PCT_TEST_STOP_TIMEOUT", "1")
	// Set percona-agent stop delay to 2 seconds
	os.Setenv("PCT_TEST_STOP_DELAY", "2")
	// Now try to start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)

	// Get the PID from the pidfile
	pid, err := readPidFile(filepath.Join(s.basedir, "percona-agent.pid"))
	// Check if we could read the pidfile
	t.Check(err, IsNil)
	// pid should be non empty
	t.Check(pid, Not(Equals), "")

	stop_cmd := exec.Command(s.initscript, "stop")
	output, err = stop_cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)

	// Script should output message
	t.Check(string(output), Equals, fmt.Sprintf("Stopping percona-agent...\nWaiting for percona-agent to exit...\n"+
		"Time out waiting for percona-agent to exit.  Trying kill -9 %v...\nStopped percona-agent.\n", pid))
	// Make sure the process was killed
	t.Assert(pct.FileExists(fmt.Sprintf("/proc/%v/stat", pid)), Equals, false)
}
<commit_msg>Fix init script tests SetUp<commit_after>/*
   Copyright (c) 2015, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package install

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"testing"

	"github.com/percona/percona-agent/pct"
	. "gopkg.in/check.v1"
)

func Test(t *testing.T) { TestingT(t) }

type MainTestSuite struct {
	cmd        *exec.Cmd
	basedir    string
	bin        string
	initscript string
	username   string
	builddir   string
	buildbin   string
}

const (
	MOCKED_PERCONA_AGENT = "github.com/percona/percona-agent/install/mock"
)

var _ = Suite(&MainTestSuite{})

func (s *MainTestSuite) SetUpSuite(t *C) {
	var err error
	// Create tmp dir to store compiled mocked agent
	s.builddir, err = ioutil.TempDir("/tmp", "percona-agent-init-test-build-")
	t.Assert(err, IsNil)

	// Lets compile and place the mocked percona-agent on the tmp builddir
	s.buildbin = filepath.Join(s.builddir, "percona-agent")
	cmd := exec.Command("go", "build", "-o", s.buildbin, MOCKED_PERCONA_AGENT)
	err = cmd.Run()
	// Failed to compile mocked percona-agent
	t.Assert(err, IsNil, Commentf("Failed to build mocked percona agent: %v", err))

	// Get current username to set test env variables on each test setup
	user, erruser := user.Current()
	t.Assert(erruser, IsNil, Commentf("Failed to obtain current user: %v", err))
	s.username = user.Username
}

func (s *MainTestSuite) SetUpTest(t *C) {
	// Make a new mocked installation for test, copying the already compiled mocked agent.
	// Using the same tmp basedir for all tests could lead to dissapearing of pid files at any given time. KILL/TERM of
	// processes is async and mocked agent will remove the pid file while shutting down, making tests fail in the most
	// unexpected ways.
	var err error
	// We can't/shouldn't use /usr/local/percona/ (the default basedir),
	// so use a tmpdir instead with only a bin dir inside
	s.basedir, err = ioutil.TempDir("/tmp", "percona-agent-init-test-")
	t.Assert(err, IsNil)
	binDir := filepath.Join(s.basedir, pct.BIN_DIR)
	err = os.Mkdir(binDir, 0777)
	t.Assert(err, IsNil)

	// Lets copy mocked and already compiled percona-agent
	s.bin = filepath.Join(binDir, "percona-agent")
	cmd := exec.Command("cp", s.buildbin, s.bin)
	err = cmd.Run()
	t.Assert(err, IsNil, Commentf("Failed to copy mocked percona-agent to tmp dir: %v", err))

	// Copy init script to tmp basedir/bin directory
	initscript, err := filepath.Abs("./percona-agent")
	// Check if absolute path resolving succedeed
	t.Assert(err, IsNil)
	// Check if init script is there
	t.Assert(pct.FileExists(initscript), Equals, true)
	s.initscript = filepath.Join(s.basedir, pct.BIN_DIR, "init-script")
	cmd = exec.Command("cp", initscript, s.initscript)
	err = cmd.Run()
	t.Assert(err, IsNil, Commentf("Failed to copy init script to tmp dir: %v", err))

	// Set all env vars to default test values
	resetTestEnvVars(s)
}

func (s *MainTestSuite) TearDownTest(t *C) {
	// Delete any left pid file and set mocked agent start delay to 0
	resetTestEnvVars(s)
	// Kill any remaining process
	if pid, err := readPidFile(filepath.Join(s.basedir, "percona-agent.pid")); pid != "" && err == nil {
		if numPid, err := strconv.ParseInt(pid, 10, 0); err == nil {
			syscall.Kill(int(numPid), syscall.SIGKILL)
		}
	}
	// Remove the complete tmp basedir
	if err := os.RemoveAll(s.basedir); err != nil {
		t.Error(err)
	}
}

func (s *MainTestSuite) TearDownSuite(t *C) {
	// Delete tmp buildir
	if err := os.RemoveAll(s.builddir); err != nil {
		t.Error(err)
	}
}

func resetTestEnvVars(s *MainTestSuite) {
	// Sadly no os.Unsetenv in Go 1.3.x
	os.Setenv("PCT_TEST_START_DELAY", "")
	os.Setenv("PCT_TEST_STOP_DELAY", "")
	os.Setenv("PCT_TEST_START_TIMEOUT", "")
	os.Setenv("PCT_TEST_STOP_TIMEOUT", "")
	os.Setenv("PCT_TEST_AGENT_USER", s.username)
	os.Setenv("PCT_TEST_AGENT_DIR", s.basedir)
}

func writePidFile(filePath, pid string) error {
	flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
	file, err := os.OpenFile(filePath, flags, 0644)
	if err != nil {
		//Could not create pidfile
		return err
	}
	// Write PID to file
	if _, err := file.WriteString(pid); err != nil {
		// Could not write to stale pidfile
		return err
	}
	return file.Close()
}

func readPidFile(pidFilePath string) (pid string, err error) {
	if bytes, err := ioutil.ReadFile(pidFilePath); err != nil {
		return "", err
	} else {
		// Remove any \n
		return strings.Replace(string(bytes), "\n", "", -1), nil
	}
}

//-----------------------------------------------------------------------------

func (s *MainTestSuite) TestStatusNoAgent(t *C) {
	cmd := exec.Command(s.initscript, "status")
	output, err := cmd.Output()
	// status exit code should be 1
	t.Check(err, NotNil)
	// script should output a message
	t.Assert(string(output), Equals, "percona-agent is not running.\n")
}

func (s *MainTestSuite) TestStopNoAgent(t *C) {
	cmd := exec.Command(s.initscript, "stop")
	output, err := cmd.Output()
	// stop exit code should be 0
	t.Check(err, IsNil)
	// script should output a message
	t.Assert(string(output), Equals, "Stopping percona-agent...\npercona-agent is not running.\n")
}

func (s *MainTestSuite) TestStartStop(t *C) {
	// Start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// Start exit code should be 0
	t.Check(err, IsNil)
	// Script should output a message
	t.Assert(string(output), Equals, "Starting percona-agent...\nWaiting for percona-agent to start...\nOK\n")

	// Check status
	cmd = exec.Command(s.initscript, "status")
	output, err = cmd.Output()
	// Status exit code should be 0
	t.Check(err, IsNil)
	// Extract PID from command output
	rePID := regexp.MustCompile(`^percona-agent\ is\ running\ \((\d+)\)\.`)
	found := rePID.FindStringSubmatch(string(output))
	// Check if the command provided a PID
	var pid string
	if len(found) == 2 {
		pid = found[1]
	} else {
		t.Error("Could not get pid for mocked percona-agent")
	}

	pidbinary, err := os.Readlink(fmt.Sprintf("/proc/%v/exe", pid))
	// Check that PID actually points to our mocked percona-agent binary
	t.Assert(pidbinary, Equals, s.bin)

	// Now try to stop
	cmd = exec.Command(s.initscript, "stop")
	output, err = cmd.Output()
	// stop exit code should be 0
	t.Check(err, IsNil)
	// script should output a message
	t.Assert(string(output), Equals, "Stopping percona-agent...\nWaiting for percona-agent to exit...\nStopped percona-agent.\n")
}

func (s *MainTestSuite) TestDoubleStart(t *C) {
	// Start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)
	// Script should output a message
	t.Assert(string(output), Equals, "Starting percona-agent...\nWaiting for percona-agent to start...\nOK\n")

	// Start service again
	cmd = exec.Command(s.initscript, "start")
	output, err = cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)
	// script should output a message
	t.Assert(string(output), Equals, "Starting percona-agent...\npercona-agent is already running.\n")
}

func (s *MainTestSuite) TestWrongBin(t *C) {
	pidFilePath := filepath.Join(s.basedir, "percona-agent.pid")
	// Create pidfile with valid PID but not corresponding to a mocked percona-agent
	if err := writePidFile(pidFilePath, string(os.Getpid())); err != nil {
		t.Errorf("Could not create pidfile: %v", err)
	}

	// Now start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)
	// Script should output a message
	t.Assert(string(output), Equals, fmt.Sprintf("Starting percona-agent...\nRemoved stale pid file: %v\nWaiting for "+
		"percona-agent to start...\nOK\n", pidFilePath))
}

func (s *MainTestSuite) TestStalePIDFile(t *C) {
	// Create pidfile with non valid PID
	pidFilePath := filepath.Join(s.basedir, "percona-agent.pid")
	pidString := fmt.Sprintf("%d", rand.Uint32())
	if err := writePidFile(pidFilePath, pidString); err != nil {
		t.Errorf("Could not create pidfile: %v", err)
	}
	// Now start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)
	// script should output a message
	t.Assert(string(output), Equals, fmt.Sprintf("Starting percona-agent...\nRemoved stale pid file: %v\nWaiting for "+
		"percona-agent to start...\nOK\n", pidFilePath))
}

func (s *MainTestSuite) TestDelayedStart(t *C) {
	// Set init script timeout to 1 second
	os.Setenv("PCT_TEST_START_TIMEOUT", "1")
	// Set percona-agent start delay to 2 seconds
	os.Setenv("PCT_TEST_START_DELAY", "2")
	// Now try to start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// start exit code should be 1
	t.Check(err, NotNil)
	// path to log file, its part of the output
	perconaLogPath := filepath.Join(s.basedir, "percona-agent.log")
	// script should output message
	t.Check(string(output), Equals, fmt.Sprintf("Starting percona-agent...\nWaiting for percona-agent to start...\nFail.  "+
		"Check %v for details.\n", perconaLogPath))
}

func (s *MainTestSuite) TestDelayedStop(t *C) {
	// Set init script stop timeout to 1 second
	os.Setenv("PCT_TEST_STOP_TIMEOUT", "1")
	// Set percona-agent stop delay to 2 seconds
	os.Setenv("PCT_TEST_STOP_DELAY", "2")
	// Now try to start service
	cmd := exec.Command(s.initscript, "start")
	output, err := cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)

	// Get the PID from the pidfile
	pid, err := readPidFile(filepath.Join(s.basedir, "percona-agent.pid"))
	// Check if we could read the pidfile
	t.Check(err, IsNil)
	// pid should be non empty
	t.Check(pid, Not(Equals), "")

	stop_cmd := exec.Command(s.initscript, "stop")
	output, err = stop_cmd.Output()
	// start exit code should be 0
	t.Check(err, IsNil)

	// Script should output message
	t.Check(string(output), Equals, fmt.Sprintf("Stopping percona-agent...\nWaiting for percona-agent to exit...\n"+
		"Time out waiting for percona-agent to exit.  Trying kill -9 %v...\nStopped percona-agent.\n", pid))
	// Make sure the process was killed
	t.Assert(pct.FileExists(fmt.Sprintf("/proc/%v/stat", pid)), Equals, false)
}
<|endoftext|>


<commit_before>package imagebuildah

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/containers/buildah/define"
	"github.com/containers/buildah/util"
	"github.com/containers/common/pkg/config"
	"github.com/containers/image/v5/docker/reference"
	"github.com/containers/storage"
	"github.com/containers/storage/pkg/archive"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/openshift/imagebuilder"
	"github.com/openshift/imagebuilder/dockerfile/parser"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	PullIfMissing = define.PullIfMissing
	PullAlways    = define.PullAlways
	PullIfNewer   = define.PullIfNewer
	PullNever     = define.PullNever

	Gzip         = archive.Gzip
	Bzip2        = archive.Bzip2
	Xz           = archive.Xz
	Zstd         = archive.Zstd
	Uncompressed = archive.Uncompressed
)

// Mount is a mountpoint for the build container.
type Mount specs.Mount

type BuildOptions = define.BuildOptions

// BuildDockerfiles parses a set of one or more Dockerfiles (which may be
// URLs), creates a new Executor, and then runs Prepare/Execute/Commit/Delete
// over the entire set of instructions.
func BuildDockerfiles(ctx context.Context, store storage.Store, options define.BuildOptions, paths ...string) (string, reference.Canonical, error) {
	if len(paths) == 0 {
		return "", nil, errors.Errorf("error building: no dockerfiles specified")
	}
	logger := logrus.New()
	if options.Err != nil {
		logger.SetOutput(options.Err)
	} else {
		logger.SetOutput(os.Stderr)
	}
	logger.SetLevel(logrus.GetLevel())

	var dockerfiles []io.ReadCloser
	defer func(dockerfiles ...io.ReadCloser) {
		for _, d := range dockerfiles {
			d.Close()
		}
	}(dockerfiles...)

	for _, tag := range append([]string{options.Output}, options.AdditionalTags...) {
		if tag == "" {
			continue
		}
		if _, err := util.VerifyTagName(tag); err != nil {
			return "", nil, errors.Wrapf(err, "tag %s", tag)
		}
	}
	for _, dfile := range paths {
		var data io.ReadCloser

		if strings.HasPrefix(dfile, "http://") || strings.HasPrefix(dfile, "https://") {
			logrus.Debugf("reading remote Dockerfile %q", dfile)
			resp, err := http.Get(dfile)
			if err != nil {
				return "", nil, err
			}
			if resp.ContentLength == 0 {
				resp.Body.Close()
				return "", nil, errors.Errorf("no contents in %q", dfile)
			}
			data = resp.Body
		} else {
			dinfo, err := os.Stat(dfile)
			if err != nil {
				// If the Dockerfile isn't available, try again with
				// context directory prepended (if not prepended yet).
				if !strings.HasPrefix(dfile, options.ContextDirectory) {
					dfile = filepath.Join(options.ContextDirectory, dfile)
					dinfo, err = os.Stat(dfile)
				}
			}
			if err != nil {
				return "", nil, err
			}

			var contents *os.File
			// If given a directory, add '/Dockerfile' to it.
			if dinfo.Mode().IsDir() {
				for _, file := range []string{"Containerfile", "Dockerfile"} {
					f := filepath.Join(dfile, file)
					logrus.Debugf("reading local %q", f)
					contents, err = os.Open(f)
					if err == nil {
						break
					}
				}
			} else {
				contents, err = os.Open(dfile)
			}

			if err != nil {
				return "", nil, err
			}
			dinfo, err = contents.Stat()
			if err != nil {
				contents.Close()
				return "", nil, errors.Wrapf(err, "error reading info about %q", dfile)
			}
			if dinfo.Mode().IsRegular() && dinfo.Size() == 0 {
				contents.Close()
				return "", nil, errors.Errorf("no contents in %q", dfile)
			}
			data = contents
		}

		// pre-process Dockerfiles with ".in" suffix
		if strings.HasSuffix(dfile, ".in") {
			pData, err := preprocessContainerfileContents(dfile, data, options.ContextDirectory)
			if err != nil {
				return "", nil, err
			}
			data = *pData
		}

		dockerfiles = append(dockerfiles, data)
	}

	mainNode, err := imagebuilder.ParseDockerfile(dockerfiles[0])
	if err != nil {
		return "", nil, errors.Wrapf(err, "error parsing main Dockerfile: %s", dockerfiles[0])
	}

	warnOnUnsetBuildArgs(logger, mainNode, options.Args)

	for _, d := range dockerfiles[1:] {
		additionalNode, err := imagebuilder.ParseDockerfile(d)
		if err != nil {
			return "", nil, errors.Wrapf(err, "error parsing additional Dockerfile %s", d)
		}
		mainNode.Children = append(mainNode.Children, additionalNode.Children...)
	}
	exec, err := NewExecutor(logger, store, options, mainNode)
	if err != nil {
		return "", nil, errors.Wrapf(err, "error creating build executor")
	}
	b := imagebuilder.NewBuilder(options.Args)
	defaultContainerConfig, err := config.Default()
	if err != nil {
		return "", nil, errors.Wrapf(err, "failed to get container config")
	}
	b.Env = append(defaultContainerConfig.GetDefaultEnv(), b.Env...)
	stages, err := imagebuilder.NewStages(mainNode, b)
	if err != nil {
		return "", nil, errors.Wrap(err, "error reading multiple stages")
	}
	if options.Target != "" {
		stagesTargeted, ok := stages.ThroughTarget(options.Target)
		if !ok {
			return "", nil, errors.Errorf("The target %q was not found in the provided Dockerfile", options.Target)
		}
		stages = stagesTargeted
	}
	return exec.Build(ctx, stages)
}

func warnOnUnsetBuildArgs(logger *logrus.Logger, node *parser.Node, args map[string]string) {
	argFound := make(map[string]bool)
	for _, child := range node.Children {
		switch strings.ToUpper(child.Value) {
		case "ARG":
			argName := child.Next.Value
			if strings.Contains(argName, "=") {
				res := strings.Split(argName, "=")
				if res[1] != "" {
					argFound[res[0]] = true
				}
			}
			argHasValue := true
			if !strings.Contains(argName, "=") {
				argHasValue = argFound[argName]
			}
			if _, ok := args[argName]; !argHasValue && !ok {
				logger.Warnf("missing %q build argument. Try adding %q to the command line", argName, fmt.Sprintf("--build-arg %s=<VALUE>", argName))
			}
		default:
			continue
		}
	}
}

// preprocessContainerfileContents runs CPP(1) in preprocess-only mode on the input
// dockerfile content and will use ctxDir as the base include path.
//
// Note: we cannot use cmd.StdoutPipe() as cmd.Wait() closes it.
func preprocessContainerfileContents(containerfile string, r io.Reader, ctxDir string) (rdrCloser *io.ReadCloser, err error) {
	cppPath := "/usr/bin/cpp"
	if _, err = os.Stat(cppPath); err != nil {
		if os.IsNotExist(err) {
			err = errors.Errorf("error: %s support requires %s to be installed", containerfile, cppPath)
		}
		return nil, err
	}

	stdout := bytes.Buffer{}
	stderr := bytes.Buffer{}

	cmd := exec.Command(cppPath, "-E", "-iquote", ctxDir, "-traditional", "-undef", "-")
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	pipe, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	defer pipe.Close()

	if err = cmd.Start(); err != nil {
		return nil, err
	}

	if _, err = io.Copy(pipe, r); err != nil {
		return nil, err
	}

	pipe.Close()
	if err = cmd.Wait(); err != nil {
		if stdout.Len() == 0 {
			return nil, errors.Wrapf(err, "error pre-processing Dockerfile")
		}
		logrus.Warnf("Ignoring %s\n", stderr.String())
	}

	rc := ioutil.NopCloser(bytes.NewReader(stdout.Bytes()))
	return &rc, nil
}
<commit_msg>imagebuildah: use the specified logger for logging preprocessing warnings<commit_after>package imagebuildah

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/containers/buildah/define"
	"github.com/containers/buildah/util"
	"github.com/containers/common/pkg/config"
	"github.com/containers/image/v5/docker/reference"
	"github.com/containers/storage"
	"github.com/containers/storage/pkg/archive"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/openshift/imagebuilder"
	"github.com/openshift/imagebuilder/dockerfile/parser"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	PullIfMissing = define.PullIfMissing
	PullAlways    = define.PullAlways
	PullIfNewer   = define.PullIfNewer
	PullNever     = define.PullNever

	Gzip         = archive.Gzip
	Bzip2        = archive.Bzip2
	Xz           = archive.Xz
	Zstd         = archive.Zstd
	Uncompressed = archive.Uncompressed
)

// Mount is a mountpoint for the build container.
type Mount specs.Mount

type BuildOptions = define.BuildOptions

// BuildDockerfiles parses a set of one or more Dockerfiles (which may be
// URLs), creates a new Executor, and then runs Prepare/Execute/Commit/Delete
// over the entire set of instructions.
func BuildDockerfiles(ctx context.Context, store storage.Store, options define.BuildOptions, paths ...string) (string, reference.Canonical, error) {
	if len(paths) == 0 {
		return "", nil, errors.Errorf("error building: no dockerfiles specified")
	}
	logger := logrus.New()
	if options.Err != nil {
		logger.SetOutput(options.Err)
	} else {
		logger.SetOutput(os.Stderr)
	}
	logger.SetLevel(logrus.GetLevel())

	var dockerfiles []io.ReadCloser
	defer func(dockerfiles ...io.ReadCloser) {
		for _, d := range dockerfiles {
			d.Close()
		}
	}(dockerfiles...)

	for _, tag := range append([]string{options.Output}, options.AdditionalTags...) {
		if tag == "" {
			continue
		}
		if _, err := util.VerifyTagName(tag); err != nil {
			return "", nil, errors.Wrapf(err, "tag %s", tag)
		}
	}
	for _, dfile := range paths {
		var data io.ReadCloser

		if strings.HasPrefix(dfile, "http://") || strings.HasPrefix(dfile, "https://") {
			logrus.Debugf("reading remote Dockerfile %q", dfile)
			resp, err := http.Get(dfile)
			if err != nil {
				return "", nil, err
			}
			if resp.ContentLength == 0 {
				resp.Body.Close()
				return "", nil, errors.Errorf("no contents in %q", dfile)
			}
			data = resp.Body
		} else {
			dinfo, err := os.Stat(dfile)
			if err != nil {
				// If the Dockerfile isn't available, try again with
				// context directory prepended (if not prepended yet).
				if !strings.HasPrefix(dfile, options.ContextDirectory) {
					dfile = filepath.Join(options.ContextDirectory, dfile)
					dinfo, err = os.Stat(dfile)
				}
			}
			if err != nil {
				return "", nil, err
			}

			var contents *os.File
			// If given a directory, add '/Dockerfile' to it.
			if dinfo.Mode().IsDir() {
				for _, file := range []string{"Containerfile", "Dockerfile"} {
					f := filepath.Join(dfile, file)
					logrus.Debugf("reading local %q", f)
					contents, err = os.Open(f)
					if err == nil {
						break
					}
				}
			} else {
				contents, err = os.Open(dfile)
			}

			if err != nil {
				return "", nil, err
			}
			dinfo, err = contents.Stat()
			if err != nil {
				contents.Close()
				return "", nil, errors.Wrapf(err, "error reading info about %q", dfile)
			}
			if dinfo.Mode().IsRegular() && dinfo.Size() == 0 {
				contents.Close()
				return "", nil, errors.Errorf("no contents in %q", dfile)
			}
			data = contents
		}

		// pre-process Dockerfiles with ".in" suffix
		if strings.HasSuffix(dfile, ".in") {
			pData, err := preprocessContainerfileContents(logger, dfile, data, options.ContextDirectory)
			if err != nil {
				return "", nil, err
			}
			data = ioutil.NopCloser(pData)
		}

		dockerfiles = append(dockerfiles, data)
	}

	mainNode, err := imagebuilder.ParseDockerfile(dockerfiles[0])
	if err != nil {
		return "", nil, errors.Wrapf(err, "error parsing main Dockerfile: %s", dockerfiles[0])
	}

	warnOnUnsetBuildArgs(logger, mainNode, options.Args)

	for _, d := range dockerfiles[1:] {
		additionalNode, err := imagebuilder.ParseDockerfile(d)
		if err != nil {
			return "", nil, errors.Wrapf(err, "error parsing additional Dockerfile %s", d)
		}
		mainNode.Children = append(mainNode.Children, additionalNode.Children...)
	}
	exec, err := NewExecutor(logger, store, options, mainNode)
	if err != nil {
		return "", nil, errors.Wrapf(err, "error creating build executor")
	}
	b := imagebuilder.NewBuilder(options.Args)
	defaultContainerConfig, err := config.Default()
	if err != nil {
		return "", nil, errors.Wrapf(err, "failed to get container config")
	}
	b.Env = append(defaultContainerConfig.GetDefaultEnv(), b.Env...)
	stages, err := imagebuilder.NewStages(mainNode, b)
	if err != nil {
		return "", nil, errors.Wrap(err, "error reading multiple stages")
	}
	if options.Target != "" {
		stagesTargeted, ok := stages.ThroughTarget(options.Target)
		if !ok {
			return "", nil, errors.Errorf("The target %q was not found in the provided Dockerfile", options.Target)
		}
		stages = stagesTargeted
	}
	return exec.Build(ctx, stages)
}

func warnOnUnsetBuildArgs(logger *logrus.Logger, node *parser.Node, args map[string]string) {
	argFound := make(map[string]bool)
	for _, child := range node.Children {
		switch strings.ToUpper(child.Value) {
		case "ARG":
			argName := child.Next.Value
			if strings.Contains(argName, "=") {
				res := strings.Split(argName, "=")
				if res[1] != "" {
					argFound[res[0]] = true
				}
			}
			argHasValue := true
			if !strings.Contains(argName, "=") {
				argHasValue = argFound[argName]
			}
			if _, ok := args[argName]; !argHasValue && !ok {
				logger.Warnf("missing %q build argument. Try adding %q to the command line", argName, fmt.Sprintf("--build-arg %s=<VALUE>", argName))
			}
		default:
			continue
		}
	}
}

// preprocessContainerfileContents runs CPP(1) in preprocess-only mode on the input
// dockerfile content and will use ctxDir as the base include path.
func preprocessContainerfileContents(logger *logrus.Logger, containerfile string, r io.Reader, ctxDir string) (stdout io.Reader, err error) {
	cppCommand := "cpp"
	cppPath, err := exec.LookPath(cppCommand)
	if err != nil {
		if os.IsNotExist(err) {
			err = errors.Errorf("error: %s support requires %s to be installed", containerfile, cppPath)
		}
		return nil, err
	}

	stdoutBuffer := bytes.Buffer{}
	stderrBuffer := bytes.Buffer{}

	cmd := exec.Command(cppPath, "-E", "-iquote", ctxDir, "-traditional", "-undef", "-")
	cmd.Stdin = r
	cmd.Stdout = &stdoutBuffer
	cmd.Stderr = &stderrBuffer

	if err = cmd.Start(); err != nil {
		return nil, errors.Wrapf(err, "preprocessing %s", containerfile)
	}
	if err = cmd.Wait(); err != nil {
		if stderrBuffer.Len() != 0 {
			logger.Warnf("Ignoring %s\n", stderrBuffer.String())
		}
		if stdoutBuffer.Len() == 0 {
			return nil, errors.Wrapf(err, "error preprocessing %s: preprocessor produced no output", containerfile)
		}
	}
	return &stdoutBuffer, nil
}
<|endoftext|>


<commit_before>package main

import (
	"appengine"
	"appengine/datastore"
	"appengine/file"
	"appengine/urlfetch"
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/hoisie/mustache"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/cloud"
	"google.golang.org/cloud/storage"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

func UserFetchHandler(w http.ResponseWriter, r *http.Request) {
	dir := path.Join(os.Getenv("PWD"), "templates")
	profile := path.Join(dir, "profile.html")
	layout := path.Join(dir, "profileLayout.html")
	vars := mux.Vars(r)
	c := appengine.NewContext(r)

	db := DB{c}
	match, _ := regexp.MatchString("^[0-9]+$", vars["user"])
	var err error
	var userMetaTemp interface{}
	var storedUserData interface{}
	var data string

	if match {
		userId, _ := strconv.ParseInt(vars["user"], 10, 64)
		userMetaTemp, err = db.GetUserMeta(userId)
		storedUserData, _ = db.GetUserData(userId)
	} else {
		temp := []StoredUserMeta{}
		q := datastore.NewQuery("UserMeta").Filter("VanityUrl =", strings.ToLower(vars["user"])).Limit(1)
		k, _ := q.GetAll(c, &temp)
		if len(temp) > 0 {
			userMetaTemp = temp[0]
			storedUserData, _ = db.GetUserData(k[0].IntID())
		} else {
			user404 := path.Join(dir, "user404.html")
			userData := map[string]string{"user": vars["user"]}
			data = mustache.RenderFileInLayout(user404, layout, userData)
			w.WriteHeader(http.StatusNotFound)
		}
	}

	if err == datastore.ErrNoSuchEntity {
		user404 := path.Join(dir, "user404.html")
		userData := map[string]string{"user": vars["user"]}
		data = mustache.RenderFileInLayout(user404, layout, userData)
		w.WriteHeader(http.StatusNotFound)
	} else if err != nil {
		fmt.Fprint(w, err.Error())
	} else if userMetaTemp != nil {

		userMeta := userMetaTemp.(StoredUserMeta)

		userData := map[string]interface{}{
			"username":    userMeta.Username,
			"userId":      userMeta.UserId,
			"description": userMeta.Description,
			"location":    userMeta.Location,
			"avatarUrl":   userMeta.AvatarUrl,
			"loops":       strconv.FormatInt(userMeta.Current.Loops, 10),
			"followers":   strconv.FormatInt(userMeta.Current.Followers, 10),
			"data":        storedUserData,
			"previous":    userMeta.Previous,
		}

		if userMeta.Background != "" {
			color := strings.SplitAfterN(userMeta.Background, "0x", 2)
			userData["profileBackground"] = color[1]
		} else {
			userData["profileBackground"] = "00BF8F"
		}

		data = mustache.RenderFileInLayout(profile, layout, userData)
	}

	fmt.Fprint(w, data)
}

func UserStoreHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vineApi := VineRequest{c}
	db := DB{c}
	_, err := GetQueuedUser(r.FormValue("id"), c)
	data := make(map[string]bool)

	if err != datastore.ErrNoSuchEntity && err != nil {
		c.Errorf("got UserStore err: %v", err)
	}

	user, apiErr := vineApi.GetUser(r.FormValue("id"))

	if err == datastore.ErrNoSuchEntity {

		if apiErr != nil {
			data["exists"] = false
			data["queued"] = false
		} else {
			QueueUser(user.UserIdStr, c)
			data["exists"] = true
			data["queued"] = true
		}

		data["stored"] = false

	} else {
		_, err := db.GetUserMeta(user.UserId)
		if err == datastore.ErrNoSuchEntity {
			data["stored"] = false
		} else {
			data["stored"] = true
		}
		data["exists"] = true
		data["queued"] = false
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(data)
}

func AboutHandler(w http.ResponseWriter, r *http.Request) {
	dir := path.Join(os.Getenv("PWD"), "templates")
	aboutPage := path.Join(dir, "about.html")
	layout := path.Join(dir, "pageLayout.html")

	db := DB{appengine.NewContext(r)}
	totalUsers, _ := db.GetTotalUsers()
	stats := map[string]interface{}{"users": totalUsers}
	stats["lastUpdated"] = db.GetLastUpdated()
	data := mustache.RenderFileInLayout(aboutPage, layout, stats)

	fmt.Fprint(w, data)
}

func DiscoverHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vineApi := VineRequest{c}
	db := DB{c}
	var recentUsers []*VineUser
	var recentVerified []StoredUserMeta

	recent := datastore.NewQuery("Queue").Order("-Discovered").Limit(5).KeysOnly()
	k, _ := recent.GetAll(c, nil)
	for i, _ := range k {
		user, err := vineApi.GetUser(strconv.FormatInt(k[i].IntID(), 10))
		if err == nil {
			recentUsers = append(recentUsers, user)
		}
	}
	verified := datastore.NewQuery("UserMeta").Filter("Verified =", true).Limit(5).KeysOnly()
	k, _ = verified.GetAll(c, nil)
	for i, _ := range k {
		user, err := db.GetUserMeta(k[i].IntID())
		if err == nil {
			recentVerified = append(recentVerified, user.(StoredUserMeta))
		}
	}
	data := map[string]interface{}{"recentUsers": recentUsers, "recentVerified": recentVerified}
	dir := path.Join(os.Getenv("PWD"), "templates")
	discover := path.Join(dir, "discover.html")
	layout := path.Join(dir, "pageLayout.html")
	page := mustache.RenderFileInLayout(discover, layout, data)
	fmt.Fprint(w, page)
}

func TopHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	db := DB{c}

	dir := path.Join(os.Getenv("PWD"), "templates")
	top := path.Join(dir, "top.html")
	layout := path.Join(dir, "pageLayout.html")
	data := db.GetTop()
	data["LastUpdated"] = db.GetLastUpdated()
	page := mustache.RenderFileInLayout(top, layout, data)
	fmt.Fprint(w, page)
}

func RandomHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	q := datastore.NewQuery("UserMeta").KeysOnly()
	keys, err := q.GetAll(c, nil)
    if err != nil {
        c.Errorf("got err %v", err)
        return
    }
	randomKey := RandomKey(keys)
	var user QueuedUser
	key := datastore.NewKey(c, "Queue", "", randomKey.IntID(), nil)
	err = datastore.Get(c, key, &user)
	if err != nil {
		c.Errorf("got err %v", err)
	} else {
		c.Infof("got user %v", user)
	}
	http.Redirect(w, r, "/u/"+user.UserID, 301)
}

func SearchHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	dir := path.Join(os.Getenv("PWD"), "templates")
	search := path.Join(dir, "search.html")
	layout := path.Join(dir, "pageLayout.html")
	data := map[string]interface{}{
		"query": r.FormValue("q"),
		"count": 0,
	}
	if len(r.FormValue("q")) > 0 {
		results, err := SearchUsers(c, r.FormValue("q"))
		if err != nil {
			c.Errorf("got err on search: %v", err)
		}

		switch r.FormValue("s") {
		case "overall":
			sort.Sort(ByOverall(results))
			break
		case "followers":
			sort.Sort(ByFollowers(results))
			break
		case "loops":
			sort.Sort(ByLoops(results))
			break
		case "posts":
			sort.Sort(ByPosts(results))
			break
		case "revines":
			sort.Sort(ByRevines(results))
			break
		}

		if r.Method == "GET" {
			data["count"] = len(results)
			data["results"] = results
		} else if r.Method == "POST" {
			jsonData, _ := json.Marshal(results)
			fmt.Fprint(w, string(jsonData))
			return
		}
	}

	page := mustache.RenderFileInLayout(search, layout, data)
	fmt.Fprint(w, page)
}

func DonateHandler(w http.ResponseWriter, r *http.Request) {
	dir := path.Join(os.Getenv("PWD"), "templates")
	donate := path.Join(dir, "donate.html")
	layout := path.Join(dir, "pageLayout.html")
	page := mustache.RenderFileInLayout(donate, layout, nil)
	fmt.Fprint(w, page)
}

func ExportHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	db := DB{c}
	vars := mux.Vars(r)

	if r.Method == "GET" {
		userId, err := strconv.ParseInt(vars["user"], 10, 64)
		if err != nil {
			c.Errorf("got err: %v", err)
			http.Redirect(w, r, "/404", 301)
			return
		}
		userMeta, err := db.GetUserMeta(userId)
		if err == datastore.ErrNoSuchEntity {
			http.Redirect(w, r, "/404", 301)
			return
		}
		user := userMeta.(StoredUserMeta)
		data := map[string]string{"username": user.Username, "userId": vars["user"], "captcha": Config["captchaPublic"]}
		dir := path.Join(os.Getenv("PWD"), "templates")
		export := path.Join(dir, "export.html")
		layout := path.Join(dir, "pageLayout.html")
		page := mustache.RenderFileInLayout(export, layout, data)
		fmt.Fprint(w, page)
	} else if r.Method == "POST" {
		client := urlfetch.Client(c)
		url := "https://www.google.com/recaptcha/api/siteverify?secret=" + Config["captchaPrivate"]
		url += "&response=" + r.FormValue("g-recaptcha-response") + "&remoteip=" + r.RemoteAddr
		req, _ := http.NewRequest("GET", url, nil)
		resp, err := client.Do(req)
		if err != nil {
			c.Errorf("got err: %v", err)
			return
		}
		body, _ := ioutil.ReadAll(resp.Body)
		var data map[string]interface{}
		json.Unmarshal(body, &data)
		if data["success"].(bool) {
			export := Export{c}
			export.User(vars["user"], w)
		} else {
			fmt.Fprint(w, "Seems like your CAPTCHA was wrong. Please press back and try again.")
		}
	}
}

func PopularFetchHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vineApi := VineRequest{c}
    start := time.Now()
    users := make(chan string, 60)

	err := vineApi.GetPopularUsers(users, cap(users))
	for v := range users {
		if _, err := GetQueuedUser(v, c); err == datastore.ErrNoSuchEntity {
			QueueUser(v, c)
		}
	}

    finish := time.Since(start)
	fmt.Fprint(w, "queuing popular users: %v w/ err %v", users, err)
	c.Infof("queueing popular users: %v w/ err %v. Took %s", users, err, finish)
}

func CronFetchHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	q := datastore.NewQuery("Queue").KeysOnly()
	keys, _ := q.GetAll(c, nil)
	db := DB{c}
    start := time.Now()

	for _, v := range keys {
		db.FetchUser(strconv.FormatInt(v.IntID(), 10))
	}

	finish := time.Since(start)

	c.Infof("Finished cron fetch, took %s", finish)

	fmt.Fprint(w, "fetching users")
}

func NotFoundHandler(w http.ResponseWriter, r *http.Request) {
	dir := path.Join(os.Getenv("PWD"), "templates")
	notFound := path.Join(dir, "404.html")
	layout := path.Join(dir, "pageLayout.html")
	data := map[string]string{"url": r.RequestURI}
	page := mustache.RenderFileInLayout(notFound, layout, data)
	w.WriteHeader(404)
	fmt.Fprint(w, page)
}

func StartupHandler(w http.ResponseWriter, r *http.Request) {
	if len(Config) == 0 {
		c := appengine.NewContext(r)
		client := &http.Client{
			Transport: &oauth2.Transport{
				Source: google.AppEngineTokenSource(c, storage.ScopeReadOnly),
				Base: &urlfetch.Transport{
					Context: c,
				},
			},
		}
		bucket, _ := file.DefaultBucketName(c)
		ctx := cloud.NewContext("davine-web", client)
		rc, err := storage.NewReader(ctx, bucket, "config.yaml")
		if err != nil {
			c.Errorf("error reading config: %v", err.Error())
			return
		}
		configFile, err := ioutil.ReadAll(rc)
		rc.Close()
		if err != nil {
			c.Errorf("error reading config: %v", err.Error())
			return
		}

		c.Infof("loaded config file: %v", configFile)
		yaml.Unmarshal(configFile, &Config)
		c.Infof("loaded config struct: %v", Config)
	}
}
<commit_msg>Assuring Config is loaded before export.<commit_after>package main

import (
	"appengine"
	"appengine/datastore"
	"appengine/file"
	"appengine/urlfetch"
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/hoisie/mustache"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/cloud"
	"google.golang.org/cloud/storage"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

func UserFetchHandler(w http.ResponseWriter, r *http.Request) {
	dir := path.Join(os.Getenv("PWD"), "templates")
	profile := path.Join(dir, "profile.html")
	layout := path.Join(dir, "profileLayout.html")
	vars := mux.Vars(r)
	c := appengine.NewContext(r)

	db := DB{c}
	match, _ := regexp.MatchString("^[0-9]+$", vars["user"])
	var err error
	var userMetaTemp interface{}
	var storedUserData interface{}
	var data string

	if match {
		userId, _ := strconv.ParseInt(vars["user"], 10, 64)
		userMetaTemp, err = db.GetUserMeta(userId)
		storedUserData, _ = db.GetUserData(userId)
	} else {
		temp := []StoredUserMeta{}
		q := datastore.NewQuery("UserMeta").Filter("VanityUrl =", strings.ToLower(vars["user"])).Limit(1)
		k, _ := q.GetAll(c, &temp)
		if len(temp) > 0 {
			userMetaTemp = temp[0]
			storedUserData, _ = db.GetUserData(k[0].IntID())
		} else {
			user404 := path.Join(dir, "user404.html")
			userData := map[string]string{"user": vars["user"]}
			data = mustache.RenderFileInLayout(user404, layout, userData)
			w.WriteHeader(http.StatusNotFound)
		}
	}

	if err == datastore.ErrNoSuchEntity {
		user404 := path.Join(dir, "user404.html")
		userData := map[string]string{"user": vars["user"]}
		data = mustache.RenderFileInLayout(user404, layout, userData)
		w.WriteHeader(http.StatusNotFound)
	} else if err != nil {
		fmt.Fprint(w, err.Error())
	} else if userMetaTemp != nil {

		userMeta := userMetaTemp.(StoredUserMeta)

		userData := map[string]interface{}{
			"username":    userMeta.Username,
			"userId":      userMeta.UserId,
			"description": userMeta.Description,
			"location":    userMeta.Location,
			"avatarUrl":   userMeta.AvatarUrl,
			"loops":       strconv.FormatInt(userMeta.Current.Loops, 10),
			"followers":   strconv.FormatInt(userMeta.Current.Followers, 10),
			"data":        storedUserData,
			"previous":    userMeta.Previous,
		}

		if userMeta.Background != "" {
			color := strings.SplitAfterN(userMeta.Background, "0x", 2)
			userData["profileBackground"] = color[1]
		} else {
			userData["profileBackground"] = "00BF8F"
		}

		data = mustache.RenderFileInLayout(profile, layout, userData)
	}

	fmt.Fprint(w, data)
}

func UserStoreHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vineApi := VineRequest{c}
	db := DB{c}
	_, err := GetQueuedUser(r.FormValue("id"), c)
	data := make(map[string]bool)

	if err != datastore.ErrNoSuchEntity && err != nil {
		c.Errorf("got UserStore err: %v", err)
	}

	user, apiErr := vineApi.GetUser(r.FormValue("id"))

	if err == datastore.ErrNoSuchEntity {

		if apiErr != nil {
			data["exists"] = false
			data["queued"] = false
		} else {
			QueueUser(user.UserIdStr, c)
			data["exists"] = true
			data["queued"] = true
		}

		data["stored"] = false

	} else {
		_, err := db.GetUserMeta(user.UserId)
		if err == datastore.ErrNoSuchEntity {
			data["stored"] = false
		} else {
			data["stored"] = true
		}
		data["exists"] = true
		data["queued"] = false
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(data)
}

func AboutHandler(w http.ResponseWriter, r *http.Request) {
	dir := path.Join(os.Getenv("PWD"), "templates")
	aboutPage := path.Join(dir, "about.html")
	layout := path.Join(dir, "pageLayout.html")

	db := DB{appengine.NewContext(r)}
	totalUsers, _ := db.GetTotalUsers()
	stats := map[string]interface{}{"users": totalUsers}
	stats["lastUpdated"] = db.GetLastUpdated()
	data := mustache.RenderFileInLayout(aboutPage, layout, stats)

	fmt.Fprint(w, data)
}

func DiscoverHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vineApi := VineRequest{c}
	db := DB{c}
	var recentUsers []*VineUser
	var recentVerified []StoredUserMeta

	recent := datastore.NewQuery("Queue").Order("-Discovered").Limit(5).KeysOnly()
	k, _ := recent.GetAll(c, nil)
	for i, _ := range k {
		user, err := vineApi.GetUser(strconv.FormatInt(k[i].IntID(), 10))
		if err == nil {
			recentUsers = append(recentUsers, user)
		}
	}
	verified := datastore.NewQuery("UserMeta").Filter("Verified =", true).Limit(5).KeysOnly()
	k, _ = verified.GetAll(c, nil)
	for i, _ := range k {
		user, err := db.GetUserMeta(k[i].IntID())
		if err == nil {
			recentVerified = append(recentVerified, user.(StoredUserMeta))
		}
	}
	data := map[string]interface{}{"recentUsers": recentUsers, "recentVerified": recentVerified}
	dir := path.Join(os.Getenv("PWD"), "templates")
	discover := path.Join(dir, "discover.html")
	layout := path.Join(dir, "pageLayout.html")
	page := mustache.RenderFileInLayout(discover, layout, data)
	fmt.Fprint(w, page)
}

func TopHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	db := DB{c}

	dir := path.Join(os.Getenv("PWD"), "templates")
	top := path.Join(dir, "top.html")
	layout := path.Join(dir, "pageLayout.html")
	data := db.GetTop()
	data["LastUpdated"] = db.GetLastUpdated()
	page := mustache.RenderFileInLayout(top, layout, data)
	fmt.Fprint(w, page)
}

func RandomHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	q := datastore.NewQuery("UserMeta").KeysOnly()
	keys, err := q.GetAll(c, nil)
    if err != nil {
        c.Errorf("got err %v", err)
        return
    }
	randomKey := RandomKey(keys)
	var user QueuedUser
	key := datastore.NewKey(c, "Queue", "", randomKey.IntID(), nil)
	err = datastore.Get(c, key, &user)
	if err != nil {
		c.Errorf("got err %v", err)
	} else {
		c.Infof("got user %v", user)
	}
	http.Redirect(w, r, "/u/"+user.UserID, 301)
}

func SearchHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	dir := path.Join(os.Getenv("PWD"), "templates")
	search := path.Join(dir, "search.html")
	layout := path.Join(dir, "pageLayout.html")
	data := map[string]interface{}{
		"query": r.FormValue("q"),
		"count": 0,
	}
	if len(r.FormValue("q")) > 0 {
		results, err := SearchUsers(c, r.FormValue("q"))
		if err != nil {
			c.Errorf("got err on search: %v", err)
		}

		switch r.FormValue("s") {
		case "overall":
			sort.Sort(ByOverall(results))
			break
		case "followers":
			sort.Sort(ByFollowers(results))
			break
		case "loops":
			sort.Sort(ByLoops(results))
			break
		case "posts":
			sort.Sort(ByPosts(results))
			break
		case "revines":
			sort.Sort(ByRevines(results))
			break
		}

		if r.Method == "GET" {
			data["count"] = len(results)
			data["results"] = results
		} else if r.Method == "POST" {
			jsonData, _ := json.Marshal(results)
			fmt.Fprint(w, string(jsonData))
			return
		}
	}

	page := mustache.RenderFileInLayout(search, layout, data)
	fmt.Fprint(w, page)
}

func DonateHandler(w http.ResponseWriter, r *http.Request) {
	dir := path.Join(os.Getenv("PWD"), "templates")
	donate := path.Join(dir, "donate.html")
	layout := path.Join(dir, "pageLayout.html")
	page := mustache.RenderFileInLayout(donate, layout, nil)
	fmt.Fprint(w, page)
}

func ExportHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	db := DB{c}
	vars := mux.Vars(r)

	if r.Method == "GET" {
	    StartupHandler(w, r)
		userId, err := strconv.ParseInt(vars["user"], 10, 64)
		if err != nil {
			c.Errorf("got err: %v", err)
			http.Redirect(w, r, "/404", 301)
			return
		}
		userMeta, err := db.GetUserMeta(userId)
		if err == datastore.ErrNoSuchEntity {
			http.Redirect(w, r, "/404", 301)
			return
		}
		user := userMeta.(StoredUserMeta)
		data := map[string]string{"username": user.Username, "userId": vars["user"], "captcha": Config["captchaPublic"]}
		dir := path.Join(os.Getenv("PWD"), "templates")
		export := path.Join(dir, "export.html")
		layout := path.Join(dir, "pageLayout.html")
		page := mustache.RenderFileInLayout(export, layout, data)
		fmt.Fprint(w, page)
	} else if r.Method == "POST" {
		client := urlfetch.Client(c)
		url := "https://www.google.com/recaptcha/api/siteverify?secret=" + Config["captchaPrivate"]
		url += "&response=" + r.FormValue("g-recaptcha-response") + "&remoteip=" + r.RemoteAddr
		req, _ := http.NewRequest("GET", url, nil)
		resp, err := client.Do(req)
		if err != nil {
			c.Errorf("got err: %v", err)
			return
		}
		body, _ := ioutil.ReadAll(resp.Body)
		var data map[string]interface{}
		json.Unmarshal(body, &data)
		if data["success"].(bool) {
			export := Export{c}
			export.User(vars["user"], w)
		} else {
			fmt.Fprint(w, "Seems like your CAPTCHA was wrong. Please press back and try again.")
		}
	}
}

func PopularFetchHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vineApi := VineRequest{c}
    start := time.Now()
    users := make(chan string, 60)

	err := vineApi.GetPopularUsers(users, cap(users))
	for v := range users {
		if _, err := GetQueuedUser(v, c); err == datastore.ErrNoSuchEntity {
			QueueUser(v, c)
		}
	}

    finish := time.Since(start)
	fmt.Fprint(w, "queuing popular users: %v w/ err %v", users, err)
	c.Infof("queueing popular users: %v w/ err %v. Took %s", users, err, finish)
}

func CronFetchHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	q := datastore.NewQuery("Queue").KeysOnly()
	keys, _ := q.GetAll(c, nil)
	db := DB{c}
    start := time.Now()

	for _, v := range keys {
		db.FetchUser(strconv.FormatInt(v.IntID(), 10))
	}

	finish := time.Since(start)

	c.Infof("Finished cron fetch, took %s", finish)

	fmt.Fprint(w, "fetching users")
}

func NotFoundHandler(w http.ResponseWriter, r *http.Request) {
	dir := path.Join(os.Getenv("PWD"), "templates")
	notFound := path.Join(dir, "404.html")
	layout := path.Join(dir, "pageLayout.html")
	data := map[string]string{"url": r.RequestURI}
	page := mustache.RenderFileInLayout(notFound, layout, data)
	w.WriteHeader(404)
	fmt.Fprint(w, page)
}

func StartupHandler(w http.ResponseWriter, r *http.Request) {
	if len(Config) == 0 {
		c := appengine.NewContext(r)
		client := &http.Client{
			Transport: &oauth2.Transport{
				Source: google.AppEngineTokenSource(c, storage.ScopeReadOnly),
				Base: &urlfetch.Transport{
					Context: c,
				},
			},
		}
		bucket, _ := file.DefaultBucketName(c)
		ctx := cloud.NewContext("davine-web", client)
		rc, err := storage.NewReader(ctx, bucket, "config.yaml")
		if err != nil {
			c.Errorf("error reading config: %v", err.Error())
			return
		}
		configFile, err := ioutil.ReadAll(rc)
		rc.Close()
		if err != nil {
			c.Errorf("error reading config: %v", err.Error())
			return
		}

		c.Infof("loaded config file: %v", configFile)
		yaml.Unmarshal(configFile, &Config)
		c.Infof("loaded config struct: %v", Config)
	}
}
<|endoftext|>