<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package simplepush

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/goprotobuf/proto"
	"github.com/mozilla-services/heka/message"
	"github.com/mozilla-services/pushgo/id"
)

// A HekaFormat specifies the Heka log message format. The binary Protobuf
// format is more efficient, and should be preferred over the JSON encoding
// when human readability is not a concern.
type HekaFormat int

// Heka message formats.
const (
	HekaProtobuf HekaFormat = iota
	HekaJSON
)

// A LogLevel represents a message log level.
type LogLevel int32

// syslog severity levels.
const (
	EMERGENCY LogLevel = iota
	ALERT
	CRITICAL
	ERROR
	WARNING
	NOTICE
	INFO
	DEBUG
)

var levelNames = map[LogLevel]string{
	EMERGENCY: "EMERGENCY",
	ALERT:     "ALERT",
	CRITICAL:  "CRITICAL",
	ERROR:     "ERROR",
	WARNING:   "WARNING",
	NOTICE:    "NOTICE",
	INFO:      "INFO",
	DEBUG:     "DEBUG",
}

func (l LogLevel) String() string {
	return levelNames[l]
}

const (
	HeaderID      = "X-Request-Id"
	TextLogTime   = "2006-01-02 15:04:05 -0700"
	CommonLogTime = "02/Jan/2006:15:04:05 -0700"
)

type LogFields map[string]string

type Logger interface {
	HasConfigStruct
	Log(level LogLevel, messageType, payload string, fields LogFields) error
	SetFilter(level LogLevel)
	ShouldLog(level LogLevel) bool
}

var AvailableLoggers = make(AvailableExtensions)

type SimpleLogger struct {
	Logger
}

// Error string helper that ignores nil errors
func ErrStr(err error) string {
	if err == nil {
		return ""
	}
	return err.Error()
}

// Attempt to convert an interface to a string, ignore failures
func IStr(i interface{}) (reply string) {
	if i == nil {
		return ""
	}
	if reply, ok := i.(string); ok {
		return reply
	}
	return "Undefined"
}

// SimplePush Logger implementation, utilizes the passed in Logger
func NewLogger(log Logger) (*SimpleLogger, error) {
	return &SimpleLogger{log}, nil
}

// Default logging calls for convenience
func (sl *SimpleLogger) Debug(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(DEBUG, mtype, msg, fields)
}

func (sl *SimpleLogger) Info(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(INFO, mtype, msg, fields)
}

func (sl *SimpleLogger) Warn(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(WARNING, mtype, msg, fields)
}

func (sl *SimpleLogger) Error(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(ERROR, mtype, msg, fields)
}

func (sl *SimpleLogger) Critical(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(CRITICAL, mtype, msg, fields)
}

func (sl *SimpleLogger) Alert(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(ALERT, mtype, msg, fields)
}

// NewHekaLogger creates a logger that writes Protobuf or JSON-encoded log
// messages to standard output.
func NewHekaLogger(format HekaFormat) *HekaLogger {
	return &HekaLogger{format: format, writer: os.Stdout}
}

type HekaLoggerConfig struct {
	Name       string
	EnvVersion string `toml:"env_version"`
	Filter     int32
}

type HekaLogger struct {
	pid        int32
	logName    string
	envVersion string
	hostname   string
	filter     LogLevel
	format     HekaFormat
	writer     io.Writer
}

func (hl *HekaLogger) ConfigStruct() interface{} {
	return &HekaLoggerConfig{
		Name:       fmt.Sprintf("push-%s", VERSION),
		EnvVersion: "1",
		Filter:     0,
	}
}

func (hl *HekaLogger) Init(app *Application, config interface{}) (err error) {
	conf := config.(*HekaLoggerConfig)
	hl.pid = int32(os.Getpid())
	hl.hostname = app.Hostname()
	hl.logName = conf.Name
	hl.envVersion = conf.EnvVersion
	hl.filter = LogLevel(conf.Filter)
	return
}

func (hl *HekaLogger) ShouldLog(level LogLevel) bool {
	return level <= hl.filter
}

func (hl *HekaLogger) SetFilter(level LogLevel) {
	hl.filter = level
}

func (hl *HekaLogger) writeMessage(m *message.Message) (err error) {
	switch hl.format {
	case HekaProtobuf:
		var data []byte
		if data, err = proto.Marshal(m); err != nil {
			return
		}
		_, err = hl.writer.Write(data)

	case HekaJSON:
		encoder := json.NewEncoder(hl.writer)
		err = encoder.Encode(m)
	}
	return
}

func (hl *HekaLogger) Log(level LogLevel, messageType, payload string, fields LogFields) (err error) {
	if !hl.ShouldLog(level) {
		return
	}
	messageID, _ := id.GenerateBytes()
	m := new(message.Message)
	m.SetUuid(messageID)
	m.SetTimestamp(time.Now().UnixNano())
	m.SetType(messageType)
	m.SetLogger(hl.logName)
	m.SetSeverity(int32(level))
	m.SetPayload(payload)
	m.SetEnvVersion(hl.envVersion)
	m.SetPid(hl.pid)
	m.SetHostname(hl.hostname)
	for name, value := range fields {
		message.NewStringField(m, name, value)
	}
	if err = hl.writeMessage(m); err != nil {
		log.Fatalf("Error writing log message: %s", err)
	}
	return
}

// NewTextLogger creates a logger that writes human-readable log messages to
// standard error.
func NewTextLogger() *TextLogger {
	return &TextLogger{writer: os.Stderr}
}

type TextLoggerConfig struct {
	Filter int32
	Trace  bool
}

type TextLogger struct {
	pid      int32
	hostname string
	filter   LogLevel
	writer   io.Writer
}

func (tl *TextLogger) ConfigStruct() interface{} {
	return &TextLoggerConfig{
		Filter: 0,
		Trace:  false,
	}
}

func (tl *TextLogger) Init(app *Application, config interface{}) (err error) {
	conf := config.(*TextLoggerConfig)
	tl.pid = int32(os.Getpid())
	tl.hostname = app.Hostname()
	tl.filter = LogLevel(conf.Filter)
	return
}

func (tl *TextLogger) ShouldLog(level LogLevel) bool {
	return level <= tl.filter
}

func (tl *TextLogger) SetFilter(level LogLevel) {
	tl.filter = level
}

func (tl *TextLogger) Log(level LogLevel, messageType, payload string, fields LogFields) (err error) {
	// Return ASAP if we shouldn't be logging
	if !tl.ShouldLog(level) {
		return
	}

	reply := new(bytes.Buffer)
	fmt.Fprintf(reply, "%s [% 8s] %s %s", time.Now().Format(TextLogTime),
		level, messageType, strconv.Quote(payload))

	if len(fields) > 0 {
		reply.WriteByte(' ')
		i := 0
		for key, val := range fields {
			fmt.Fprintf(reply, "%s:%s", key, val)
			if i < len(fields)-1 {
				reply.WriteString(", ")
			}
			i++
		}
	}

	reply.WriteByte('\n')
	_, err = reply.WriteTo(tl.writer)

	return
}

// logResponseWriter wraps an http.ResponseWriter, recording its status code,
// content length, and response time.
type logResponseWriter struct {
	http.ResponseWriter
	wroteHeader   bool
	StatusCode    int
	ContentLength int
	RespondedAt   time.Time
}

// writerOnly hides the optional ReadFrom method of an io.Writer from io.Copy.
// Taken from package net/http, copyright 2009, The Go Authors.
type writerOnly struct {
	io.Writer
}

// ReadFrom calls the ReadFrom method of the underlying ResponseWriter. Defined
// for compatibility with *response.ReadFrom from package net/http.
func (w *logResponseWriter) ReadFrom(r io.Reader) (int64, error) {
	if dest, ok := w.ResponseWriter.(io.ReaderFrom); ok {
		return dest.ReadFrom(r)
	}
	return io.Copy(writerOnly{w}, r)
}

// WriteHeader records the response time and status code, then calls the
// WriteHeader method of the underlying ResponseWriter.
func (w *logResponseWriter) WriteHeader(statusCode int) {
	if !w.wroteHeader {
		w.wroteHeader = true
		w.setStatus(statusCode)
	}
	w.ResponseWriter.WriteHeader(statusCode)
}

// setStatus records the response time and status code.
func (w *logResponseWriter) setStatus(statusCode int) {
	w.RespondedAt = time.Now()
	w.StatusCode = statusCode
}

// Write calls the Write method of the underlying ResponseWriter and records
// the number of bytes written.
func (w *logResponseWriter) Write(data []byte) (n int, err error) {
	if !w.wroteHeader {
		w.WriteHeader(http.StatusOK)
	}
	n, err = w.ResponseWriter.Write(data)
	w.ContentLength += len(data)
	return
}

// Hijack calls the Hijack method of the underlying ResponseWriter, allowing a
// custom protocol handler (e.g., WebSockets) to take over the connection.
func (w *logResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	if hj, ok := w.ResponseWriter.(http.Hijacker); ok {
		return hj.Hijack()
	}
	return nil, nil, io.EOF
}

// CloseNotify calls the CloseNotify method of the underlying ResponseWriter,
// or returns a nil channel if the operation is not supported.
func (w *logResponseWriter) CloseNotify() <-chan bool {
	if cn, ok := w.ResponseWriter.(http.CloseNotifier); ok {
		return cn.CloseNotify()
	}
	return nil
}

// Flush calls the Flush method of the underlying ResponseWriter, recording a
// successful response if WriteHeader was not called.
func (w *logResponseWriter) Flush() {
	if !w.wroteHeader {
		w.wroteHeader = true
		w.setStatus(http.StatusOK)
	}
	if f, ok := w.ResponseWriter.(http.Flusher); ok {
		f.Flush()
	}
}

// LogHandler logs the result of an HTTP request.
type LogHandler struct {
	http.Handler
	Log *SimpleLogger
}

// formatRequest generates a Common Log Format request line.
func (h *LogHandler) formatRequest(writer *logResponseWriter, req *http.Request, remoteAddrs []string) string {
	remoteAddr := "-"
	if len(remoteAddrs) > 0 {
		remoteAddr = remoteAddrs[0]
	}
	requestInfo := fmt.Sprintf("%s %s %s", req.Method, req.URL.Path, req.Proto)
	return fmt.Sprintf(`%s - - [%s] %s %d %d`, remoteAddr, writer.RespondedAt.Format(CommonLogTime),
		strconv.Quote(requestInfo), writer.StatusCode, writer.ContentLength)
}

// logResponse logs a response to an HTTP request.
func (h *LogHandler) logResponse(writer *logResponseWriter, req *http.Request, requestID string, receivedAt time.Time) {
	if !h.Log.ShouldLog(INFO) {
		return
	}
	forwardedFor := req.Header[http.CanonicalHeaderKey("X-Forwarded-For")]
	remoteAddrs := make([]string, len(forwardedFor)+1)
	copy(remoteAddrs, forwardedFor)
	remoteAddrs[len(remoteAddrs)-1], _, _ = net.SplitHostPort(req.RemoteAddr)
	h.Log.Info("http", h.formatRequest(writer, req, remoteAddrs), LogFields{
		"rid":                requestID,
		"agent":              req.Header.Get("User-Agent"),
		"path":               req.URL.Path,
		"method":             req.Method,
		"code":               strconv.Itoa(writer.StatusCode),
		"remoteAddressChain": fmt.Sprintf("[%s]", strings.Join(remoteAddrs, ", ")),
		"t":                  strconv.FormatInt(int64(writer.RespondedAt.Sub(receivedAt)/time.Millisecond), 10)})
}

// ServeHTTP implements http.Handler.ServeHTTP.
func (h *LogHandler) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	receivedAt := time.Now()

	// The `X-Request-Id` header is used by Heroku, restify, etc. to correlate
	// logs for the same request.
	requestID := req.Header.Get(HeaderID)
	if !id.Valid(requestID) {
		requestID, _ = id.Generate()
		req.Header.Set(HeaderID, requestID)
	}

	writer := &logResponseWriter{ResponseWriter: res, StatusCode: http.StatusOK}
	defer h.logResponse(writer, req, requestID, receivedAt)

	h.Handler.ServeHTTP(writer, req)
}

func init() {
	AvailableLoggers["protobuf"] = func() HasConfigStruct { return NewHekaLogger(HekaProtobuf) }
	AvailableLoggers["json"] = func() HasConfigStruct { return NewHekaLogger(HekaJSON) }
	AvailableLoggers["text"] = func() HasConfigStruct { return NewTextLogger() }
	AvailableLoggers["default"] = AvailableLoggers["text"]
}
<commit_msg>Add `SimpleLogger.{Notice, Panic}()`.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package simplepush

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/goprotobuf/proto"
	"github.com/mozilla-services/heka/message"
	"github.com/mozilla-services/pushgo/id"
)

// A HekaFormat specifies the Heka log message format. The binary Protobuf
// format is more efficient, and should be preferred over the JSON encoding
// when human readability is not a concern.
type HekaFormat int

// Heka message formats.
const (
	HekaProtobuf HekaFormat = iota
	HekaJSON
)

// A LogLevel represents a message log level.
type LogLevel int32

// syslog severity levels.
const (
	EMERGENCY LogLevel = iota
	ALERT
	CRITICAL
	ERROR
	WARNING
	NOTICE
	INFO
	DEBUG
)

var levelNames = map[LogLevel]string{
	EMERGENCY: "EMERGENCY",
	ALERT:     "ALERT",
	CRITICAL:  "CRITICAL",
	ERROR:     "ERROR",
	WARNING:   "WARNING",
	NOTICE:    "NOTICE",
	INFO:      "INFO",
	DEBUG:     "DEBUG",
}

func (l LogLevel) String() string {
	return levelNames[l]
}

const (
	HeaderID      = "X-Request-Id"
	TextLogTime   = "2006-01-02 15:04:05 -0700"
	CommonLogTime = "02/Jan/2006:15:04:05 -0700"
)

type LogFields map[string]string

type Logger interface {
	HasConfigStruct
	Log(level LogLevel, messageType, payload string, fields LogFields) error
	SetFilter(level LogLevel)
	ShouldLog(level LogLevel) bool
}

var AvailableLoggers = make(AvailableExtensions)

type SimpleLogger struct {
	Logger
}

// Error string helper that ignores nil errors
func ErrStr(err error) string {
	if err == nil {
		return ""
	}
	return err.Error()
}

// Attempt to convert an interface to a string, ignore failures
func IStr(i interface{}) (reply string) {
	if i == nil {
		return ""
	}
	if reply, ok := i.(string); ok {
		return reply
	}
	return "Undefined"
}

// SimplePush Logger implementation, utilizes the passed in Logger
func NewLogger(log Logger) (*SimpleLogger, error) {
	return &SimpleLogger{log}, nil
}

// Default logging calls for convenience
func (sl *SimpleLogger) Debug(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(DEBUG, mtype, msg, fields)
}

func (sl *SimpleLogger) Info(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(INFO, mtype, msg, fields)
}

func (sl *SimpleLogger) Notice(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(NOTICE, mtype, msg, fields)
}

func (sl *SimpleLogger) Warn(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(WARNING, mtype, msg, fields)
}

func (sl *SimpleLogger) Error(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(ERROR, mtype, msg, fields)
}

func (sl *SimpleLogger) Critical(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(CRITICAL, mtype, msg, fields)
}

func (sl *SimpleLogger) Alert(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(ALERT, mtype, msg, fields)
}

func (sl *SimpleLogger) Panic(mtype, msg string, fields LogFields) error {
	return sl.Logger.Log(EMERGENCY, mtype, msg, fields)
}

// NewHekaLogger creates a logger that writes Protobuf or JSON-encoded log
// messages to standard output.
func NewHekaLogger(format HekaFormat) *HekaLogger {
	return &HekaLogger{format: format, writer: os.Stdout}
}

type HekaLoggerConfig struct {
	Name       string
	EnvVersion string `toml:"env_version"`
	Filter     int32
}

type HekaLogger struct {
	pid        int32
	logName    string
	envVersion string
	hostname   string
	filter     LogLevel
	format     HekaFormat
	writer     io.Writer
}

func (hl *HekaLogger) ConfigStruct() interface{} {
	return &HekaLoggerConfig{
		Name:       fmt.Sprintf("push-%s", VERSION),
		EnvVersion: "1",
		Filter:     0,
	}
}

func (hl *HekaLogger) Init(app *Application, config interface{}) (err error) {
	conf := config.(*HekaLoggerConfig)
	hl.pid = int32(os.Getpid())
	hl.hostname = app.Hostname()
	hl.logName = conf.Name
	hl.envVersion = conf.EnvVersion
	hl.filter = LogLevel(conf.Filter)
	return
}

func (hl *HekaLogger) ShouldLog(level LogLevel) bool {
	return level <= hl.filter
}

func (hl *HekaLogger) SetFilter(level LogLevel) {
	hl.filter = level
}

func (hl *HekaLogger) writeMessage(m *message.Message) (err error) {
	switch hl.format {
	case HekaProtobuf:
		var data []byte
		if data, err = proto.Marshal(m); err != nil {
			return
		}
		_, err = hl.writer.Write(data)

	case HekaJSON:
		encoder := json.NewEncoder(hl.writer)
		err = encoder.Encode(m)
	}
	return
}

func (hl *HekaLogger) Log(level LogLevel, messageType, payload string, fields LogFields) (err error) {
	if !hl.ShouldLog(level) {
		return
	}
	messageID, _ := id.GenerateBytes()
	m := new(message.Message)
	m.SetUuid(messageID)
	m.SetTimestamp(time.Now().UnixNano())
	m.SetType(messageType)
	m.SetLogger(hl.logName)
	m.SetSeverity(int32(level))
	m.SetPayload(payload)
	m.SetEnvVersion(hl.envVersion)
	m.SetPid(hl.pid)
	m.SetHostname(hl.hostname)
	for name, value := range fields {
		message.NewStringField(m, name, value)
	}
	if err = hl.writeMessage(m); err != nil {
		log.Fatalf("Error writing log message: %s", err)
	}
	return
}

// NewTextLogger creates a logger that writes human-readable log messages to
// standard error.
func NewTextLogger() *TextLogger {
	return &TextLogger{writer: os.Stderr}
}

type TextLoggerConfig struct {
	Filter int32
	Trace  bool
}

type TextLogger struct {
	pid      int32
	hostname string
	filter   LogLevel
	writer   io.Writer
}

func (tl *TextLogger) ConfigStruct() interface{} {
	return &TextLoggerConfig{
		Filter: 0,
		Trace:  false,
	}
}

func (tl *TextLogger) Init(app *Application, config interface{}) (err error) {
	conf := config.(*TextLoggerConfig)
	tl.pid = int32(os.Getpid())
	tl.hostname = app.Hostname()
	tl.filter = LogLevel(conf.Filter)
	return
}

func (tl *TextLogger) ShouldLog(level LogLevel) bool {
	return level <= tl.filter
}

func (tl *TextLogger) SetFilter(level LogLevel) {
	tl.filter = level
}

func (tl *TextLogger) Log(level LogLevel, messageType, payload string, fields LogFields) (err error) {
	// Return ASAP if we shouldn't be logging
	if !tl.ShouldLog(level) {
		return
	}

	reply := new(bytes.Buffer)
	fmt.Fprintf(reply, "%s [% 8s] %s %s", time.Now().Format(TextLogTime),
		level, messageType, strconv.Quote(payload))

	if len(fields) > 0 {
		reply.WriteByte(' ')
		i := 0
		for key, val := range fields {
			fmt.Fprintf(reply, "%s:%s", key, val)
			if i < len(fields)-1 {
				reply.WriteString(", ")
			}
			i++
		}
	}

	reply.WriteByte('\n')
	_, err = reply.WriteTo(tl.writer)

	return
}

// logResponseWriter wraps an http.ResponseWriter, recording its status code,
// content length, and response time.
type logResponseWriter struct {
	http.ResponseWriter
	wroteHeader   bool
	StatusCode    int
	ContentLength int
	RespondedAt   time.Time
}

// writerOnly hides the optional ReadFrom method of an io.Writer from io.Copy.
// Taken from package net/http, copyright 2009, The Go Authors.
type writerOnly struct {
	io.Writer
}

// ReadFrom calls the ReadFrom method of the underlying ResponseWriter. Defined
// for compatibility with *response.ReadFrom from package net/http.
func (w *logResponseWriter) ReadFrom(r io.Reader) (int64, error) {
	if dest, ok := w.ResponseWriter.(io.ReaderFrom); ok {
		return dest.ReadFrom(r)
	}
	return io.Copy(writerOnly{w}, r)
}

// WriteHeader records the response time and status code, then calls the
// WriteHeader method of the underlying ResponseWriter.
func (w *logResponseWriter) WriteHeader(statusCode int) {
	if !w.wroteHeader {
		w.wroteHeader = true
		w.setStatus(statusCode)
	}
	w.ResponseWriter.WriteHeader(statusCode)
}

// setStatus records the response time and status code.
func (w *logResponseWriter) setStatus(statusCode int) {
	w.RespondedAt = time.Now()
	w.StatusCode = statusCode
}

// Write calls the Write method of the underlying ResponseWriter and records
// the number of bytes written.
func (w *logResponseWriter) Write(data []byte) (n int, err error) {
	if !w.wroteHeader {
		w.WriteHeader(http.StatusOK)
	}
	n, err = w.ResponseWriter.Write(data)
	w.ContentLength += len(data)
	return
}

// Hijack calls the Hijack method of the underlying ResponseWriter, allowing a
// custom protocol handler (e.g., WebSockets) to take over the connection.
func (w *logResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	if hj, ok := w.ResponseWriter.(http.Hijacker); ok {
		return hj.Hijack()
	}
	return nil, nil, io.EOF
}

// CloseNotify calls the CloseNotify method of the underlying ResponseWriter,
// or returns a nil channel if the operation is not supported.
func (w *logResponseWriter) CloseNotify() <-chan bool {
	if cn, ok := w.ResponseWriter.(http.CloseNotifier); ok {
		return cn.CloseNotify()
	}
	return nil
}

// Flush calls the Flush method of the underlying ResponseWriter, recording a
// successful response if WriteHeader was not called.
func (w *logResponseWriter) Flush() {
	if !w.wroteHeader {
		w.wroteHeader = true
		w.setStatus(http.StatusOK)
	}
	if f, ok := w.ResponseWriter.(http.Flusher); ok {
		f.Flush()
	}
}

// LogHandler logs the result of an HTTP request.
type LogHandler struct {
	http.Handler
	Log *SimpleLogger
}

// formatRequest generates a Common Log Format request line.
func (h *LogHandler) formatRequest(writer *logResponseWriter, req *http.Request, remoteAddrs []string) string {
	remoteAddr := "-"
	if len(remoteAddrs) > 0 {
		remoteAddr = remoteAddrs[0]
	}
	requestInfo := fmt.Sprintf("%s %s %s", req.Method, req.URL.Path, req.Proto)
	return fmt.Sprintf(`%s - - [%s] %s %d %d`, remoteAddr, writer.RespondedAt.Format(CommonLogTime),
		strconv.Quote(requestInfo), writer.StatusCode, writer.ContentLength)
}

// logResponse logs a response to an HTTP request.
func (h *LogHandler) logResponse(writer *logResponseWriter, req *http.Request, requestID string, receivedAt time.Time) {
	if !h.Log.ShouldLog(INFO) {
		return
	}
	forwardedFor := req.Header[http.CanonicalHeaderKey("X-Forwarded-For")]
	remoteAddrs := make([]string, len(forwardedFor)+1)
	copy(remoteAddrs, forwardedFor)
	remoteAddrs[len(remoteAddrs)-1], _, _ = net.SplitHostPort(req.RemoteAddr)
	h.Log.Info("http", h.formatRequest(writer, req, remoteAddrs), LogFields{
		"rid":                requestID,
		"agent":              req.Header.Get("User-Agent"),
		"path":               req.URL.Path,
		"method":             req.Method,
		"code":               strconv.Itoa(writer.StatusCode),
		"remoteAddressChain": fmt.Sprintf("[%s]", strings.Join(remoteAddrs, ", ")),
		"t":                  strconv.FormatInt(int64(writer.RespondedAt.Sub(receivedAt)/time.Millisecond), 10)})
}

// ServeHTTP implements http.Handler.ServeHTTP.
func (h *LogHandler) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	receivedAt := time.Now()

	// The `X-Request-Id` header is used by Heroku, restify, etc. to correlate
	// logs for the same request.
	requestID := req.Header.Get(HeaderID)
	if !id.Valid(requestID) {
		requestID, _ = id.Generate()
		req.Header.Set(HeaderID, requestID)
	}

	writer := &logResponseWriter{ResponseWriter: res, StatusCode: http.StatusOK}
	defer h.logResponse(writer, req, requestID, receivedAt)

	h.Handler.ServeHTTP(writer, req)
}

func init() {
	AvailableLoggers["protobuf"] = func() HasConfigStruct { return NewHekaLogger(HekaProtobuf) }
	AvailableLoggers["json"] = func() HasConfigStruct { return NewHekaLogger(HekaJSON) }
	AvailableLoggers["text"] = func() HasConfigStruct { return NewTextLogger() }
	AvailableLoggers["default"] = AvailableLoggers["text"]
}
<|endoftext|>


<commit_before>package bitfinex

import (
	"os"
	"strconv"
	"testing"
)

var APIKey = os.Getenv("BITFINEX_API_KEY")
var APISecret = os.Getenv("BITFINEX_API_SECRET")

var apiPublic = New("", "")
var apiPrivate = New(APIKey, APISecret)

func checkEnv(t *testing.T) {
	if APIKey == "" || APISecret == "" {
		t.Skip("Skipping test because because APIKey and/or APISecret env variables are not set")
	}
}

func TestTicker(t *testing.T) {
	// Test normal request
	ticker, err := apiPublic.Ticker("btcusd")
	if err != nil || ticker.LastPrice == 0 {
		t.Error("Failed: " + err.Error())
		return
	}

	// Test bad request,
	// which must return an error
	ticker, err = apiPublic.Ticker("random")
	if err == nil {
		t.Error("Failed")
		return
	}
}

func TestStats(t *testing.T) {
	// Test normal request
	stats, err := apiPublic.Stats("btcusd")
	if err != nil || len(stats) == 0 {
		t.Error("Failed: " + err.Error())
		return
	}

	// Test bad request,
	// which must return an error
	stats, err = apiPublic.Stats("random")
	if err == nil {
		t.Error("Failed")
		return
	}
}

func TestOrderbook(t *testing.T) {
	// Test normal request
	orderbook, err := apiPublic.Orderbook("btcusd", 2, 2, 1)
	if err != nil || len(orderbook.Asks) != 2 || len(orderbook.Bids) != 2 {
		t.Error("Failed: " + err.Error())
		return
	}
}

func TestLendbook(t *testing.T) {
	// Test normal request
	lendbook, err := apiPublic.Lendbook("btc", 2, 2)
	if err != nil || len(lendbook.Asks) != 2 || len(lendbook.Bids) != 2 {
		t.Error("Failed: " + err.Error())
		return
	}

	// Test bad request,
	// which must return an error
	lendbook, err = apiPublic.Lendbook("random", 2, 2)
	if err == nil {
		t.Error("Failed")
		return
	}
}

func TestMyTrades(t *testing.T) {
	checkEnv(t)

	// Test normal request
	tn := strconv.FormatInt(0, 10)
	mytrades, err := apiPrivate.MyTrades("BTCUSD", tn, 50)
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}

	if len(mytrades) == 0 {
		t.Log("No trades detected, please inspect")
		return
	}
}

func TestWalletBalances(t *testing.T) {
	checkEnv(t)

	balances, err := apiPrivate.WalletBalances()
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}

	if len(balances) == 0 {
		t.Log("No wallet balances detected, please inspect")
		return
	}

	t.Log("Detected wallet balances, please inspect:")
	for k, v := range balances {
		t.Log("\t" + k.Type + ": " + strconv.FormatFloat(v.Amount, 'f', -1, 64) +
			" (available: " + strconv.FormatFloat(v.Available, 'f', -1, 64) + ") " + k.Currency)

	}

}

func TestNewOffer(t *testing.T) {
	checkEnv(t)

	offer, err := apiPrivate.NewOffer("BTC", 0.2, 365.0, 2, LEND)
	if err != nil || offer.ID == 0 {
		t.Error("Failed: " + err.Error())
		return
	}

	t.Log("Placed a new offer of 0.2BTC @ 1%/day for 2 days with ID: " + strconv.Itoa(offer.ID) + ", please inspect")
}

func TestActiveOffers(t *testing.T) {
	checkEnv(t)

	offers, err := apiPrivate.ActiveOffers()
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}

	if len(offers) == 0 {
		t.Log("No active offers detected, please inspect")
		return
	}

	t.Log("Detected active offers, please inspect:")
	for _, o := range offers {
		t.Log("\t" + strconv.Itoa(o.ID) + ": " + strconv.FormatFloat(o.OriginalAmount, 'f', -1, 64) +
			o.Currency + " @ " + strconv.FormatFloat(o.Rate/365., 'f', -1, 64) + "%/day for " + strconv.Itoa(o.Period) + " days")
	}

}

func TestActiveCredits(t *testing.T) {
	checkEnv(t)

	credits, err := apiPrivate.ActiveCredits()
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}

	if len(credits) == 0 {
		t.Log("No active credits found, please inspect")
		return
	}

	t.Log("Detected active credits, please inspect:")
	for _, c := range credits {
		t.Log("\t" + strconv.Itoa(c.ID) + ": " + strconv.FormatFloat(c.Amount, 'f', -1, 64) +
			c.Currency + " @ " + strconv.FormatFloat(c.Rate/365., 'f', -1, 64) + "%/day for " + strconv.Itoa(c.Period) + " days")
	}

}

func TestCancelOffer(t *testing.T) {
	checkEnv(t)

	// Assuming TestActiveOffers has PASSED
	offers, err := apiPrivate.ActiveOffers()
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}

	if len(offers) == 0 {
		t.Log("No active offers, nothing to cancel, please inspect")
		return
	}

	t.Log("Cancelling offer # " + strconv.Itoa(offers[0].ID))
	err = apiPrivate.CancelOffer(offers[0].ID)
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}
}
<commit_msg>updated testing routines to represent recent BTC/USD rate changes<commit_after>package bitfinex

import (
	"os"
	"strconv"
	"testing"
)

var APIKey = os.Getenv("BITFINEX_API_KEY")
var APISecret = os.Getenv("BITFINEX_API_SECRET")

var apiPublic = New("", "")
var apiPrivate = New(APIKey, APISecret)

func checkEnv(t *testing.T) {
	if APIKey == "" || APISecret == "" {
		t.Skip("Skipping test because because APIKey and/or APISecret env variables are not set")
	}
}

func TestTicker(t *testing.T) {
	// Test normal request
	ticker, err := apiPublic.Ticker("btcusd")
	if err != nil || ticker.LastPrice == 0 {
		t.Error("Failed: " + err.Error())
		return
	}

	// Test bad request,
	// which must return an error
	ticker, err = apiPublic.Ticker("random")
	if err == nil {
		t.Error("Failed")
		return
	}
}

func TestStats(t *testing.T) {
	// Test normal request
	stats, err := apiPublic.Stats("btcusd")
	if err != nil || len(stats) == 0 {
		t.Error("Failed: " + err.Error())
		return
	}

	// Test bad request,
	// which must return an error
	stats, err = apiPublic.Stats("random")
	if err == nil {
		t.Error("Failed")
		return
	}
}

func TestOrderbook(t *testing.T) {
	// Test normal request
	orderbook, err := apiPublic.Orderbook("btcusd", 2, 2, 1)
	if err != nil || len(orderbook.Asks) != 2 || len(orderbook.Bids) != 2 {
		t.Error("Failed: " + err.Error())
		return
	}
}

func TestLendbook(t *testing.T) {
	// Test normal request
	lendbook, err := apiPublic.Lendbook("btc", 2, 2)
	if err != nil || len(lendbook.Asks) != 2 || len(lendbook.Bids) != 2 {
		t.Error("Failed: " + err.Error())
		return
	}

	// Test bad request,
	// which must return an error
	lendbook, err = apiPublic.Lendbook("random", 2, 2)
	if err == nil {
		t.Error("Failed")
		return
	}
}

func TestMyTrades(t *testing.T) {
	checkEnv(t)

	// Test normal request
	tn := strconv.FormatInt(0, 10)
	mytrades, err := apiPrivate.MyTrades("BTCUSD", tn, 50)
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}

	if len(mytrades) == 0 {
		t.Log("No BTC/USD trades detected, please inspect")
		return
	}
}

func TestWalletBalances(t *testing.T) {
	checkEnv(t)

	balances, err := apiPrivate.WalletBalances()
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}

	if len(balances) == 0 {
		t.Log("No wallet balances detected, please inspect")
		return
	}

	t.Log("Detected wallet balances, please inspect:")
	for k, v := range balances {
		t.Log("\t" + k.Type + ": " + strconv.FormatFloat(v.Amount, 'f', -1, 64) +
			" (available: " + strconv.FormatFloat(v.Available, 'f', -1, 64) + ") " + k.Currency)

	}

}

func TestNewOffer(t *testing.T) {
	checkEnv(t)

	offer, err := apiPrivate.NewOffer("BTC", 0.5, 365.0, 2, LEND)
	if err != nil || offer.ID == 0 {
		t.Error("Failed: " + err.Error())
		return
	}

	t.Log("Placed a new offer of 0.5BTC @ 1%/day for 2 days with ID: " + strconv.Itoa(offer.ID) + ", please inspect")
}

func TestActiveOffers(t *testing.T) {
	checkEnv(t)

	offers, err := apiPrivate.ActiveOffers()
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}

	if len(offers) == 0 {
		t.Log("No active offers detected, please inspect")
		return
	}

	t.Log("Detected active offers, please inspect:")
	for _, o := range offers {
		t.Log("\t" + strconv.Itoa(o.ID) + ": " + strconv.FormatFloat(o.OriginalAmount, 'f', -1, 64) +
			o.Currency + " @ " + strconv.FormatFloat(o.Rate/365., 'f', -1, 64) + "%/day for " + strconv.Itoa(o.Period) + " days")
	}

}

func TestActiveCredits(t *testing.T) {
	checkEnv(t)

	credits, err := apiPrivate.ActiveCredits()
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}

	if len(credits) == 0 {
		t.Log("No active credits found, please inspect")
		return
	}

	t.Log("Detected active credits, please inspect:")
	for _, c := range credits {
		t.Log("\t" + strconv.Itoa(c.ID) + ": " + strconv.FormatFloat(c.Amount, 'f', -1, 64) +
			c.Currency + " @ " + strconv.FormatFloat(c.Rate/365., 'f', -1, 64) + "%/day for " + strconv.Itoa(c.Period) + " days")
	}

}

func TestCancelOffer(t *testing.T) {
	checkEnv(t)

	// Assuming TestActiveOffers has PASSED
	offers, err := apiPrivate.ActiveOffers()
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}

	if len(offers) == 0 {
		t.Log("No active offers, nothing to cancel, please inspect")
		return
	}

	t.Log("Cancelling offer # " + strconv.Itoa(offers[0].ID))
	err = apiPrivate.CancelOffer(offers[0].ID)
	if err != nil {
		t.Error("Failed: " + err.Error())
		return
	}
}
<|endoftext|>


<commit_before>package cybergrab_test

import (
	"bitbucket/cybergrab"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"syscall"
	"testing"
)

type simpleCrawlPolicy struct{}

func (scp simpleCrawlPolicy) ShouldDownload(url string) bool {
	return strings.HasSuffix(url, "jpg") ||
		strings.HasSuffix(url, "gif") ||
		strings.HasSuffix(url, "pdg")
}

func (scp simpleCrawlPolicy) ShouldCrawl(url string) bool {
	return !scp.ShouldDownload(url)
}

/////////////////////////////////////////////////////////////

// TestMain is a setup function for the rest of the
// tests in this file.
func TestMain(m *testing.M) {
	// setup a signal handler so that "Ctrl-\" shows a stack-trace.
	go func() {
		sigs := make(chan os.Signal, 1)
		signal.Notify(sigs, syscall.SIGQUIT)
		buf := make([]byte, 1<<20)
		for {
			<-sigs
			runtime.Stack(buf, true)
			log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf)
		}
	}()

	os.Exit(m.Run())
}

func TestMaxGoRoutines(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, cybergrab.MAX_DOWNLOADS, cybergrab.MAX_WORKERS)
	if err == nil {
		spider.Crawl("http://www.salami.com")
	} else {
		fmt.Println(err)
	}
}

func TestMaxWorkers(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, 100, cybergrab.MAX_WORKERS)
	if err == nil {
		spider.Crawl("http://www.quickmeme.com")
	} else {
		fmt.Println(err)
	}
}

func TestMaxDownloads(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, cybergrab.MAX_DOWNLOADS, 1000)
	if err == nil {
		spider.Crawl("http://www.spam.com")
	} else {
		fmt.Println(err)
	}
}
<commit_msg>Fixed import path<commit_after>package cybergrab_test

import (
	"github.com/nightowlware/cybergrab"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"syscall"
	"testing"
)

type simpleCrawlPolicy struct{}

func (scp simpleCrawlPolicy) ShouldDownload(url string) bool {
	return strings.HasSuffix(url, "jpg") ||
		strings.HasSuffix(url, "gif") ||
		strings.HasSuffix(url, "pdg")
}

func (scp simpleCrawlPolicy) ShouldCrawl(url string) bool {
	return !scp.ShouldDownload(url)
}

/////////////////////////////////////////////////////////////

// TestMain is a setup function for the rest of the
// tests in this file.
func TestMain(m *testing.M) {
	// setup a signal handler so that "Ctrl-\" shows a stack-trace.
	go func() {
		sigs := make(chan os.Signal, 1)
		signal.Notify(sigs, syscall.SIGQUIT)
		buf := make([]byte, 1<<20)
		for {
			<-sigs
			runtime.Stack(buf, true)
			log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf)
		}
	}()

	os.Exit(m.Run())
}

func TestMaxGoRoutines(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, cybergrab.MAX_DOWNLOADS, cybergrab.MAX_WORKERS)
	if err == nil {
		spider.Crawl("http://www.salami.com")
	} else {
		fmt.Println(err)
	}
}

func TestMaxWorkers(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, 100, cybergrab.MAX_WORKERS)
	if err == nil {
		spider.Crawl("http://www.quickmeme.com")
	} else {
		fmt.Println(err)
	}
}

func TestMaxDownloads(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, cybergrab.MAX_DOWNLOADS, 1000)
	if err == nil {
		spider.Crawl("http://www.spam.com")
	} else {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

var (
	Config        = NewConfig()
	defaultRemote = "origin"
)

// FetchPruneConfig collects together the config options that control fetching and pruning
type FetchPruneConfig struct {
	// The number of days prior to current date for which (local) refs other than HEAD
	// will be fetched with --recent (default 7, 0 = only fetch HEAD)
	FetchRecentRefsDays int
	// Makes the FetchRecentRefsDays option apply to all remote refs as well (default false)
	FetchRecentRefsIncludeRemotes bool
	// number of days prior to latest commit on a ref that we'll fetch previous
	// LFS changes too (default 3, 0 = only fetch at ref)
	FetchRecentCommitsDays int
	// Number of days added to FetchRecent*; data outside combined window will be
	// deleted when prune is run. (default 3)
	PruneOffsetDays int
}

type Configuration struct {
	CurrentRemote         string
	httpClient            *HttpClient
	redirectingHttpClient *http.Client
	envVars               map[string]string
	isTracingHttp         bool
	isLoggingStats        bool

	loading           sync.Mutex // guards initialization of gitConfig and remotes
	gitConfig         map[string]string
	remotes           []string
	extensions        map[string]Extension
	fetchIncludePaths []string
	fetchExcludePaths []string
	fetchPruneConfig  *FetchPruneConfig
}

func NewConfig() *Configuration {
	c := &Configuration{
		CurrentRemote: defaultRemote,
		envVars:       make(map[string]string),
	}
	c.isTracingHttp = c.GetenvBool("GIT_CURL_VERBOSE", false)
	c.isLoggingStats = c.GetenvBool("GIT_LOG_STATS", false)
	return c
}

func (c *Configuration) Getenv(key string) string {
	if i, ok := c.envVars[key]; ok {
		return i
	}

	v := os.Getenv(key)
	c.envVars[key] = v
	return v
}

func (c *Configuration) Setenv(key, value string) error {
	// Check see if we have this in our cache, if so update it
	if _, ok := c.envVars[key]; ok {
		c.envVars[key] = value
	}

	// Now set in process
	return os.Setenv(key, value)
}

// GetenvBool parses a boolean environment variable and returns the result as a bool.
// If the environment variable is unset, empty, or if the parsing fails,
// the value of def (default) is returned instead.
func (c *Configuration) GetenvBool(key string, def bool) bool {
	s := c.Getenv(key)
	if len(s) == 0 {
		return def
	}

	b, err := strconv.ParseBool(s)
	if err != nil {
		return def
	}
	return b
}

func (c *Configuration) Endpoint() Endpoint {
	if url, ok := c.GitConfig("lfs.url"); ok {
		return NewEndpoint(url)
	}

	if len(c.CurrentRemote) > 0 && c.CurrentRemote != defaultRemote {
		if endpoint := c.RemoteEndpoint(c.CurrentRemote); len(endpoint.Url) > 0 {
			return endpoint
		}
	}

	return c.RemoteEndpoint(defaultRemote)
}

func (c *Configuration) ConcurrentTransfers() int {
	uploads := 3

	if v, ok := c.GitConfig("lfs.concurrenttransfers"); ok {
		n, err := strconv.Atoi(v)
		if err == nil && n > 0 {
			uploads = n
		}
	}

	return uploads
}

func (c *Configuration) BatchTransfer() bool {
	if v, ok := c.GitConfig("lfs.batch"); ok {
		if v == "true" || v == "" {
			return true
		}

		// Any numeric value except 0 is considered true
		if n, err := strconv.Atoi(v); err == nil && n != 0 {
			return true
		}
	}
	return false
}

// PrivateAccess will retrieve the access value and return true if
// the value is set to private. When a repo is marked as having private
// access, the http requests for the batch api will fetch the credentials
// before running, otherwise the request will run without credentials.
func (c *Configuration) PrivateAccess() bool {
	key := fmt.Sprintf("lfs.%s.access", c.Endpoint().Url)
	if v, ok := c.GitConfig(key); ok {
		if strings.ToLower(v) == "private" {
			return true
		}
	}
	return false
}

// SetPrivateAccess will set the private access flag in .git/config.
func (c *Configuration) SetPrivateAccess() {
	tracerx.Printf("setting repository access to private")
	key := fmt.Sprintf("lfs.%s.access", c.Endpoint().Url)
	configFile := filepath.Join(LocalGitDir, "config")
	git.Config.SetLocal(configFile, key, "private")

	// Modify the config cache because it's checked again in this process
	// without being reloaded.
	c.loading.Lock()
	c.gitConfig[key] = "private"
	c.loading.Unlock()
}

func (c *Configuration) FetchIncludePaths() []string {
	c.loadGitConfig()
	return c.fetchIncludePaths
}
func (c *Configuration) FetchExcludePaths() []string {
	c.loadGitConfig()
	return c.fetchExcludePaths
}

func (c *Configuration) RemoteEndpoint(remote string) Endpoint {
	if len(remote) == 0 {
		remote = defaultRemote
	}
	if url, ok := c.GitConfig("remote." + remote + ".lfsurl"); ok {
		return NewEndpoint(url)
	}

	if url, ok := c.GitConfig("remote." + remote + ".url"); ok {
		return NewEndpointFromCloneURL(url)
	}

	return Endpoint{}
}

func (c *Configuration) Remotes() []string {
	c.loadGitConfig()
	return c.remotes
}

func (c *Configuration) Extensions() map[string]Extension {
	c.loadGitConfig()
	return c.extensions
}

func (c *Configuration) GitConfig(key string) (string, bool) {
	c.loadGitConfig()
	value, ok := c.gitConfig[strings.ToLower(key)]
	return value, ok
}

func (c *Configuration) SetConfig(key, value string) {
	c.loadGitConfig()
	c.gitConfig[key] = value
}

func (c *Configuration) ObjectUrl(oid string) (*url.URL, error) {
	return ObjectUrl(c.Endpoint(), oid)
}

func (c *Configuration) FetchPruneConfig() *FetchPruneConfig {
	if c.fetchPruneConfig == nil {
		c.fetchPruneConfig = &FetchPruneConfig{
			FetchRecentRefsDays:           7,
			FetchRecentRefsIncludeRemotes: false,
			FetchRecentCommitsDays:        3,
			PruneOffsetDays:               3,
		}
		if v, ok := c.GitConfig("lfs.fetchrecentrefsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n > 0 {
				c.fetchPruneConfig.FetchRecentRefsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentrefsincluderemotes"); ok {

			if v == "true" || v == "" {
				c.fetchPruneConfig.FetchRecentRefsIncludeRemotes = true
			}

			// Any numeric value except 0 is considered true
			if n, err := strconv.Atoi(v); err == nil && n != 0 {
				c.fetchPruneConfig.FetchRecentRefsIncludeRemotes = true
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentcommitsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n > 0 {
				c.fetchPruneConfig.FetchRecentCommitsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.pruneoffsetdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n > 0 {
				c.fetchPruneConfig.PruneOffsetDays = n
			}
		}

	}
	return c.fetchPruneConfig
}

func (c *Configuration) loadGitConfig() {
	c.loading.Lock()
	defer c.loading.Unlock()

	if c.gitConfig != nil {
		return
	}

	uniqRemotes := make(map[string]bool)

	c.gitConfig = make(map[string]string)
	c.extensions = make(map[string]Extension)

	var output string
	listOutput, err := git.Config.List()
	if err != nil {
		panic(fmt.Errorf("Error listing git config: %s", err))
	}

	configFile := filepath.Join(LocalWorkingDir, ".gitconfig")
	fileOutput, err := git.Config.ListFromFile(configFile)
	if err != nil {
		panic(fmt.Errorf("Error listing git config from file: %s", err))
	}

	localConfig := filepath.Join(LocalGitDir, "config")
	localOutput, err := git.Config.ListFromFile(localConfig)
	if err != nil {
		panic(fmt.Errorf("Error listing git config from file %s", err))
	}

	output = fileOutput + "\n" + listOutput + "\n" + localOutput

	lines := strings.Split(output, "\n")
	for _, line := range lines {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}
		key := strings.ToLower(pieces[0])
		value := pieces[1]
		c.gitConfig[key] = value

		keyParts := strings.Split(key, ".")
		if len(keyParts) > 1 && keyParts[0] == "remote" {
			remote := keyParts[1]
			uniqRemotes[remote] = remote == "origin"
		} else if len(keyParts) == 4 && keyParts[0] == "lfs" && keyParts[1] == "extension" {
			name := keyParts[2]
			ext := c.extensions[name]
			switch keyParts[3] {
			case "clean":
				ext.Clean = value
			case "smudge":
				ext.Smudge = value
			case "priority":
				p, err := strconv.Atoi(value)
				if err == nil && p >= 0 {
					ext.Priority = p
				}
			}
			ext.Name = name
			c.extensions[name] = ext
		} else if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchinclude" {
			for _, inc := range strings.Split(value, ",") {
				inc = strings.TrimSpace(inc)
				c.fetchIncludePaths = append(c.fetchIncludePaths, inc)
			}
		} else if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchexclude" {
			for _, ex := range strings.Split(value, ",") {
				ex = strings.TrimSpace(ex)
				c.fetchExcludePaths = append(c.fetchExcludePaths, ex)
			}
		}
	}

	c.remotes = make([]string, 0, len(uniqRemotes))
	for remote, isOrigin := range uniqRemotes {
		if isOrigin {
			continue
		}
		c.remotes = append(c.remotes, remote)
	}
}
<commit_msg>Add lfs.fetchrecentalways config option<commit_after>package lfs

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

var (
	Config        = NewConfig()
	defaultRemote = "origin"
)

// FetchPruneConfig collects together the config options that control fetching and pruning
type FetchPruneConfig struct {
	// The number of days prior to current date for which (local) refs other than HEAD
	// will be fetched with --recent (default 7, 0 = only fetch HEAD)
	FetchRecentRefsDays int
	// Makes the FetchRecentRefsDays option apply to all remote refs as well (default false)
	FetchRecentRefsIncludeRemotes bool
	// number of days prior to latest commit on a ref that we'll fetch previous
	// LFS changes too (default 3, 0 = only fetch at ref)
	FetchRecentCommitsDays int
	// Whether to always fetch recent even without --recent
	FetchRecentAlways bool
	// Number of days added to FetchRecent*; data outside combined window will be
	// deleted when prune is run. (default 3)
	PruneOffsetDays int
}

type Configuration struct {
	CurrentRemote         string
	httpClient            *HttpClient
	redirectingHttpClient *http.Client
	envVars               map[string]string
	isTracingHttp         bool
	isLoggingStats        bool

	loading           sync.Mutex // guards initialization of gitConfig and remotes
	gitConfig         map[string]string
	remotes           []string
	extensions        map[string]Extension
	fetchIncludePaths []string
	fetchExcludePaths []string
	fetchPruneConfig  *FetchPruneConfig
}

func NewConfig() *Configuration {
	c := &Configuration{
		CurrentRemote: defaultRemote,
		envVars:       make(map[string]string),
	}
	c.isTracingHttp = c.GetenvBool("GIT_CURL_VERBOSE", false)
	c.isLoggingStats = c.GetenvBool("GIT_LOG_STATS", false)
	return c
}

func (c *Configuration) Getenv(key string) string {
	if i, ok := c.envVars[key]; ok {
		return i
	}

	v := os.Getenv(key)
	c.envVars[key] = v
	return v
}

func (c *Configuration) Setenv(key, value string) error {
	// Check see if we have this in our cache, if so update it
	if _, ok := c.envVars[key]; ok {
		c.envVars[key] = value
	}

	// Now set in process
	return os.Setenv(key, value)
}

// GetenvBool parses a boolean environment variable and returns the result as a bool.
// If the environment variable is unset, empty, or if the parsing fails,
// the value of def (default) is returned instead.
func (c *Configuration) GetenvBool(key string, def bool) bool {
	s := c.Getenv(key)
	if len(s) == 0 {
		return def
	}

	b, err := strconv.ParseBool(s)
	if err != nil {
		return def
	}
	return b
}

func (c *Configuration) Endpoint() Endpoint {
	if url, ok := c.GitConfig("lfs.url"); ok {
		return NewEndpoint(url)
	}

	if len(c.CurrentRemote) > 0 && c.CurrentRemote != defaultRemote {
		if endpoint := c.RemoteEndpoint(c.CurrentRemote); len(endpoint.Url) > 0 {
			return endpoint
		}
	}

	return c.RemoteEndpoint(defaultRemote)
}

func (c *Configuration) ConcurrentTransfers() int {
	uploads := 3

	if v, ok := c.GitConfig("lfs.concurrenttransfers"); ok {
		n, err := strconv.Atoi(v)
		if err == nil && n > 0 {
			uploads = n
		}
	}

	return uploads
}

func (c *Configuration) BatchTransfer() bool {
	if v, ok := c.GitConfig("lfs.batch"); ok {
		if v == "true" || v == "" {
			return true
		}

		// Any numeric value except 0 is considered true
		if n, err := strconv.Atoi(v); err == nil && n != 0 {
			return true
		}
	}
	return false
}

// PrivateAccess will retrieve the access value and return true if
// the value is set to private. When a repo is marked as having private
// access, the http requests for the batch api will fetch the credentials
// before running, otherwise the request will run without credentials.
func (c *Configuration) PrivateAccess() bool {
	key := fmt.Sprintf("lfs.%s.access", c.Endpoint().Url)
	if v, ok := c.GitConfig(key); ok {
		if strings.ToLower(v) == "private" {
			return true
		}
	}
	return false
}

// SetPrivateAccess will set the private access flag in .git/config.
func (c *Configuration) SetPrivateAccess() {
	tracerx.Printf("setting repository access to private")
	key := fmt.Sprintf("lfs.%s.access", c.Endpoint().Url)
	configFile := filepath.Join(LocalGitDir, "config")
	git.Config.SetLocal(configFile, key, "private")

	// Modify the config cache because it's checked again in this process
	// without being reloaded.
	c.loading.Lock()
	c.gitConfig[key] = "private"
	c.loading.Unlock()
}

func (c *Configuration) FetchIncludePaths() []string {
	c.loadGitConfig()
	return c.fetchIncludePaths
}
func (c *Configuration) FetchExcludePaths() []string {
	c.loadGitConfig()
	return c.fetchExcludePaths
}

func (c *Configuration) RemoteEndpoint(remote string) Endpoint {
	if len(remote) == 0 {
		remote = defaultRemote
	}
	if url, ok := c.GitConfig("remote." + remote + ".lfsurl"); ok {
		return NewEndpoint(url)
	}

	if url, ok := c.GitConfig("remote." + remote + ".url"); ok {
		return NewEndpointFromCloneURL(url)
	}

	return Endpoint{}
}

func (c *Configuration) Remotes() []string {
	c.loadGitConfig()
	return c.remotes
}

func (c *Configuration) Extensions() map[string]Extension {
	c.loadGitConfig()
	return c.extensions
}

func (c *Configuration) GitConfig(key string) (string, bool) {
	c.loadGitConfig()
	value, ok := c.gitConfig[strings.ToLower(key)]
	return value, ok
}

func (c *Configuration) SetConfig(key, value string) {
	c.loadGitConfig()
	c.gitConfig[key] = value
}

func (c *Configuration) ObjectUrl(oid string) (*url.URL, error) {
	return ObjectUrl(c.Endpoint(), oid)
}

func (c *Configuration) FetchPruneConfig() *FetchPruneConfig {
	if c.fetchPruneConfig == nil {
		c.fetchPruneConfig = &FetchPruneConfig{
			FetchRecentRefsDays:           7,
			FetchRecentRefsIncludeRemotes: false,
			FetchRecentCommitsDays:        3,
			PruneOffsetDays:               3,
		}
		if v, ok := c.GitConfig("lfs.fetchrecentrefsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n > 0 {
				c.fetchPruneConfig.FetchRecentRefsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentrefsincluderemotes"); ok {

			if v == "true" || v == "" {
				c.fetchPruneConfig.FetchRecentRefsIncludeRemotes = true
			}

			// Any numeric value except 0 is considered true
			if n, err := strconv.Atoi(v); err == nil && n != 0 {
				c.fetchPruneConfig.FetchRecentRefsIncludeRemotes = true
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentcommitsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n > 0 {
				c.fetchPruneConfig.FetchRecentCommitsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentalways"); ok {
			if v == "true" || v == "" {
				c.fetchPruneConfig.FetchRecentAlways = true
			}
		}
		if v, ok := c.GitConfig("lfs.pruneoffsetdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n > 0 {
				c.fetchPruneConfig.PruneOffsetDays = n
			}
		}

	}
	return c.fetchPruneConfig
}

func (c *Configuration) loadGitConfig() {
	c.loading.Lock()
	defer c.loading.Unlock()

	if c.gitConfig != nil {
		return
	}

	uniqRemotes := make(map[string]bool)

	c.gitConfig = make(map[string]string)
	c.extensions = make(map[string]Extension)

	var output string
	listOutput, err := git.Config.List()
	if err != nil {
		panic(fmt.Errorf("Error listing git config: %s", err))
	}

	configFile := filepath.Join(LocalWorkingDir, ".gitconfig")
	fileOutput, err := git.Config.ListFromFile(configFile)
	if err != nil {
		panic(fmt.Errorf("Error listing git config from file: %s", err))
	}

	localConfig := filepath.Join(LocalGitDir, "config")
	localOutput, err := git.Config.ListFromFile(localConfig)
	if err != nil {
		panic(fmt.Errorf("Error listing git config from file %s", err))
	}

	output = fileOutput + "\n" + listOutput + "\n" + localOutput

	lines := strings.Split(output, "\n")
	for _, line := range lines {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}
		key := strings.ToLower(pieces[0])
		value := pieces[1]
		c.gitConfig[key] = value

		keyParts := strings.Split(key, ".")
		if len(keyParts) > 1 && keyParts[0] == "remote" {
			remote := keyParts[1]
			uniqRemotes[remote] = remote == "origin"
		} else if len(keyParts) == 4 && keyParts[0] == "lfs" && keyParts[1] == "extension" {
			name := keyParts[2]
			ext := c.extensions[name]
			switch keyParts[3] {
			case "clean":
				ext.Clean = value
			case "smudge":
				ext.Smudge = value
			case "priority":
				p, err := strconv.Atoi(value)
				if err == nil && p >= 0 {
					ext.Priority = p
				}
			}
			ext.Name = name
			c.extensions[name] = ext
		} else if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchinclude" {
			for _, inc := range strings.Split(value, ",") {
				inc = strings.TrimSpace(inc)
				c.fetchIncludePaths = append(c.fetchIncludePaths, inc)
			}
		} else if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchexclude" {
			for _, ex := range strings.Split(value, ",") {
				ex = strings.TrimSpace(ex)
				c.fetchExcludePaths = append(c.fetchExcludePaths, ex)
			}
		}
	}

	c.remotes = make([]string, 0, len(uniqRemotes))
	for remote, isOrigin := range uniqRemotes {
		if isOrigin {
			continue
		}
		c.remotes = append(c.remotes, remote)
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/elasticbeanstalk"
)

func resourceAwsElasticBeanstalkConfigurationTemplate() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsElasticBeanstalkConfigurationTemplateCreate,
		Read:   resourceAwsElasticBeanstalkConfigurationTemplateRead,
		Update: resourceAwsElasticBeanstalkConfigurationTemplateUpdate,
		Delete: resourceAwsElasticBeanstalkConfigurationTemplateDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"application": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"environment_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"setting": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Computed: true,
				Elem:     resourceAwsElasticBeanstalkOptionSetting(),
				Set:      optionSettingValueHash,
			},
			"solution_stack_name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAwsElasticBeanstalkConfigurationTemplateCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticbeanstalkconn

	// Get the relevant properties
	name := d.Get("name").(string)
	appName := d.Get("application").(string)

	optionSettings := gatherOptionSettings(d)

	opts := elasticbeanstalk.CreateConfigurationTemplateInput{
		ApplicationName: aws.String(appName),
		TemplateName:    aws.String(name),
		OptionSettings:  optionSettings,
	}

	if attr, ok := d.GetOk("description"); ok {
		opts.Description = aws.String(attr.(string))
	}

	if attr, ok := d.GetOk("environment_id"); ok {
		opts.EnvironmentId = aws.String(attr.(string))
	}

	if attr, ok := d.GetOk("solution_stack_name"); ok {
		opts.SolutionStackName = aws.String(attr.(string))
	}

	log.Printf("[DEBUG] Elastic Beanstalk configuration template create opts: %s", opts)
	if _, err := conn.CreateConfigurationTemplate(&opts); err != nil {
		return fmt.Errorf("Error creating Elastic Beanstalk configuration template: %s", err)
	}

	d.SetId(name)

	return resourceAwsElasticBeanstalkConfigurationTemplateRead(d, meta)
}

func resourceAwsElasticBeanstalkConfigurationTemplateRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticbeanstalkconn

	log.Printf("[DEBUG] Elastic Beanstalk configuration template read: %s", d.Get("name").(string))

	resp, err := conn.DescribeConfigurationSettings(&elasticbeanstalk.DescribeConfigurationSettingsInput{
		TemplateName:    aws.String(d.Id()),
		ApplicationName: aws.String(d.Get("application").(string)),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "InvalidParameterValue" && strings.Contains(awsErr.Message(), "No Configuration Template named") {
				log.Printf("[WARN] No Configuration Template named (%s) found", d.Id())
				d.SetId("")
				return nil
			}
		}
		return err
	}

	if len(resp.ConfigurationSettings) != 1 {
		log.Printf("[DEBUG] Elastic Beanstalk unexpected describe configuration template response: %+v", resp)
		return fmt.Errorf("Error reading application properties: found %d applications, expected 1", len(resp.ConfigurationSettings))
	}

	d.Set("description", resp.ConfigurationSettings[0].Description)
	return nil
}

func resourceAwsElasticBeanstalkConfigurationTemplateUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticbeanstalkconn

	log.Printf("[DEBUG] Elastic Beanstalk configuration template update: %s", d.Get("name").(string))

	if d.HasChange("description") {
		if err := resourceAwsElasticBeanstalkConfigurationTemplateDescriptionUpdate(conn, d); err != nil {
			return err
		}
	}

	if d.HasChange("setting") {
		if err := resourceAwsElasticBeanstalkConfigurationTemplateOptionSettingsUpdate(conn, d); err != nil {
			return err
		}
	}

	return resourceAwsElasticBeanstalkConfigurationTemplateRead(d, meta)
}

func resourceAwsElasticBeanstalkConfigurationTemplateDescriptionUpdate(conn *elasticbeanstalk.ElasticBeanstalk, d *schema.ResourceData) error {
	_, err := conn.UpdateConfigurationTemplate(&elasticbeanstalk.UpdateConfigurationTemplateInput{
		ApplicationName: aws.String(d.Get("application").(string)),
		TemplateName:    aws.String(d.Get("name").(string)),
		Description:     aws.String(d.Get("description").(string)),
	})

	return err
}

func resourceAwsElasticBeanstalkConfigurationTemplateOptionSettingsUpdate(conn *elasticbeanstalk.ElasticBeanstalk, d *schema.ResourceData) error {
	if d.HasChange("setting") {
		_, err := conn.ValidateConfigurationSettings(&elasticbeanstalk.ValidateConfigurationSettingsInput{
			ApplicationName: aws.String(d.Get("application").(string)),
			TemplateName:    aws.String(d.Get("name").(string)),
			OptionSettings:  gatherOptionSettings(d),
		})
		if err != nil {
			return err
		}

		o, n := d.GetChange("setting")
		if o == nil {
			o = new(schema.Set)
		}
		if n == nil {
			n = new(schema.Set)
		}

		os := o.(*schema.Set)
		ns := n.(*schema.Set)

		rm := extractOptionSettings(os.Difference(ns))
		add := extractOptionSettings(ns.Difference(os))

		// Additions and removals of options are done in a single API call, so we
		// can't do our normal "remove these" and then later "add these", re-adding
		// any updated settings.
		// Because of this, we need to remove any settings in the "removable"
		// settings that are also found in the "add" settings, otherwise they
		// conflict. Here we loop through all the initial removables from the set
		// difference, and we build up a slice of settings not found in the "add"
		// set
		var remove []*elasticbeanstalk.ConfigurationOptionSetting
		for _, r := range rm {
			for _, a := range add {
				if *r.Namespace == *a.Namespace && *r.OptionName == *a.OptionName {
					continue
				}
				remove = append(remove, r)
			}
		}

		req := &elasticbeanstalk.UpdateConfigurationTemplateInput{
			ApplicationName: aws.String(d.Get("application").(string)),
			TemplateName:    aws.String(d.Get("name").(string)),
			OptionSettings:  add,
		}

		for _, elem := range remove {
			req.OptionsToRemove = append(req.OptionsToRemove, &elasticbeanstalk.OptionSpecification{
				Namespace:  elem.Namespace,
				OptionName: elem.OptionName,
			})
		}

		log.Printf("[DEBUG] Update Configuration Template request: %s", req)
		if _, err := conn.UpdateConfigurationTemplate(req); err != nil {
			return err
		}
	}

	return nil
}

func resourceAwsElasticBeanstalkConfigurationTemplateDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticbeanstalkconn

	application := d.Get("application").(string)

	_, err := conn.DeleteConfigurationTemplate(&elasticbeanstalk.DeleteConfigurationTemplateInput{
		ApplicationName: aws.String(application),
		TemplateName:    aws.String(d.Id()),
	})

	return err
}

func gatherOptionSettings(d *schema.ResourceData) []*elasticbeanstalk.ConfigurationOptionSetting {
	optionSettingsSet, ok := d.Get("setting").(*schema.Set)
	if !ok || optionSettingsSet == nil {
		optionSettingsSet = new(schema.Set)
	}

	return extractOptionSettings(optionSettingsSet)
}
<commit_msg>resource/elastic_beanstalk_configuration_template: Handle missing platform<commit_after>package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/elasticbeanstalk"
)

func resourceAwsElasticBeanstalkConfigurationTemplate() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsElasticBeanstalkConfigurationTemplateCreate,
		Read:   resourceAwsElasticBeanstalkConfigurationTemplateRead,
		Update: resourceAwsElasticBeanstalkConfigurationTemplateUpdate,
		Delete: resourceAwsElasticBeanstalkConfigurationTemplateDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"application": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"environment_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"setting": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Computed: true,
				Elem:     resourceAwsElasticBeanstalkOptionSetting(),
				Set:      optionSettingValueHash,
			},
			"solution_stack_name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAwsElasticBeanstalkConfigurationTemplateCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticbeanstalkconn

	// Get the relevant properties
	name := d.Get("name").(string)
	appName := d.Get("application").(string)

	optionSettings := gatherOptionSettings(d)

	opts := elasticbeanstalk.CreateConfigurationTemplateInput{
		ApplicationName: aws.String(appName),
		TemplateName:    aws.String(name),
		OptionSettings:  optionSettings,
	}

	if attr, ok := d.GetOk("description"); ok {
		opts.Description = aws.String(attr.(string))
	}

	if attr, ok := d.GetOk("environment_id"); ok {
		opts.EnvironmentId = aws.String(attr.(string))
	}

	if attr, ok := d.GetOk("solution_stack_name"); ok {
		opts.SolutionStackName = aws.String(attr.(string))
	}

	log.Printf("[DEBUG] Elastic Beanstalk configuration template create opts: %s", opts)
	if _, err := conn.CreateConfigurationTemplate(&opts); err != nil {
		return fmt.Errorf("Error creating Elastic Beanstalk configuration template: %s", err)
	}

	d.SetId(name)

	return resourceAwsElasticBeanstalkConfigurationTemplateRead(d, meta)
}

func resourceAwsElasticBeanstalkConfigurationTemplateRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticbeanstalkconn

	log.Printf("[DEBUG] Elastic Beanstalk configuration template read: %s", d.Get("name").(string))

	resp, err := conn.DescribeConfigurationSettings(&elasticbeanstalk.DescribeConfigurationSettingsInput{
		TemplateName:    aws.String(d.Id()),
		ApplicationName: aws.String(d.Get("application").(string)),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "InvalidParameterValue" && strings.Contains(awsErr.Message(), "No Configuration Template named") {
				log.Printf("[WARN] No Configuration Template named (%s) found", d.Id())
				d.SetId("")
				return nil
			} else if awsErr.Code() == "InvalidParameterValue" && strings.Contains(awsErr.Message(), "No Platform named") {
				log.Printf("[WARN] No Platform named (%s) found", d.Get("solution_stack_name").(string))
				d.SetId("")
				return nil
			}
		}
		return err
	}

	if len(resp.ConfigurationSettings) != 1 {
		log.Printf("[DEBUG] Elastic Beanstalk unexpected describe configuration template response: %+v", resp)
		return fmt.Errorf("Error reading application properties: found %d applications, expected 1", len(resp.ConfigurationSettings))
	}

	d.Set("description", resp.ConfigurationSettings[0].Description)
	return nil
}

func resourceAwsElasticBeanstalkConfigurationTemplateUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticbeanstalkconn

	log.Printf("[DEBUG] Elastic Beanstalk configuration template update: %s", d.Get("name").(string))

	if d.HasChange("description") {
		if err := resourceAwsElasticBeanstalkConfigurationTemplateDescriptionUpdate(conn, d); err != nil {
			return err
		}
	}

	if d.HasChange("setting") {
		if err := resourceAwsElasticBeanstalkConfigurationTemplateOptionSettingsUpdate(conn, d); err != nil {
			return err
		}
	}

	return resourceAwsElasticBeanstalkConfigurationTemplateRead(d, meta)
}

func resourceAwsElasticBeanstalkConfigurationTemplateDescriptionUpdate(conn *elasticbeanstalk.ElasticBeanstalk, d *schema.ResourceData) error {
	_, err := conn.UpdateConfigurationTemplate(&elasticbeanstalk.UpdateConfigurationTemplateInput{
		ApplicationName: aws.String(d.Get("application").(string)),
		TemplateName:    aws.String(d.Get("name").(string)),
		Description:     aws.String(d.Get("description").(string)),
	})

	return err
}

func resourceAwsElasticBeanstalkConfigurationTemplateOptionSettingsUpdate(conn *elasticbeanstalk.ElasticBeanstalk, d *schema.ResourceData) error {
	if d.HasChange("setting") {
		_, err := conn.ValidateConfigurationSettings(&elasticbeanstalk.ValidateConfigurationSettingsInput{
			ApplicationName: aws.String(d.Get("application").(string)),
			TemplateName:    aws.String(d.Get("name").(string)),
			OptionSettings:  gatherOptionSettings(d),
		})
		if err != nil {
			return err
		}

		o, n := d.GetChange("setting")
		if o == nil {
			o = new(schema.Set)
		}
		if n == nil {
			n = new(schema.Set)
		}

		os := o.(*schema.Set)
		ns := n.(*schema.Set)

		rm := extractOptionSettings(os.Difference(ns))
		add := extractOptionSettings(ns.Difference(os))

		// Additions and removals of options are done in a single API call, so we
		// can't do our normal "remove these" and then later "add these", re-adding
		// any updated settings.
		// Because of this, we need to remove any settings in the "removable"
		// settings that are also found in the "add" settings, otherwise they
		// conflict. Here we loop through all the initial removables from the set
		// difference, and we build up a slice of settings not found in the "add"
		// set
		var remove []*elasticbeanstalk.ConfigurationOptionSetting
		for _, r := range rm {
			for _, a := range add {
				if *r.Namespace == *a.Namespace && *r.OptionName == *a.OptionName {
					continue
				}
				remove = append(remove, r)
			}
		}

		req := &elasticbeanstalk.UpdateConfigurationTemplateInput{
			ApplicationName: aws.String(d.Get("application").(string)),
			TemplateName:    aws.String(d.Get("name").(string)),
			OptionSettings:  add,
		}

		for _, elem := range remove {
			req.OptionsToRemove = append(req.OptionsToRemove, &elasticbeanstalk.OptionSpecification{
				Namespace:  elem.Namespace,
				OptionName: elem.OptionName,
			})
		}

		log.Printf("[DEBUG] Update Configuration Template request: %s", req)
		if _, err := conn.UpdateConfigurationTemplate(req); err != nil {
			return err
		}
	}

	return nil
}

func resourceAwsElasticBeanstalkConfigurationTemplateDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticbeanstalkconn

	application := d.Get("application").(string)

	_, err := conn.DeleteConfigurationTemplate(&elasticbeanstalk.DeleteConfigurationTemplateInput{
		ApplicationName: aws.String(application),
		TemplateName:    aws.String(d.Id()),
	})

	return err
}

func gatherOptionSettings(d *schema.ResourceData) []*elasticbeanstalk.ConfigurationOptionSetting {
	optionSettingsSet, ok := d.Get("setting").(*schema.Set)
	if !ok || optionSettingsSet == nil {
		optionSettingsSet = new(schema.Set)
	}

	return extractOptionSettings(optionSettingsSet)
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bufio

import (
	"bytes"
	"errors"
	"io"
	"unicode/utf8"
)

// Scanner provides a convenient interface for reading data such as
// a file of newline-delimited lines of text. Successive calls to
// the Scan method will step through the 'tokens' of a file, skipping
// the bytes between the tokens. The specification of a token is
// defined by a split function of type SplitFunc; the default split
// function breaks the input into lines with line termination stripped. Split
// functions are defined in this package for scanning a file into
// lines, bytes, UTF-8-encoded runes, and space-delimited words. The
// client may instead provide a custom split function.
//
// Scanning stops unrecoverably at EOF, the first I/O error, or a token too
// large to fit in the buffer. When a scan stops, the reader may have
// advanced arbitrarily far past the last token. Programs that need more
// control over error handling or large tokens, or must run sequential scans
// on a reader, should use bufio.Reader instead.
//
type Scanner struct {
	r            io.Reader // The reader provided by the client.
	split        SplitFunc // The function to split the tokens.
	maxTokenSize int       // Maximum size of a token; modified by tests.
	token        []byte    // Last token returned by split.
	buf          []byte    // Buffer used as argument to split.
	start        int       // First non-processed byte in buf.
	end          int       // End of data in buf.
	err          error     // Sticky error.
}

// SplitFunc is the signature of the split function used to tokenize the
// input. The arguments are an initial substring of the remaining unprocessed
// data and a flag, atEOF, that reports whether the Reader has no more data
// to give. The return values are the number of bytes to advance the input
// and the next token to return to the user, plus an error, if any. If the
// data does not yet hold a complete token, for instance if it has no newline
// while scanning lines, SplitFunc can return (0, nil) to signal the Scanner
// to read more data into the slice and try again with a longer slice
// starting at the same point in the input.
//
// If the returned error is non-nil, scanning stops and the error
// is returned to the client.
//
// The function is never called with an empty data slice unless atEOF
// is true. If atEOF is true, however, data may be non-empty and,
// as always, holds unprocessed text.
type SplitFunc func(data []byte, atEOF bool) (advance int, token []byte, err error)

// Errors returned by Scanner.
var (
	ErrTooLong         = errors.New("bufio.Scanner: token too long")
	ErrNegativeAdvance = errors.New("bufio.Scanner: SplitFunc returns negative advance count")
	ErrAdvanceTooFar   = errors.New("bufio.Scanner: SplitFunc returns advance count beyond input")
)

const (
	// Maximum size used to buffer a token. The actual maximum token size
	// may be smaller as the buffer may need to include, for instance, a newline.
	MaxScanTokenSize = 64 * 1024
)

// NewScanner returns a new Scanner to read from r.
// The split function defaults to ScanLines.
func NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		r:            r,
		split:        ScanLines,
		maxTokenSize: MaxScanTokenSize,
		buf:          make([]byte, 4096), // Plausible starting size; needn't be large.
	}
}

// Err returns the first non-EOF error that was encountered by the Scanner.
func (s *Scanner) Err() error {
	if s.err == io.EOF {
		return nil
	}
	return s.err
}

// Bytes returns the most recent token generated by a call to Scan.
// The underlying array may point to data that will be overwritten
// by a subsequent call to Scan. It does no allocation.
func (s *Scanner) Bytes() []byte {
	return s.token
}

// Text returns the most recent token generated by a call to Scan
// as a newly allocated string holding its bytes.
func (s *Scanner) Text() string {
	return string(s.token)
}

// Scan advances the Scanner to the next token, which will then be
// available through the Bytes or Text method. It returns false when the
// scan stops, either by reaching the end of the input or an error.
// After Scan returns false, the Err method will return any error that
// occurred during scanning, except that if it was io.EOF, Err
// will return nil.
func (s *Scanner) Scan() bool {
	// Loop until we have a token.
	for {
		// See if we can get a token with what we already have.
		if s.end > s.start {
			advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil)
			if err != nil {
				s.setErr(err)
				return false
			}
			if !s.advance(advance) {
				return false
			}
			s.token = token
			if token != nil {
				return true
			}
		}
		// We cannot generate a token with what we are holding.
		// If we've already hit EOF or an I/O error, we are done.
		if s.err != nil {
			// Shut it down.
			s.start = 0
			s.end = 0
			return false
		}
		// Must read more data.
		// First, shift data to beginning of buffer if there's lots of empty space
		// or space is neded.
		if s.start > 0 && (s.end == len(s.buf) || s.start > len(s.buf)/2) {
			copy(s.buf, s.buf[s.start:s.end])
			s.end -= s.start
			s.start = 0
		}
		// Is the buffer full? If so, resize.
		if s.end == len(s.buf) {
			if len(s.buf) >= s.maxTokenSize {
				s.setErr(ErrTooLong)
				return false
			}
			newSize := len(s.buf) * 2
			if newSize > s.maxTokenSize {
				newSize = s.maxTokenSize
			}
			newBuf := make([]byte, newSize)
			copy(newBuf, s.buf[s.start:s.end])
			s.buf = newBuf
			s.end -= s.start
			s.start = 0
			continue
		}
		// Finally we can read some input. Make sure we don't get stuck with
		// a misbehaving Reader. Officially we don't need to do this, but let's
		// be extra careful: Scanner is for safe, simple jobs.
		for loop := 0; ; {
			n, err := s.r.Read(s.buf[s.end:len(s.buf)])
			s.end += n
			if err != nil {
				s.setErr(err)
				break
			}
			if n > 0 {
				break
			}
			loop++
			if loop > 100 {
				s.setErr(io.ErrNoProgress)
				break
			}
		}
	}
}

// advance consumes n bytes of the buffer. It reports whether the advance was legal.
func (s *Scanner) advance(n int) bool {
	if n < 0 {
		s.setErr(ErrNegativeAdvance)
		return false
	}
	if n > s.end-s.start {
		s.setErr(ErrAdvanceTooFar)
		return false
	}
	s.start += n
	return true
}

// setErr records the first error encountered.
func (s *Scanner) setErr(err error) {
	if s.err == nil || s.err == io.EOF {
		s.err = err
	}
}

// Split sets the split function for the Scanner. If called, it must be
// called before Scan. The default split function is ScanLines.
func (s *Scanner) Split(split SplitFunc) {
	s.split = split
}

// Split functions

// ScanBytes is a split function for a Scanner that returns each byte as a token.
func ScanBytes(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	return 1, data[0:1], nil
}

var errorRune = []byte(string(utf8.RuneError))

// ScanRunes is a split function for a Scanner that returns each
// UTF-8-encoded rune as a token. The sequence of runes returned is
// equivalent to that from a range loop over the input as a string, which
// means that erroneous UTF-8 encodings translate to U+FFFD = "\xef\xbf\xbd".
// Because of the Scan interface, this makes it impossible for the client to
// distinguish correctly encoded replacement runes from encoding errors.
func ScanRunes(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// Fast path 1: ASCII.
	if data[0] < utf8.RuneSelf {
		return 1, data[0:1], nil
	}

	// Fast path 2: Correct UTF-8 decode without error.
	_, width := utf8.DecodeRune(data)
	if width > 1 {
		// It's a valid encoding. Width cannot be one for a correctly encoded
		// non-ASCII rune.
		return width, data[0:width], nil
	}

	// We know it's an error: we have width==1 and implicitly r==utf8.RuneError.
	// Is the error because there wasn't a full rune to be decoded?
	// FullRune distinguishes correctly between erroneous and incomplete encodings.
	if !atEOF && !utf8.FullRune(data) {
		// Incomplete; get more bytes.
		return 0, nil, nil
	}

	// We have a real UTF-8 encoding error. Return a properly encoded error rune
	// but advance only one byte. This matches the behavior of a range loop over
	// an incorrectly encoded string.
	return 1, errorRune, nil
}

// dropCR drops a terminal \r from the data.
func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

// ScanLines is a split function for a Scanner that returns each line of
// text, stripped of any trailing end-of-line marker. The returned line may
// be empty. The end-of-line marker is one optional carriage return followed
// by one mandatory newline. In regular expression notation, it is `\r?\n`.
// The last non-empty line of input will be returned even if it has no
// newline.
func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, '\n'); i >= 0 {
		// We have a full newline-terminated line.
		return i + 1, dropCR(data[0:i]), nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), dropCR(data), nil
	}
	// Request more data.
	return 0, nil, nil
}

// isSpace returns whether the character is a Unicode white space character.
// We avoid dependency on the unicode package, but check validity of the implementation
// in the tests.
func isSpace(r rune) bool {
	if r <= '\u00FF' {
		// Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs.
		switch r {
		case ' ', '\t', '\n', '\v', '\f', '\r':
			return true
		case '\u0085', '\u00A0':
			return true
		}
		return false
	}
	// High-valued ones.
	if '\u2000' <= r && r <= '\u200a' {
		return true
	}
	switch r {
	case '\u1680', '\u180e', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000':
		return true
	}
	return false
}

// ScanWords is a split function for a Scanner that returns each
// space-separated word of text, with surrounding spaces deleted. It will
// never return an empty string. The definition of space is set by
// unicode.IsSpace.
func ScanWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !isSpace(r) {
			break
		}
	}
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	// Scan until space, marking end of word.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if isSpace(r) {
			return i + width, data[start:i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data[start:], nil
	}
	// Request more data.
	return 0, nil, nil
}
<commit_msg>bufio: fix SplitFunc docs Fixes issue 5532.<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bufio

import (
	"bytes"
	"errors"
	"io"
	"unicode/utf8"
)

// Scanner provides a convenient interface for reading data such as
// a file of newline-delimited lines of text. Successive calls to
// the Scan method will step through the 'tokens' of a file, skipping
// the bytes between the tokens. The specification of a token is
// defined by a split function of type SplitFunc; the default split
// function breaks the input into lines with line termination stripped. Split
// functions are defined in this package for scanning a file into
// lines, bytes, UTF-8-encoded runes, and space-delimited words. The
// client may instead provide a custom split function.
//
// Scanning stops unrecoverably at EOF, the first I/O error, or a token too
// large to fit in the buffer. When a scan stops, the reader may have
// advanced arbitrarily far past the last token. Programs that need more
// control over error handling or large tokens, or must run sequential scans
// on a reader, should use bufio.Reader instead.
//
type Scanner struct {
	r            io.Reader // The reader provided by the client.
	split        SplitFunc // The function to split the tokens.
	maxTokenSize int       // Maximum size of a token; modified by tests.
	token        []byte    // Last token returned by split.
	buf          []byte    // Buffer used as argument to split.
	start        int       // First non-processed byte in buf.
	end          int       // End of data in buf.
	err          error     // Sticky error.
}

// SplitFunc is the signature of the split function used to tokenize the
// input. The arguments are an initial substring of the remaining unprocessed
// data and a flag, atEOF, that reports whether the Reader has no more data
// to give. The return values are the number of bytes to advance the input
// and the next token to return to the user, plus an error, if any. If the
// data does not yet hold a complete token, for instance if it has no newline
// while scanning lines, SplitFunc can return (0, nil, nil) to signal the
// Scanner to read more data into the slice and try again with a longer slice
// starting at the same point in the input.
//
// If the returned error is non-nil, scanning stops and the error
// is returned to the client.
//
// The function is never called with an empty data slice unless atEOF
// is true. If atEOF is true, however, data may be non-empty and,
// as always, holds unprocessed text.
type SplitFunc func(data []byte, atEOF bool) (advance int, token []byte, err error)

// Errors returned by Scanner.
var (
	ErrTooLong         = errors.New("bufio.Scanner: token too long")
	ErrNegativeAdvance = errors.New("bufio.Scanner: SplitFunc returns negative advance count")
	ErrAdvanceTooFar   = errors.New("bufio.Scanner: SplitFunc returns advance count beyond input")
)

const (
	// Maximum size used to buffer a token. The actual maximum token size
	// may be smaller as the buffer may need to include, for instance, a newline.
	MaxScanTokenSize = 64 * 1024
)

// NewScanner returns a new Scanner to read from r.
// The split function defaults to ScanLines.
func NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		r:            r,
		split:        ScanLines,
		maxTokenSize: MaxScanTokenSize,
		buf:          make([]byte, 4096), // Plausible starting size; needn't be large.
	}
}

// Err returns the first non-EOF error that was encountered by the Scanner.
func (s *Scanner) Err() error {
	if s.err == io.EOF {
		return nil
	}
	return s.err
}

// Bytes returns the most recent token generated by a call to Scan.
// The underlying array may point to data that will be overwritten
// by a subsequent call to Scan. It does no allocation.
func (s *Scanner) Bytes() []byte {
	return s.token
}

// Text returns the most recent token generated by a call to Scan
// as a newly allocated string holding its bytes.
func (s *Scanner) Text() string {
	return string(s.token)
}

// Scan advances the Scanner to the next token, which will then be
// available through the Bytes or Text method. It returns false when the
// scan stops, either by reaching the end of the input or an error.
// After Scan returns false, the Err method will return any error that
// occurred during scanning, except that if it was io.EOF, Err
// will return nil.
func (s *Scanner) Scan() bool {
	// Loop until we have a token.
	for {
		// See if we can get a token with what we already have.
		if s.end > s.start {
			advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil)
			if err != nil {
				s.setErr(err)
				return false
			}
			if !s.advance(advance) {
				return false
			}
			s.token = token
			if token != nil {
				return true
			}
		}
		// We cannot generate a token with what we are holding.
		// If we've already hit EOF or an I/O error, we are done.
		if s.err != nil {
			// Shut it down.
			s.start = 0
			s.end = 0
			return false
		}
		// Must read more data.
		// First, shift data to beginning of buffer if there's lots of empty space
		// or space is neded.
		if s.start > 0 && (s.end == len(s.buf) || s.start > len(s.buf)/2) {
			copy(s.buf, s.buf[s.start:s.end])
			s.end -= s.start
			s.start = 0
		}
		// Is the buffer full? If so, resize.
		if s.end == len(s.buf) {
			if len(s.buf) >= s.maxTokenSize {
				s.setErr(ErrTooLong)
				return false
			}
			newSize := len(s.buf) * 2
			if newSize > s.maxTokenSize {
				newSize = s.maxTokenSize
			}
			newBuf := make([]byte, newSize)
			copy(newBuf, s.buf[s.start:s.end])
			s.buf = newBuf
			s.end -= s.start
			s.start = 0
			continue
		}
		// Finally we can read some input. Make sure we don't get stuck with
		// a misbehaving Reader. Officially we don't need to do this, but let's
		// be extra careful: Scanner is for safe, simple jobs.
		for loop := 0; ; {
			n, err := s.r.Read(s.buf[s.end:len(s.buf)])
			s.end += n
			if err != nil {
				s.setErr(err)
				break
			}
			if n > 0 {
				break
			}
			loop++
			if loop > 100 {
				s.setErr(io.ErrNoProgress)
				break
			}
		}
	}
}

// advance consumes n bytes of the buffer. It reports whether the advance was legal.
func (s *Scanner) advance(n int) bool {
	if n < 0 {
		s.setErr(ErrNegativeAdvance)
		return false
	}
	if n > s.end-s.start {
		s.setErr(ErrAdvanceTooFar)
		return false
	}
	s.start += n
	return true
}

// setErr records the first error encountered.
func (s *Scanner) setErr(err error) {
	if s.err == nil || s.err == io.EOF {
		s.err = err
	}
}

// Split sets the split function for the Scanner. If called, it must be
// called before Scan. The default split function is ScanLines.
func (s *Scanner) Split(split SplitFunc) {
	s.split = split
}

// Split functions

// ScanBytes is a split function for a Scanner that returns each byte as a token.
func ScanBytes(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	return 1, data[0:1], nil
}

var errorRune = []byte(string(utf8.RuneError))

// ScanRunes is a split function for a Scanner that returns each
// UTF-8-encoded rune as a token. The sequence of runes returned is
// equivalent to that from a range loop over the input as a string, which
// means that erroneous UTF-8 encodings translate to U+FFFD = "\xef\xbf\xbd".
// Because of the Scan interface, this makes it impossible for the client to
// distinguish correctly encoded replacement runes from encoding errors.
func ScanRunes(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// Fast path 1: ASCII.
	if data[0] < utf8.RuneSelf {
		return 1, data[0:1], nil
	}

	// Fast path 2: Correct UTF-8 decode without error.
	_, width := utf8.DecodeRune(data)
	if width > 1 {
		// It's a valid encoding. Width cannot be one for a correctly encoded
		// non-ASCII rune.
		return width, data[0:width], nil
	}

	// We know it's an error: we have width==1 and implicitly r==utf8.RuneError.
	// Is the error because there wasn't a full rune to be decoded?
	// FullRune distinguishes correctly between erroneous and incomplete encodings.
	if !atEOF && !utf8.FullRune(data) {
		// Incomplete; get more bytes.
		return 0, nil, nil
	}

	// We have a real UTF-8 encoding error. Return a properly encoded error rune
	// but advance only one byte. This matches the behavior of a range loop over
	// an incorrectly encoded string.
	return 1, errorRune, nil
}

// dropCR drops a terminal \r from the data.
func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

// ScanLines is a split function for a Scanner that returns each line of
// text, stripped of any trailing end-of-line marker. The returned line may
// be empty. The end-of-line marker is one optional carriage return followed
// by one mandatory newline. In regular expression notation, it is `\r?\n`.
// The last non-empty line of input will be returned even if it has no
// newline.
func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, '\n'); i >= 0 {
		// We have a full newline-terminated line.
		return i + 1, dropCR(data[0:i]), nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), dropCR(data), nil
	}
	// Request more data.
	return 0, nil, nil
}

// isSpace returns whether the character is a Unicode white space character.
// We avoid dependency on the unicode package, but check validity of the implementation
// in the tests.
func isSpace(r rune) bool {
	if r <= '\u00FF' {
		// Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs.
		switch r {
		case ' ', '\t', '\n', '\v', '\f', '\r':
			return true
		case '\u0085', '\u00A0':
			return true
		}
		return false
	}
	// High-valued ones.
	if '\u2000' <= r && r <= '\u200a' {
		return true
	}
	switch r {
	case '\u1680', '\u180e', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000':
		return true
	}
	return false
}

// ScanWords is a split function for a Scanner that returns each
// space-separated word of text, with surrounding spaces deleted. It will
// never return an empty string. The definition of space is set by
// unicode.IsSpace.
func ScanWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !isSpace(r) {
			break
		}
	}
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	// Scan until space, marking end of word.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if isSpace(r) {
			return i + width, data[start:i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data[start:], nil
	}
	// Request more data.
	return 0, nil, nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

/*
	Hypot -- sqrt(p*p + q*q), but overflows only if the result does.
*/

// Hypot returns Sqrt(p*p + q*q), taking care to avoid
// unnecessary overflow and underflow.
//
// Special cases are:
//	Hypot(p, q) = +Inf if p or q is infinite
//	Hypot(p, q) = NaN if p or q is NaN
func Hypot(p, q float64) float64

func hypot(p, q float64) float64 {
	// special cases
	switch {
	case IsInf(p, 0) || IsInf(q, 0):
		return Inf(1)
	case IsNaN(p) || IsNaN(q):
		return NaN()
	}
	if p < 0 {
		p = -p
	}
	if q < 0 {
		q = -q
	}
	if p < q {
		p, q = q, p
	}
	if p == 0 {
		return 0
	}
	q = q / p
	return p * Sqrt(1+q*q)
}
<commit_msg>math: make Hypot special cases look like others<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

/*
	Hypot -- sqrt(p*p + q*q), but overflows only if the result does.
*/

// Hypot returns Sqrt(p*p + q*q), taking care to avoid
// unnecessary overflow and underflow.
//
// Special cases are:
//	Hypot(±Inf, q) = +Inf
//	Hypot(p, ±Inf) = +Inf
//	Hypot(NaN, q) = NaN
//	Hypot(p, NaN) = NaN
func Hypot(p, q float64) float64

func hypot(p, q float64) float64 {
	// special cases
	switch {
	case IsInf(p, 0) || IsInf(q, 0):
		return Inf(1)
	case IsNaN(p) || IsNaN(q):
		return NaN()
	}
	if p < 0 {
		p = -p
	}
	if q < 0 {
		q = -q
	}
	if p < q {
		p, q = q, p
	}
	if p == 0 {
		return 0
	}
	q = q / p
	return p * Sqrt(1+q*q)
}
<|endoftext|>


<commit_before>package main

import "os"
import "fmt"

func printUsage(progName string) {
	fmt.Println(progName + " is a scraper of a well knwon classified ads website.")
	fmt.Println()
	fmt.Println("Usage:")
	fmt.Println()
	fmt.Println("    " + progName + " configFile.json")
	fmt.Println()
}

func main() {

	// parse arguments
	args := os.Args
	if len(args) != 2 {
		printUsage(args[0])
		return
	}

	// read configuration
	config, err := ReadConfigFile(args[1])
	if err != nil {
		fmt.Print(err.Error())
		return
	}

	// retrieve data about previous launches
	dbAdData, err := LoadOrCreate(config.SearchTerms)
	if err != nil {
		fmt.Print(err.Error())
		return
	}

	// build URL to scrap & scrap new data
	// TODO build an URL builder.
	url := "https://www.leboncoin.fr/voitures/offres/bretagne/occasions/?q=" + config.SearchTerms
	ads, err := Scraper(url, *dbAdData, config.SearchTerms)
	if err != nil {
		fmt.Print(err.Error())
		return
	}

	// it could be possible to compute a distance to a point, on the basis of this webservice :
	// $ curl "http://api-adresse.data.gouv.fr/search/?type=city&q=Carcassonne" |jq

	// print data
	for _, ad := range ads {
		PrintTextAbridged(ad)
		//PrintText(ad)
		//PrintLineSeparator()
	}

	// build & send a mail
	fmt.Println("Sending mail")
	err = SendAdsByMail(config.SMTPUser, config.MailFrom, config.To, ads)
	if err != nil {
		fmt.Print(err.Error())
		return
	}

	fmt.Println("DONE")
}
<commit_msg>compilation error<commit_after>package main

import "os"
import "fmt"

func printUsage(progName string) {
	fmt.Println(progName + " is a scraper of a well knwon classified ads website.")
	fmt.Println()
	fmt.Println("Usage:")
	fmt.Println()
	fmt.Println("    " + progName + " configFile.json")
	fmt.Println()
}

func main() {

	// parse arguments
	args := os.Args
	if len(args) != 2 {
		printUsage(args[0])
		return
	}

	// read configuration
	config, err := ReadConfigFile(args[1])
	if err != nil {
		fmt.Print(err.Error())
		return
	}

	// retrieve data about previous launches
	dbAdData, err := LoadOrCreate(config.SearchTerms)
	if err != nil {
		fmt.Print(err.Error())
		return
	}

	// build URL to scrap & scrap new data
	// TODO build an URL builder.
	url := "https://www.leboncoin.fr/voitures/offres/bretagne/occasions/?q=" + config.SearchTerms
	ads, err := Scraper(url, *dbAdData, config.SearchTerms)
	if err != nil {
		fmt.Print(err.Error())
		return
	}

	// it could be possible to compute a distance to a point, on the basis of this webservice :
	// $ curl "http://api-adresse.data.gouv.fr/search/?type=city&q=Carcassonne" |jq

	// print data
	for _, ad := range ads {
		PrintTextAbridged(ad)
		//PrintText(ad)
		//PrintLineSeparator()
	}

	// build & send a mail
	fmt.Println("Sending mail")
	err = SendAdsByMail(config.SMTPUser, config.MailFrom, config.MailTo, ads)
	if err != nil {
		fmt.Print(err.Error())
		return
	}

	fmt.Println("DONE")
}
<|endoftext|>