<commit_before>// Copyright (c) 2015-2016 Jeevanandam M (jeeva@myjeeva.com), All rights reserved.
// resty source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package resty

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strings"
)

//
// Request Middleware(s)
//

func parseRequestURL(c *Client, r *Request) error {
	// Parsing request URL
	reqURL, err := url.Parse(r.URL)
	if err != nil {
		return err
	}

	// If Request.Url is relative path then added c.HostUrl into
	// the request URL otherwise Request.Url will be used as-is
	if !reqURL.IsAbs() {
		if !strings.HasPrefix(r.URL, "/") {
			r.URL = "/" + r.URL
		}

		reqURL, err = url.Parse(c.HostURL + r.URL)
		if err != nil {
			return err
		}
	}

	// Adding Query Param
	query := reqURL.Query()
	for k := range c.QueryParam {
		query.Set(k, c.QueryParam.Get(k))
	}
	for k := range r.QueryParam {
		query.Set(k, r.QueryParam.Get(k))
	}

	reqURL.RawQuery = query.Encode()
	r.URL = reqURL.String()

	return nil
}

func parseRequestHeader(c *Client, r *Request) error {
	hdr := http.Header{}
	for k := range c.Header {
		hdr.Set(k, c.Header.Get(k))
	}
	for k := range r.Header {
		hdr.Set(k, r.Header.Get(k))
	}

	if IsStringEmpty(hdr.Get(hdrUserAgentKey)) {
		hdr.Set(hdrUserAgentKey, fmt.Sprintf(hdrUserAgentValue, Version))
	} else {
		hdr.Set("X-"+hdrUserAgentKey, fmt.Sprintf(hdrUserAgentValue, Version))
	}

	if IsStringEmpty(hdr.Get(hdrAcceptKey)) && !IsStringEmpty(hdr.Get(hdrContentTypeKey)) {
		hdr.Set(hdrAcceptKey, hdr.Get(hdrContentTypeKey))
	}

	r.Header = hdr

	return nil
}

func parseRequestBody(c *Client, r *Request) (err error) {
	if isPayloadSupported(r.Method) {

		// Handling Multipart
		if r.isMultiPart && !(r.Method == PATCH) {
			if err = handleMultipart(c, r); err != nil {
				return
			}

			goto CL
		}

		// Handling Form Data
		if len(c.FormData) > 0 || len(r.FormData) > 0 {
			handleFormData(c, r)

			goto CL
		}

		// Handling Request body
		if r.Body != nil {
			if err = handleRequestBody(c, r); err != nil {
				return
			}
		}
	} else {
		r.Header.Del(hdrContentTypeKey)
	}

CL:
	// by default resty won't set content length, you can if you want to :)
	if c.setContentLength || r.setContentLength {
		r.Header.Set(hdrContentLengthKey, fmt.Sprintf("%d", r.bodyBuf.Len()))
	}

	return
}

func createHTTPRequest(c *Client, r *Request) (err error) {
	if r.bodyBuf == nil {
		r.RawRequest, err = http.NewRequest(r.Method, r.URL, nil)
	} else {
		r.RawRequest, err = http.NewRequest(r.Method, r.URL, r.bodyBuf)
	}

	// Add headers into http request
	r.RawRequest.Header = r.Header

	// Add cookies into http request
	for _, cookie := range c.Cookies {
		r.RawRequest.AddCookie(cookie)
	}

	return
}

func addCredentials(c *Client, r *Request) error {
	var isBasicAuth bool
	// Basic Auth
	if r.UserInfo != nil { // takes precedence
		r.RawRequest.SetBasicAuth(r.UserInfo.Username, r.UserInfo.Password)
		isBasicAuth = true
	} else if c.UserInfo != nil {
		r.RawRequest.SetBasicAuth(c.UserInfo.Username, c.UserInfo.Password)
		isBasicAuth = true
	}

	if !c.DisableWarn {
		if isBasicAuth && !strings.HasPrefix(r.URL, "https") {
			c.Log.Println("WARNING - Using Basic Auth in HTTP mode is not secure.")
		}
	}

	// Token Auth
	if !IsStringEmpty(r.Token) { // takes precedence
		r.RawRequest.Header.Set(hdrAuthorizationKey, "Bearer "+r.Token)
	} else if !IsStringEmpty(c.Token) {
		r.RawRequest.Header.Set(hdrAuthorizationKey, "Bearer "+c.Token)
	}

	return nil
}

func requestLogger(c *Client, r *Request) error {
	if c.Debug {
		rr := r.RawRequest
		c.Log.Println()
		c.disableLogPrefix()
		c.Log.Println("---------------------- REQUEST LOG -----------------------")
		c.Log.Printf("%s  %s  %s\n", r.Method, rr.URL.RequestURI(), rr.Proto)
		c.Log.Printf("HOST   : %s", rr.URL.Host)
		c.Log.Println("HEADERS:")
		for h, v := range rr.Header {
			c.Log.Printf("%25s: %v", h, strings.Join(v, ", "))
		}
		c.Log.Printf("BODY   :\n%v", r.fmtBodyString())
		c.Log.Println("----------------------------------------------------------")
		c.enableLogPrefix()
	}

	return nil
}

//
// Response Middleware(s)
//

func responseLogger(c *Client, res *Response) error {
	if c.Debug {
		c.Log.Println()
		c.disableLogPrefix()
		c.Log.Println("---------------------- RESPONSE LOG -----------------------")
		c.Log.Printf("STATUS 		: %s", res.Status())
		c.Log.Printf("RECEIVED AT	: %v", res.ReceivedAt())
		c.Log.Printf("RESPONSE TIME	: %v", res.Time())
		c.Log.Println("HEADERS:")
		for h, v := range res.Header() {
			c.Log.Printf("%30s: %v", h, strings.Join(v, ", "))
		}
		if res.Request.isSaveResponse {
			c.Log.Printf("BODY   :\n***** RESPONSE WRITTEN INTO FILE *****")
		} else {
			c.Log.Printf("BODY   :\n%v", res.fmtBodyString())
		}
		c.Log.Println("----------------------------------------------------------")
		c.enableLogPrefix()
	}

	return nil
}

func parseResponseBody(c *Client, res *Response) (err error) {
	// Handles only JSON or XML content type
	ct := res.Header().Get(hdrContentTypeKey)
	if IsJSONType(ct) || IsXMLType(ct) {
		// Considered as Result
		if res.StatusCode() > 199 && res.StatusCode() < 300 {
			if res.Request.Result != nil {
				err = Unmarshal(ct, res.body, res.Request.Result)
			}
		}

		// Considered as Error
		if res.StatusCode() > 399 {
			// global error interface
			if res.Request.Error == nil && c.Error != nil {
				res.Request.Error = reflect.New(c.Error).Interface()
			}

			if res.Request.Error != nil {
				err = Unmarshal(ct, res.body, res.Request.Error)
			}
		}
	}

	return
}

func handleMultipart(c *Client, r *Request) (err error) {
	r.bodyBuf = &bytes.Buffer{}
	w := multipart.NewWriter(r.bodyBuf)

	for p := range c.FormData {
		w.WriteField(p, c.FormData.Get(p))
	}

	for p := range r.FormData {
		if strings.HasPrefix(p, "@") { // file
			err = addFile(w, p[1:], r.FormData.Get(p))
			if err != nil {
				return
			}
		} else { // form value
			w.WriteField(p, r.FormData.Get(p))
		}
	}

	// #21 - adding io.Reader support
	if len(r.multipartFiles) > 0 {
		for _, f := range r.multipartFiles {
			err = addFileReader(w, f)
			if err != nil {
				return
			}
		}
	}

	r.Header.Set(hdrContentTypeKey, w.FormDataContentType())
	err = w.Close()

	return
}

func handleFormData(c *Client, r *Request) {
	formData := url.Values{}

	for p := range c.FormData {
		formData.Set(p, c.FormData.Get(p))
	}

	for p := range r.FormData { // takes precedence
		formData.Set(p, r.FormData.Get(p))
	}

	r.bodyBuf = bytes.NewBuffer([]byte(formData.Encode()))
	r.Header.Set(hdrContentTypeKey, formContentType)
	r.isFormData = true
}

func handleRequestBody(c *Client, r *Request) (err error) {
	contentType := r.Header.Get(hdrContentTypeKey)
	if IsStringEmpty(contentType) {
		contentType = DetectContentType(r.Body)
		r.Header.Set(hdrContentTypeKey, contentType)
	}

	var bodyBytes []byte
	kind := kindOf(r.Body)
	if reader, ok := r.Body.(io.Reader); ok {
		r.bodyBuf = &bytes.Buffer{}
		r.bodyBuf.ReadFrom(reader)
	} else if b, ok := r.Body.([]byte); ok {
		bodyBytes = b
	} else if s, ok := r.Body.(string); ok {
		bodyBytes = []byte(s)
	} else if IsJSONType(contentType) &&
		(kind == reflect.Struct || kind == reflect.Map || kind == reflect.Slice) {
		bodyBytes, err = json.Marshal(r.Body)
	} else if IsXMLType(contentType) && (kind == reflect.Struct) {
		bodyBytes, err = xml.Marshal(r.Body)
	}

	if bodyBytes == nil && r.bodyBuf == nil {
		err = errors.New("Unsupported 'Body' type/value")
	}

	// if any errors during body bytes handling, return it
	if err != nil {
		return
	}

	// []byte into Buffer
	if bodyBytes != nil && r.bodyBuf == nil {
		r.bodyBuf = bytes.NewBuffer(bodyBytes)
	}

	return
}

func saveResponseIntoFile(c *Client, res *Response) error {
	if res.Request.isSaveResponse {
		file := ""

		if len(c.outputDirectory) > 0 && !filepath.IsAbs(res.Request.outputFile) {
			file += c.outputDirectory + string(filepath.Separator)
		}

		file = filepath.Clean(file + res.Request.outputFile)
		err := createDirectory(filepath.Dir(file))
		if err != nil {
			return err
		}

		outFile, err := os.Create(file)
		if err != nil {
			return err
		}
		defer outFile.Close()

		// io.Copy reads maximum 32kb size, it is perfect for large file download too
		defer res.RawResponse.Body.Close()
		written, err := io.Copy(outFile, res.RawResponse.Body)
		if err != nil {
			return err
		}

		res.size = written
	}

	return nil
}
<commit_msg>method organize<commit_after>// Copyright (c) 2015-2016 Jeevanandam M (jeeva@myjeeva.com), All rights reserved.
// resty source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package resty

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strings"
)

//
// Request Middleware(s)
//

func parseRequestURL(c *Client, r *Request) error {
	// Parsing request URL
	reqURL, err := url.Parse(r.URL)
	if err != nil {
		return err
	}

	// If Request.Url is relative path then added c.HostUrl into
	// the request URL otherwise Request.Url will be used as-is
	if !reqURL.IsAbs() {
		if !strings.HasPrefix(r.URL, "/") {
			r.URL = "/" + r.URL
		}

		reqURL, err = url.Parse(c.HostURL + r.URL)
		if err != nil {
			return err
		}
	}

	// Adding Query Param
	query := reqURL.Query()
	for k := range c.QueryParam {
		query.Set(k, c.QueryParam.Get(k))
	}
	for k := range r.QueryParam {
		query.Set(k, r.QueryParam.Get(k))
	}

	reqURL.RawQuery = query.Encode()
	r.URL = reqURL.String()

	return nil
}

func parseRequestHeader(c *Client, r *Request) error {
	hdr := http.Header{}
	for k := range c.Header {
		hdr.Set(k, c.Header.Get(k))
	}
	for k := range r.Header {
		hdr.Set(k, r.Header.Get(k))
	}

	if IsStringEmpty(hdr.Get(hdrUserAgentKey)) {
		hdr.Set(hdrUserAgentKey, fmt.Sprintf(hdrUserAgentValue, Version))
	} else {
		hdr.Set("X-"+hdrUserAgentKey, fmt.Sprintf(hdrUserAgentValue, Version))
	}

	if IsStringEmpty(hdr.Get(hdrAcceptKey)) && !IsStringEmpty(hdr.Get(hdrContentTypeKey)) {
		hdr.Set(hdrAcceptKey, hdr.Get(hdrContentTypeKey))
	}

	r.Header = hdr

	return nil
}

func parseRequestBody(c *Client, r *Request) (err error) {
	if isPayloadSupported(r.Method) {

		// Handling Multipart
		if r.isMultiPart && !(r.Method == PATCH) {
			if err = handleMultipart(c, r); err != nil {
				return
			}

			goto CL
		}

		// Handling Form Data
		if len(c.FormData) > 0 || len(r.FormData) > 0 {
			handleFormData(c, r)

			goto CL
		}

		// Handling Request body
		if r.Body != nil {
			handleContentType(c, r)

			if err = handleRequestBody(c, r); err != nil {
				return
			}
		}
	} else {
		r.Header.Del(hdrContentTypeKey)
	}

CL:
	// by default resty won't set content length, you can if you want to :)
	if c.setContentLength || r.setContentLength {
		r.Header.Set(hdrContentLengthKey, fmt.Sprintf("%d", r.bodyBuf.Len()))
	}

	return
}

func createHTTPRequest(c *Client, r *Request) (err error) {
	if r.bodyBuf == nil {
		r.RawRequest, err = http.NewRequest(r.Method, r.URL, nil)
	} else {
		r.RawRequest, err = http.NewRequest(r.Method, r.URL, r.bodyBuf)
	}

	// Add headers into http request
	r.RawRequest.Header = r.Header

	// Add cookies into http request
	for _, cookie := range c.Cookies {
		r.RawRequest.AddCookie(cookie)
	}

	return
}

func addCredentials(c *Client, r *Request) error {
	var isBasicAuth bool
	// Basic Auth
	if r.UserInfo != nil { // takes precedence
		r.RawRequest.SetBasicAuth(r.UserInfo.Username, r.UserInfo.Password)
		isBasicAuth = true
	} else if c.UserInfo != nil {
		r.RawRequest.SetBasicAuth(c.UserInfo.Username, c.UserInfo.Password)
		isBasicAuth = true
	}

	if !c.DisableWarn {
		if isBasicAuth && !strings.HasPrefix(r.URL, "https") {
			c.Log.Println("WARNING - Using Basic Auth in HTTP mode is not secure.")
		}
	}

	// Token Auth
	if !IsStringEmpty(r.Token) { // takes precedence
		r.RawRequest.Header.Set(hdrAuthorizationKey, "Bearer "+r.Token)
	} else if !IsStringEmpty(c.Token) {
		r.RawRequest.Header.Set(hdrAuthorizationKey, "Bearer "+c.Token)
	}

	return nil
}

func requestLogger(c *Client, r *Request) error {
	if c.Debug {
		rr := r.RawRequest
		c.Log.Println()
		c.disableLogPrefix()
		c.Log.Println("---------------------- REQUEST LOG -----------------------")
		c.Log.Printf("%s  %s  %s\n", r.Method, rr.URL.RequestURI(), rr.Proto)
		c.Log.Printf("HOST   : %s", rr.URL.Host)
		c.Log.Println("HEADERS:")
		for h, v := range rr.Header {
			c.Log.Printf("%25s: %v", h, strings.Join(v, ", "))
		}
		c.Log.Printf("BODY   :\n%v", r.fmtBodyString())
		c.Log.Println("----------------------------------------------------------")
		c.enableLogPrefix()
	}

	return nil
}

//
// Response Middleware(s)
//

func responseLogger(c *Client, res *Response) error {
	if c.Debug {
		c.Log.Println()
		c.disableLogPrefix()
		c.Log.Println("---------------------- RESPONSE LOG -----------------------")
		c.Log.Printf("STATUS 		: %s", res.Status())
		c.Log.Printf("RECEIVED AT	: %v", res.ReceivedAt())
		c.Log.Printf("RESPONSE TIME	: %v", res.Time())
		c.Log.Println("HEADERS:")
		for h, v := range res.Header() {
			c.Log.Printf("%30s: %v", h, strings.Join(v, ", "))
		}
		if res.Request.isSaveResponse {
			c.Log.Printf("BODY   :\n***** RESPONSE WRITTEN INTO FILE *****")
		} else {
			c.Log.Printf("BODY   :\n%v", res.fmtBodyString())
		}
		c.Log.Println("----------------------------------------------------------")
		c.enableLogPrefix()
	}

	return nil
}

func parseResponseBody(c *Client, res *Response) (err error) {
	// Handles only JSON or XML content type
	ct := res.Header().Get(hdrContentTypeKey)
	if IsJSONType(ct) || IsXMLType(ct) {
		// Considered as Result
		if res.StatusCode() > 199 && res.StatusCode() < 300 {
			if res.Request.Result != nil {
				err = Unmarshal(ct, res.body, res.Request.Result)
			}
		}

		// Considered as Error
		if res.StatusCode() > 399 {
			// global error interface
			if res.Request.Error == nil && c.Error != nil {
				res.Request.Error = reflect.New(c.Error).Interface()
			}

			if res.Request.Error != nil {
				err = Unmarshal(ct, res.body, res.Request.Error)
			}
		}
	}

	return
}

func handleMultipart(c *Client, r *Request) (err error) {
	r.bodyBuf = &bytes.Buffer{}
	w := multipart.NewWriter(r.bodyBuf)

	for p := range c.FormData {
		w.WriteField(p, c.FormData.Get(p))
	}

	for p := range r.FormData {
		if strings.HasPrefix(p, "@") { // file
			err = addFile(w, p[1:], r.FormData.Get(p))
			if err != nil {
				return
			}
		} else { // form value
			w.WriteField(p, r.FormData.Get(p))
		}
	}

	// #21 - adding io.Reader support
	if len(r.multipartFiles) > 0 {
		for _, f := range r.multipartFiles {
			err = addFileReader(w, f)
			if err != nil {
				return
			}
		}
	}

	r.Header.Set(hdrContentTypeKey, w.FormDataContentType())
	err = w.Close()

	return
}

func handleFormData(c *Client, r *Request) {
	formData := url.Values{}

	for p := range c.FormData {
		formData.Set(p, c.FormData.Get(p))
	}

	for p := range r.FormData { // takes precedence
		formData.Set(p, r.FormData.Get(p))
	}

	r.bodyBuf = bytes.NewBuffer([]byte(formData.Encode()))
	r.Header.Set(hdrContentTypeKey, formContentType)
	r.isFormData = true
}

func handleContentType(c *Client, r *Request) {
	contentType := r.Header.Get(hdrContentTypeKey)
	if IsStringEmpty(contentType) {
		contentType = DetectContentType(r.Body)
		r.Header.Set(hdrContentTypeKey, contentType)
	}
}

func handleRequestBody(c *Client, r *Request) (err error) {
	var bodyBytes []byte
	contentType := r.Header.Get(hdrContentTypeKey)
	kind := kindOf(r.Body)

	if reader, ok := r.Body.(io.Reader); ok {
		r.bodyBuf = &bytes.Buffer{}
		r.bodyBuf.ReadFrom(reader)
	} else if b, ok := r.Body.([]byte); ok {
		bodyBytes = b
	} else if s, ok := r.Body.(string); ok {
		bodyBytes = []byte(s)
	} else if IsJSONType(contentType) &&
		(kind == reflect.Struct || kind == reflect.Map || kind == reflect.Slice) {
		bodyBytes, err = json.Marshal(r.Body)
	} else if IsXMLType(contentType) && (kind == reflect.Struct) {
		bodyBytes, err = xml.Marshal(r.Body)
	}

	if bodyBytes == nil && r.bodyBuf == nil {
		err = errors.New("Unsupported 'Body' type/value")
	}

	// if any errors during body bytes handling, return it
	if err != nil {
		return
	}

	// []byte into Buffer
	if bodyBytes != nil && r.bodyBuf == nil {
		r.bodyBuf = bytes.NewBuffer(bodyBytes)
	}

	return
}

func saveResponseIntoFile(c *Client, res *Response) error {
	if res.Request.isSaveResponse {
		file := ""

		if len(c.outputDirectory) > 0 && !filepath.IsAbs(res.Request.outputFile) {
			file += c.outputDirectory + string(filepath.Separator)
		}

		file = filepath.Clean(file + res.Request.outputFile)
		err := createDirectory(filepath.Dir(file))
		if err != nil {
			return err
		}

		outFile, err := os.Create(file)
		if err != nil {
			return err
		}
		defer outFile.Close()

		// io.Copy reads maximum 32kb size, it is perfect for large file download too
		defer res.RawResponse.Body.Close()
		written, err := io.Copy(outFile, res.RawResponse.Body)
		if err != nil {
			return err
		}

		res.size = written
	}

	return nil
}
<|endoftext|>


<commit_before>package negronilogrus

import (
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/negroni"
)

// Middleware is a middleware handler that logs the request as it goes in and the response as it goes out.
type Middleware struct {
	// Logger is the log.Logger instance used to log messages with the Logger middleware
	Logger *logrus.Logger
}

// NewMiddleware returns a new *Middleware, yay!
func NewMiddleware() *Middleware {
	return NewCustomMiddleware(logrus.Info, &logrus.TextFormatter{})
}

// NewCustomMiddleware builds a *Middleware with the given level and formatter
func NewCustomMiddleware(level logrus.Level, formatter logrus.Formatter) *Middleware {
	log := logrus.New()
	log.Level = level
	log.Formatter = formatter

	return &Middleware{Logger: log}
}

func (l *Middleware) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	start := time.Now()
	l.Logger.WithFields(logrus.Fields{
		"method":  r.Method,
		"request": r.RequestURI,
		"remote":  r.RemoteAddr,
	}).Info("started handling request")

	next(rw, r)

	latency := time.Since(start)
	res := rw.(negroni.ResponseWriter)
	l.Logger.WithFields(logrus.Fields{
		"status":              res.Status(),
		"method":              r.Method,
		"request":             r.RequestURI,
		"remote":              r.RemoteAddr,
		"text_status":         http.StatusText(res.Status()),
		"took":                latency,
		"measure#web.latency": latency,
	}).Info("completed handling request")
}
<commit_msg>Allowing for custom latency metric key name<commit_after>package negronilogrus

import (
	"fmt"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/negroni"
)

// Middleware is a middleware handler that logs the request as it goes in and the response as it goes out.
type Middleware struct {
	// Logger is the log.Logger instance used to log messages with the Logger middleware
	Logger *logrus.Logger
	// Name is the name of the application as recorded in latency metrics
	Name string
}

// NewMiddleware returns a new *Middleware, yay!
func NewMiddleware() *Middleware {
	return NewCustomMiddleware(logrus.Info, &logrus.TextFormatter{}, "web")
}

// NewCustomMiddleware builds a *Middleware with the given level and formatter
func NewCustomMiddleware(level logrus.Level, formatter logrus.Formatter, name string) *Middleware {
	log := logrus.New()
	log.Level = level
	log.Formatter = formatter

	return &Middleware{Logger: log, Name: name}
}

func (l *Middleware) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	start := time.Now()
	l.Logger.WithFields(logrus.Fields{
		"method":  r.Method,
		"request": r.RequestURI,
		"remote":  r.RemoteAddr,
	}).Info("started handling request")

	next(rw, r)

	latency := time.Since(start)
	res := rw.(negroni.ResponseWriter)
	l.Logger.WithFields(logrus.Fields{
		"status":      res.Status(),
		"method":      r.Method,
		"request":     r.RequestURI,
		"remote":      r.RemoteAddr,
		"text_status": http.StatusText(res.Status()),
		"took":        latency,
		fmt.Sprintf("measure#%s.latency", l.Name): latency,
	}).Info("completed handling request")
}
<|endoftext|>


<commit_before>package date

import (
	"database/sql/driver"
	"errors"
	"time"
)

const (
	MillisecondsInSecond = 1000
	MicrosecondsInSecond = 1000 * MillisecondsInSecond
	NanosecondsInSecond  = 1000 * MicrosecondsInSecond
)

// Date is a plain date, without time or timezone info (use time.Time for those!)
type Date struct {
	Year     int
	Month    time.Month
	Day      int
	Location *time.Location
}

// Create a Date from a time.Time object
func From(t time.Time) Date {
	y, m, d := t.Date()
	return Date{y, m, d, t.Location()}
}

func (d Date) PrevDay() Date {
	if d.Day > 1 {
		return Date{d.Year, d.Month, d.Day - 1, d.Location}
	} else if d.Month > 1 {
		return Date{d.Year, d.Month - 1, DaysInMonth(d.Month-1, d.Year), d.Location}
	} else {
		return Date{d.Year - 1, time.December, 31, d.Location}
	}
}

func (d Date) NextDay() Date {
	if d.Day < DaysInMonth(d.Month, d.Year) {
		return Date{d.Year, d.Month, d.Day + 1, d.Location}
	} else if d.Month < time.December {
		return Date{d.Year, d.Month + 1, 1, d.Location}
	} else {
		return Date{d.Year + 1, time.January, 1, d.Location}
	}
}

func (d Date) Before(other Date) bool {
	return (d.Year < other.Year ||
		(d.Year == other.Year &&
			(d.Month < other.Month ||
				(d.Month == other.Month && d.Day < other.Day))))
}

func (d Date) BeginningOfDay() time.Time {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, d.Location)
}

func (d Date) BeginningOfDayIn(timezone *time.Location) time.Time {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, timezone)
}

func (d Date) EndOfDay() time.Time {
	return time.Date(d.Year, d.Month, d.Day, 23, 59, 59, NanosecondsInSecond-1, d.Location)
}

func (d Date) EndOfDayIn(timezone *time.Location) time.Time {
	return time.Date(d.Year, d.Month, d.Day, 23, 59, 59, NanosecondsInSecond-1, timezone)
}

func (d Date) String() string {
	return d.BeginningOfDayIn(time.UTC).Format("2006-01-02")
}

// Implement sql.Scanner interface
func (d Date) Scan(src interface{}) error {
	t, ok := src.(time.Time)
	if !ok {
		return errors.New("unstable/date: scan value was not a Time")
	}

	d = From(t)
	return nil
}

// Implement sql.driver.Valuer interface
func (d Date) Value() (driver.Value, error) {
	return d.BeginningOfDayIn(time.UTC), nil
}

// Implement json.Marshaler interface
func (d Date) MarshalJSON() ([]byte, error) {
	return []byte(d.BeginningOfDayIn(time.UTC).Format(`"2006-01-02"`)), nil
}

// Implement json.Unmarshaler interface
func (d *Date) UnmarshalJSON(bytes []byte) error {
	t, err := time.Parse(`"2006-01-02"`, string(bytes))
	if err != nil {
		return err
	}

	*d = From(t)
	return nil
}

func IsLeapYear(year int) bool {
	return year%4 == 0 && (year%100 != 0 || year%400 == 0)
}

var DaysInNonLeapMonth = [12]int32{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}

func DaysInMonth(m time.Month, year int) int {
	if m == time.February && IsLeapYear(year) {
		return 29
	} else {
		return int(DaysInNonLeapMonth[m-1])
	}
}
<commit_msg>date: add more comparison methods<commit_after>package date

import (
	"database/sql/driver"
	"errors"
	"time"
)

const (
	MillisecondsInSecond = 1000
	MicrosecondsInSecond = 1000 * MillisecondsInSecond
	NanosecondsInSecond  = 1000 * MicrosecondsInSecond
)

// Date is a plain date, without time or timezone info (use time.Time for those!)
type Date struct {
	Year     int
	Month    time.Month
	Day      int
	Location *time.Location
}

// Create a Date from a time.Time object
func From(t time.Time) Date {
	y, m, d := t.Date()
	return Date{y, m, d, t.Location()}
}

func (d Date) PrevDay() Date {
	if d.Day > 1 {
		return Date{d.Year, d.Month, d.Day - 1, d.Location}
	} else if d.Month > 1 {
		return Date{d.Year, d.Month - 1, DaysInMonth(d.Month-1, d.Year), d.Location}
	} else {
		return Date{d.Year - 1, time.December, 31, d.Location}
	}
}

func (d Date) NextDay() Date {
	if d.Day < DaysInMonth(d.Month, d.Year) {
		return Date{d.Year, d.Month, d.Day + 1, d.Location}
	} else if d.Month < time.December {
		return Date{d.Year, d.Month + 1, 1, d.Location}
	} else {
		return Date{d.Year + 1, time.January, 1, d.Location}
	}
}

func (d Date) Before(other Date) bool {
	return (d.Year < other.Year ||
		(d.Year == other.Year &&
			(d.Month < other.Month ||
				(d.Month == other.Month && d.Day < other.Day))))
}

func (d Date) After(other Date) bool {
	return (d.Year > other.Year ||
		(d.Year == other.Year &&
			(d.Month > other.Month ||
				(d.Month == other.Month && d.Day > other.Day))))
}

func (d Date) AtLeast(other Date) bool {
	return d == other || d.After(other)
}

func (d Date) AtMost(other Date) bool {
	return d == other || d.Before(other)
}

func (d Date) BeginningOfDay() time.Time {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, d.Location)
}

func (d Date) BeginningOfDayIn(timezone *time.Location) time.Time {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, timezone)
}

func (d Date) EndOfDay() time.Time {
	return time.Date(d.Year, d.Month, d.Day, 23, 59, 59, NanosecondsInSecond-1, d.Location)
}

func (d Date) EndOfDayIn(timezone *time.Location) time.Time {
	return time.Date(d.Year, d.Month, d.Day, 23, 59, 59, NanosecondsInSecond-1, timezone)
}

func (d Date) String() string {
	return d.BeginningOfDayIn(time.UTC).Format("2006-01-02")
}

// Implement sql.Scanner interface
func (d Date) Scan(src interface{}) error {
	t, ok := src.(time.Time)
	if !ok {
		return errors.New("unstable/date: scan value was not a Time")
	}

	d = From(t)
	return nil
}

// Implement sql.driver.Valuer interface
func (d Date) Value() (driver.Value, error) {
	return d.BeginningOfDayIn(time.UTC), nil
}

// Implement json.Marshaler interface
func (d Date) MarshalJSON() ([]byte, error) {
	return []byte(d.BeginningOfDayIn(time.UTC).Format(`"2006-01-02"`)), nil
}

// Implement json.Unmarshaler interface
func (d *Date) UnmarshalJSON(bytes []byte) error {
	t, err := time.Parse(`"2006-01-02"`, string(bytes))
	if err != nil {
		return err
	}

	*d = From(t)
	return nil
}

func IsLeapYear(year int) bool {
	return year%4 == 0 && (year%100 != 0 || year%400 == 0)
}

var DaysInNonLeapMonth = [12]int32{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}

func DaysInMonth(m time.Month, year int) int {
	if m == time.February && IsLeapYear(year) {
		return 29
	} else {
		return int(DaysInNonLeapMonth[m-1])
	}
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"reflect"

	"github.com/coreos/go-oidc/oidc"
	"github.com/go-gorp/gorp"
	"github.com/lib/pq"
	"golang.org/x/crypto/bcrypt"

	"github.com/coreos/dex/client"
	pcrypto "github.com/coreos/dex/pkg/crypto"
	"github.com/coreos/dex/pkg/log"
)

const (
	clientIdentityTableName = "client_identity"

	bcryptHashCost = 10

	// Blowfish, the algorithm underlying bcrypt, has a maximum
	// password length of 72. We explicitly track and check this
	// since the bcrypt library will silently ignore portions of
	// a password past the first 72 characters.
	maxSecretLength = 72

	// postgres error codes
	pgErrorCodeUniqueViolation = "23505" // unique_violation
)

func init() {
	register(table{
		name:    clientIdentityTableName,
		model:   clientIdentityModel{},
		autoinc: false,
		pkey:    []string{"id"},
	})
}

func newClientIdentityModel(id string, secret []byte, meta *oidc.ClientMetadata) (*clientIdentityModel, error) {
	hashed, err := bcrypt.GenerateFromPassword(secret, bcryptHashCost)
	if err != nil {
		return nil, err
	}

	bmeta, err := json.Marshal(newClientMetadataJSON(meta))
	if err != nil {
		return nil, err
	}

	cim := clientIdentityModel{
		ID:       id,
		Secret:   hashed,
		Metadata: string(bmeta),
	}

	return &cim, nil
}

type clientIdentityModel struct {
	ID       string `db:"id"`
	Secret   []byte `db:"secret"`
	Metadata string `db:"metadata"`
	DexAdmin bool   `db:"dex_admin"`
}

func newClientMetadataJSON(cm *oidc.ClientMetadata) *clientMetadataJSON {
	cmj := clientMetadataJSON{
		RedirectURLs: make([]string, len(cm.RedirectURLs)),
	}

	for i, u := range cm.RedirectURLs {
		cmj.RedirectURLs[i] = (&u).String()
	}

	return &cmj
}

type clientMetadataJSON struct {
	RedirectURLs []string `json:"redirectURLs"`
}

func (cmj clientMetadataJSON) ClientMetadata() (*oidc.ClientMetadata, error) {
	cm := oidc.ClientMetadata{
		RedirectURLs: make([]url.URL, len(cmj.RedirectURLs)),
	}

	for i, us := range cmj.RedirectURLs {
		up, err := url.Parse(us)
		if err != nil {
			return nil, err
		}
		cm.RedirectURLs[i] = *up
	}

	return &cm, nil
}

func (m *clientIdentityModel) ClientIdentity() (*oidc.ClientIdentity, error) {
	ci := oidc.ClientIdentity{
		Credentials: oidc.ClientCredentials{
			ID:     m.ID,
			Secret: string(m.Secret),
		},
	}

	var cmj clientMetadataJSON
	err := json.Unmarshal([]byte(m.Metadata), &cmj)
	if err != nil {
		return nil, err
	}

	cm, err := cmj.ClientMetadata()
	if err != nil {
		return nil, err
	}

	ci.Metadata = *cm
	return &ci, nil
}

func NewClientIdentityRepo(dbm *gorp.DbMap) client.ClientIdentityRepo {
	return &clientIdentityRepo{dbMap: dbm}
}

func NewClientIdentityRepoFromClients(dbm *gorp.DbMap, clients []oidc.ClientIdentity) (client.ClientIdentityRepo, error) {
	repo := NewClientIdentityRepo(dbm).(*clientIdentityRepo)
	for _, c := range clients {
		dec, err := base64.URLEncoding.DecodeString(c.Credentials.Secret)
		if err != nil {
			return nil, err
		}

		cm, err := newClientIdentityModel(c.Credentials.ID, dec, &c.Metadata)
		if err != nil {
			return nil, err
		}
		err = repo.dbMap.Insert(cm)
		if err != nil {
			return nil, err
		}
	}
	return repo, nil
}

type clientIdentityRepo struct {
	dbMap *gorp.DbMap
}

func (r *clientIdentityRepo) Metadata(clientID string) (*oidc.ClientMetadata, error) {
	m, err := r.dbMap.Get(clientIdentityModel{}, clientID)
	if err == sql.ErrNoRows || m == nil {
		return nil, client.ErrorNotFound
	}
	if err != nil {
		return nil, err
	}

	cim, ok := m.(*clientIdentityModel)
	if !ok {
		log.Errorf("expected clientIdentityModel but found %v", reflect.TypeOf(m))
		return nil, errors.New("unrecognized model")
	}

	ci, err := cim.ClientIdentity()
	if err != nil {
		return nil, err
	}

	return &ci.Metadata, nil
}

func (r *clientIdentityRepo) IsDexAdmin(clientID string) (bool, error) {
	m, err := r.dbMap.Get(clientIdentityModel{}, clientID)
	if m == nil || err != nil {
		return false, err
	}

	cim, ok := m.(*clientIdentityModel)
	if !ok {
		log.Errorf("expected clientIdentityModel but found %v", reflect.TypeOf(m))
		return false, errors.New("unrecognized model")
	}

	return cim.DexAdmin, nil
}

func (r *clientIdentityRepo) SetDexAdmin(clientID string, isAdmin bool) error {
	tx, err := r.dbMap.Begin()
	if err != nil {
		return err
	}

	m, err := r.dbMap.Get(clientIdentityModel{}, clientID)
	if m == nil || err != nil {
		rollback(tx)
		return err
	}

	cim, ok := m.(*clientIdentityModel)
	if !ok {
		rollback(tx)
		log.Errorf("expected clientIdentityModel but found %v", reflect.TypeOf(m))
		return errors.New("unrecognized model")
	}

	cim.DexAdmin = isAdmin
	_, err = r.dbMap.Update(cim)
	if err != nil {
		rollback(tx)
		return err
	}

	err = tx.Commit()
	if err != nil {
		rollback(tx)
		return err
	}

	return nil
}

func (r *clientIdentityRepo) Authenticate(creds oidc.ClientCredentials) (bool, error) {
	m, err := r.dbMap.Get(clientIdentityModel{}, creds.ID)
	if m == nil || err != nil {
		return false, err
	}

	cim, ok := m.(*clientIdentityModel)
	if !ok {
		log.Errorf("expected clientIdentityModel but found %v", reflect.TypeOf(m))
		return false, errors.New("unrecognized model")
	}

	dec, err := base64.URLEncoding.DecodeString(creds.Secret)
	if err != nil {
		log.Errorf("error Decoding client creds ")
		return false, nil
	}

	if len(dec) > maxSecretLength {
		return false, nil
	}

	ok = bcrypt.CompareHashAndPassword(cim.Secret, dec) == nil
	return ok, nil
}

func (r *clientIdentityRepo) New(id string, meta oidc.ClientMetadata) (*oidc.ClientCredentials, error) {
	secret, err := pcrypto.RandBytes(maxSecretLength)
	if err != nil {
		return nil, err
	}

	cim, err := newClientIdentityModel(id, secret, &meta)
	if err != nil {
		return nil, err
	}

	if err := r.dbMap.Insert(cim); err != nil {
		if perr, ok := err.(*pq.Error); ok && perr.Code == pgErrorCodeUniqueViolation {
			err = errors.New("client ID already exists")
		}

		return nil, err
	}

	cc := oidc.ClientCredentials{
		ID:     id,
		Secret: base64.URLEncoding.EncodeToString(secret),
	}

	return &cc, nil
}

func (r *clientIdentityRepo) All() ([]oidc.ClientIdentity, error) {
	qt := pq.QuoteIdentifier(clientIdentityTableName)
	q := fmt.Sprintf("SELECT * FROM %s", qt)
	objs, err := r.dbMap.Select(&clientIdentityModel{}, q)
	if err != nil {
		return nil, err
	}

	cs := make([]oidc.ClientIdentity, len(objs))
	for i, obj := range objs {
		m, ok := obj.(*clientIdentityModel)
		if !ok {
			return nil, errors.New("unable to cast client identity to clientIdentityModel")
		}

		ci, err := m.ClientIdentity()
		if err != nil {
			return nil, err
		}
		cs[i] = *ci
	}
	return cs, nil
}
<commit_msg>db: log ignored base64 decode error<commit_after>package db

import (
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"reflect"

	"github.com/coreos/go-oidc/oidc"
	"github.com/go-gorp/gorp"
	"github.com/lib/pq"
	"golang.org/x/crypto/bcrypt"

	"github.com/coreos/dex/client"
	pcrypto "github.com/coreos/dex/pkg/crypto"
	"github.com/coreos/dex/pkg/log"
)

const (
	clientIdentityTableName = "client_identity"

	bcryptHashCost = 10

	// Blowfish, the algorithm underlying bcrypt, has a maximum
	// password length of 72. We explicitly track and check this
	// since the bcrypt library will silently ignore portions of
	// a password past the first 72 characters.
	maxSecretLength = 72

	// postgres error codes
	pgErrorCodeUniqueViolation = "23505" // unique_violation
)

func init() {
	register(table{
		name:    clientIdentityTableName,
		model:   clientIdentityModel{},
		autoinc: false,
		pkey:    []string{"id"},
	})
}

func newClientIdentityModel(id string, secret []byte, meta *oidc.ClientMetadata) (*clientIdentityModel, error) {
	hashed, err := bcrypt.GenerateFromPassword(secret, bcryptHashCost)
	if err != nil {
		return nil, err
	}

	bmeta, err := json.Marshal(newClientMetadataJSON(meta))
	if err != nil {
		return nil, err
	}

	cim := clientIdentityModel{
		ID:       id,
		Secret:   hashed,
		Metadata: string(bmeta),
	}

	return &cim, nil
}

type clientIdentityModel struct {
	ID       string `db:"id"`
	Secret   []byte `db:"secret"`
	Metadata string `db:"metadata"`
	DexAdmin bool   `db:"dex_admin"`
}

func newClientMetadataJSON(cm *oidc.ClientMetadata) *clientMetadataJSON {
	cmj := clientMetadataJSON{
		RedirectURLs: make([]string, len(cm.RedirectURLs)),
	}

	for i, u := range cm.RedirectURLs {
		cmj.RedirectURLs[i] = (&u).String()
	}

	return &cmj
}

type clientMetadataJSON struct {
	RedirectURLs []string `json:"redirectURLs"`
}

func (cmj clientMetadataJSON) ClientMetadata() (*oidc.ClientMetadata, error) {
	cm := oidc.ClientMetadata{
		RedirectURLs: make([]url.URL, len(cmj.RedirectURLs)),
	}

	for i, us := range cmj.RedirectURLs {
		up, err := url.Parse(us)
		if err != nil {
			return nil, err
		}
		cm.RedirectURLs[i] = *up
	}

	return &cm, nil
}

func (m *clientIdentityModel) ClientIdentity() (*oidc.ClientIdentity, error) {
	ci := oidc.ClientIdentity{
		Credentials: oidc.ClientCredentials{
			ID:     m.ID,
			Secret: string(m.Secret),
		},
	}

	var cmj clientMetadataJSON
	err := json.Unmarshal([]byte(m.Metadata), &cmj)
	if err != nil {
		return nil, err
	}

	cm, err := cmj.ClientMetadata()
	if err != nil {
		return nil, err
	}

	ci.Metadata = *cm
	return &ci, nil
}

func NewClientIdentityRepo(dbm *gorp.DbMap) client.ClientIdentityRepo {
	return &clientIdentityRepo{dbMap: dbm}
}

func NewClientIdentityRepoFromClients(dbm *gorp.DbMap, clients []oidc.ClientIdentity) (client.ClientIdentityRepo, error) {
	repo := NewClientIdentityRepo(dbm).(*clientIdentityRepo)
	for _, c := range clients {
		dec, err := base64.URLEncoding.DecodeString(c.Credentials.Secret)
		if err != nil {
			return nil, err
		}

		cm, err := newClientIdentityModel(c.Credentials.ID, dec, &c.Metadata)
		if err != nil {
			return nil, err
		}
		err = repo.dbMap.Insert(cm)
		if err != nil {
			return nil, err
		}
	}
	return repo, nil
}

type clientIdentityRepo struct {
	dbMap *gorp.DbMap
}

func (r *clientIdentityRepo) Metadata(clientID string) (*oidc.ClientMetadata, error) {
	m, err := r.dbMap.Get(clientIdentityModel{}, clientID)
	if err == sql.ErrNoRows || m == nil {
		return nil, client.ErrorNotFound
	}
	if err != nil {
		return nil, err
	}

	cim, ok := m.(*clientIdentityModel)
	if !ok {
		log.Errorf("expected clientIdentityModel but found %v", reflect.TypeOf(m))
		return nil, errors.New("unrecognized model")
	}

	ci, err := cim.ClientIdentity()
	if err != nil {
		return nil, err
	}

	return &ci.Metadata, nil
}

func (r *clientIdentityRepo) IsDexAdmin(clientID string) (bool, error) {
	m, err := r.dbMap.Get(clientIdentityModel{}, clientID)
	if m == nil || err != nil {
		return false, err
	}

	cim, ok := m.(*clientIdentityModel)
	if !ok {
		log.Errorf("expected clientIdentityModel but found %v", reflect.TypeOf(m))
		return false, errors.New("unrecognized model")
	}

	return cim.DexAdmin, nil
}

func (r *clientIdentityRepo) SetDexAdmin(clientID string, isAdmin bool) error {
	tx, err := r.dbMap.Begin()
	if err != nil {
		return err
	}

	m, err := r.dbMap.Get(clientIdentityModel{}, clientID)
	if m == nil || err != nil {
		rollback(tx)
		return err
	}

	cim, ok := m.(*clientIdentityModel)
	if !ok {
		rollback(tx)
		log.Errorf("expected clientIdentityModel but found %v", reflect.TypeOf(m))
		return errors.New("unrecognized model")
	}

	cim.DexAdmin = isAdmin
	_, err = r.dbMap.Update(cim)
	if err != nil {
		rollback(tx)
		return err
	}

	err = tx.Commit()
	if err != nil {
		rollback(tx)
		return err
	}

	return nil
}

func (r *clientIdentityRepo) Authenticate(creds oidc.ClientCredentials) (bool, error) {
	m, err := r.dbMap.Get(clientIdentityModel{}, creds.ID)
	if m == nil || err != nil {
		return false, err
	}

	cim, ok := m.(*clientIdentityModel)
	if !ok {
		log.Errorf("expected clientIdentityModel but found %v", reflect.TypeOf(m))
		return false, errors.New("unrecognized model")
	}

	dec, err := base64.URLEncoding.DecodeString(creds.Secret)
	if err != nil {
		log.Errorf("error Decoding client creds: %v", err)
		return false, nil
	}

	if len(dec) > maxSecretLength {
		return false, nil
	}

	ok = bcrypt.CompareHashAndPassword(cim.Secret, dec) == nil
	return ok, nil
}

func (r *clientIdentityRepo) New(id string, meta oidc.ClientMetadata) (*oidc.ClientCredentials, error) {
	secret, err := pcrypto.RandBytes(maxSecretLength)
	if err != nil {
		return nil, err
	}

	cim, err := newClientIdentityModel(id, secret, &meta)
	if err != nil {
		return nil, err
	}

	if err := r.dbMap.Insert(cim); err != nil {
		if perr, ok := err.(*pq.Error); ok && perr.Code == pgErrorCodeUniqueViolation {
			err = errors.New("client ID already exists")
		}

		return nil, err
	}

	cc := oidc.ClientCredentials{
		ID:     id,
		Secret: base64.URLEncoding.EncodeToString(secret),
	}

	return &cc, nil
}

func (r *clientIdentityRepo) All() ([]oidc.ClientIdentity, error) {
	qt := pq.QuoteIdentifier(clientIdentityTableName)
	q := fmt.Sprintf("SELECT * FROM %s", qt)
	objs, err := r.dbMap.Select(&clientIdentityModel{}, q)
	if err != nil {
		return nil, err
	}

	cs := make([]oidc.ClientIdentity, len(objs))
	for i, obj := range objs {
		m, ok := obj.(*clientIdentityModel)
		if !ok {
			return nil, errors.New("unable to cast client identity to clientIdentityModel")
		}

		ci, err := m.ClientIdentity()
		if err != nil {
			return nil, err
		}
		cs[i] = *ci
	}
	return cs, nil
}
<|endoftext|>


<commit_before>package mysql

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"github.com/satori/go.uuid"
	"github.com/siddontang/go/hack"
	"io"
	"sort"
	"strconv"
	"strings"
)

// Like MySQL GTID Interval struct, [start, stop), left closed and right open
// See MySQL rpl_gtid.h
type Interval struct {
	// The first GID of this interval.
	Start int64
	// The first GID after this interval.
	Stop int64
}

// Interval is [start, stop), but the GTID string's format is [n] or [n1-n2], closed interval
func parseInterval(str string) (i Interval, err error) {
	p := strings.Split(str, "-")
	switch len(p) {
	case 1:
		i.Start, err = strconv.ParseInt(p[0], 10, 64)
		i.Stop = i.Start + 1
	case 2:
		i.Start, err = strconv.ParseInt(p[0], 10, 64)
		i.Stop, err = strconv.ParseInt(p[1], 10, 64)
		i.Stop = i.Stop + 1
	default:
		err = fmt.Errorf("invalid interval format, must n[-n]")
	}

	if err != nil {
		return
	}

	if i.Stop <= i.Start {
		err = fmt.Errorf("invalid interval format, must n[-n] and the end must >= start")
	}

	return
}

func (i Interval) String() string {
	if i.Stop == i.Start+1 {
		return fmt.Sprintf("%d", i.Start)
	} else {
		return fmt.Sprintf("%d-%d", i.Start, i.Stop-1)
	}
}

type IntervalSlice []Interval

func (s IntervalSlice) Len() int {
	return len(s)
}

func (s IntervalSlice) Less(i, j int) bool {
	if s[i].Start < s[j].Start {
		return true
	} else if s[i].Start > s[j].Start {
		return false
	} else {
		return s[i].Stop < s[j].Stop
	}
}

func (s IntervalSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s IntervalSlice) Sort() {
	sort.Sort(s)
}

func (s IntervalSlice) Normalize() IntervalSlice {
	var n IntervalSlice
	if len(s) == 0 {
		return n
	}

	s.Sort()

	n = append(n, s[0])

	for i := 1; i < len(s); i++ {
		last := n[len(n)-1]
		if s[i].Start > last.Stop {
			n = append(n, s[i])
			continue
		} else {
			n[len(n)-1] = Interval{last.Start, s[i].Stop}
		}
	}

	return n
}

// Return true if sub in s
func (s IntervalSlice) Subset(sub IntervalSlice) bool {
	j := 0
	for i := 0; i < len(sub); i++ {
		for ; j < len(s); j++ {
			if sub[i].Start > s[j].Stop {
				continue
			} else {
				break
			}
		}
		if j == len(s) {
			return false
		}

		if sub[i].Start < s[j].Start || sub[i].Stop > s[j].Stop {
			return false
		}
	}

	return true
}

func (s IntervalSlice) Equal(o IntervalSlice) bool {
	if len(s) != len(o) {
		return false
	}

	for i := 0; i < len(s); i++ {
		if s[i].Start != o[i].Start || s[i].Stop != o[i].Stop {
			return false
		}
	}

	return true
}

// Refer http://dev.mysql.com/doc/refman/5.6/en/replication-gtids-concepts.html
type UUIDSet struct {
	SID uuid.UUID

	Intervals IntervalSlice
}

func ParseUUIDSet(str string) (*UUIDSet, error) {
	sep := strings.Split(str, ":")
	if len(sep) < 2 {
		return nil, fmt.Errorf("invalid GTID format, must UUID:interval[:interval]")
	}

	var err error
	s := new(UUIDSet)
	if s.SID, err = uuid.FromString(sep[0]); err != nil {
		return nil, err
	}

	// Handle interval
	for i := 1; i < len(sep); i++ {
		if in, err := parseInterval(sep[i]); err != nil {
			return nil, err
		} else {
			s.Intervals = append(s.Intervals, in)
		}
	}

	s.Intervals = s.Intervals.Normalize()

	return s, nil
}

func NewUUIDSet(sid uuid.UUID, in ...Interval) *UUIDSet {
	s := new(UUIDSet)
	s.SID = sid

	s.Intervals = in
	s.Intervals = s.Intervals.Normalize()

	return s
}

func (s *UUIDSet) Subset(sub *UUIDSet) bool {
	if !bytes.Equal(s.SID.Bytes(), sub.SID.Bytes()) {
		return false
	}

	return s.Intervals.Subset(sub.Intervals)
}

func (s *UUIDSet) Bytes() []byte {
	var buf bytes.Buffer

	buf.WriteString(s.SID.String())

	for _, i := range s.Intervals {
		buf.WriteString(":")
		buf.WriteString(i.String())
	}

	return buf.Bytes()
}

func (s *UUIDSet) AddInterval(in IntervalSlice) {
	s.Intervals = append(s.Intervals, in...)
	s.Intervals = s.Intervals.Normalize()
}

func (s *UUIDSet) String() string {
	return hack.String(s.Bytes())
}

func (s *UUIDSet) encode(w io.Writer) {
	w.Write(s.SID.Bytes())
	n := int64(len(s.Intervals))

	binary.Write(w, binary.LittleEndian, n)

	for _, i := range s.Intervals {
		binary.Write(w, binary.LittleEndian, i.Start)
		binary.Write(w, binary.LittleEndian, i.Stop)
	}
}

func (s *UUIDSet) Encode() []byte {
	var buf bytes.Buffer

	s.encode(&buf)

	return buf.Bytes()
}

func (s *UUIDSet) decode(data []byte) (int, error) {
	if len(data) < 24 {
		return 0, fmt.Errorf("invalid uuid set buffer, less 24")
	}

	pos := 0
	var err error
	if s.SID, err = uuid.FromBytes(data[0:16]); err != nil {
		return 0, err
	}
	pos += 16

	n := int64(binary.LittleEndian.Uint64(data[pos : pos+8]))
	pos += 8
	if len(data) < int(16*n)+pos {
		return 0, fmt.Errorf("invalid uuid set buffer, must %d, but %d", pos+int(16*n), len(data))
	}

	s.Intervals = make([]Interval, 0, n)

	var in Interval
	for i := int64(0); i < n; i++ {
		in.Start = int64(binary.LittleEndian.Uint64(data[pos : pos+8]))
		pos += 8
		in.Stop = int64(binary.LittleEndian.Uint64(data[pos : pos+8]))
		pos += 8
		s.Intervals = append(s.Intervals, in)
	}

	return pos, nil
}

func (s *UUIDSet) Decode(data []byte) error {
	n, err := s.decode(data)
	if n != len(data) {
		return fmt.Errorf("invalid uuid set buffer, must %d, but %d", n, len(data))
	}
	return err
}

type GTIDSet struct {
	Sets map[string]*UUIDSet
}

func ParseGTIDSet(str string) (*GTIDSet, error) {
	s := new(GTIDSet)

	sp := strings.Split(str, ",")

	s.Sets = make(map[string]*UUIDSet, len(sp))

	//todo, handle redundant same uuid
	for i := 0; i < len(sp); i++ {
		if set, err := ParseUUIDSet(sp[i]); err != nil {
			return nil, err
		} else {
			s.AddSet(set)
		}

	}
	return s, nil
}

func (s *GTIDSet) AddSet(set *UUIDSet) {
	sid := set.SID.String()
	o, ok := s.Sets[sid]
	if ok {
		o.AddInterval(set.Intervals)
	} else {
		s.Sets[sid] = set
	}
}

// Return true if sub in s
func (s *GTIDSet) Subset(sub *GTIDSet) bool {
	for key, set := range sub.Sets {
		o, ok := s.Sets[key]
		if !ok {
			return false
		}

		if !o.Subset(set) {
			return false
		}
	}

	return true
}

func (s *GTIDSet) String() string {
	var buf bytes.Buffer
	sep := ""
	for _, set := range s.Sets {
		buf.WriteString(sep)
		buf.WriteString(set.String())
		sep = ","
	}

	return hack.String(buf.Bytes())
}

func (s *GTIDSet) Encode() []byte {
	var buf bytes.Buffer

	binary.Write(&buf, binary.LittleEndian, uint64(len(s.Sets)))

	for i, _ := range s.Sets {
		s.Sets[i].encode(&buf)
	}

	return buf.Bytes()
}

func (s *GTIDSet) Decode(data []byte) error {
	if len(data) < 8 {
		return fmt.Errorf("invalid gtid set buffer, less 4")
	}

	n := int(binary.LittleEndian.Uint64(data))
	s.Sets = make(map[string]*UUIDSet, n)

	pos := 8

	for i := 0; i < n; i++ {
		set := new(UUIDSet)
		if n, err := set.decode(data[pos:]); err != nil {
			return err
		} else {
			pos += n

			s.AddSet(set)
		}
	}
	return nil
}
<commit_msg>trim space for parsing GTID first<commit_after>package mysql

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"github.com/satori/go.uuid"
	"github.com/siddontang/go/hack"
	"io"
	"sort"
	"strconv"
	"strings"
)

// Like MySQL GTID Interval struct, [start, stop), left closed and right open
// See MySQL rpl_gtid.h
type Interval struct {
	// The first GID of this interval.
	Start int64
	// The first GID after this interval.
	Stop int64
}

// Interval is [start, stop), but the GTID string's format is [n] or [n1-n2], closed interval
func parseInterval(str string) (i Interval, err error) {
	p := strings.Split(str, "-")
	switch len(p) {
	case 1:
		i.Start, err = strconv.ParseInt(p[0], 10, 64)
		i.Stop = i.Start + 1
	case 2:
		i.Start, err = strconv.ParseInt(p[0], 10, 64)
		i.Stop, err = strconv.ParseInt(p[1], 10, 64)
		i.Stop = i.Stop + 1
	default:
		err = fmt.Errorf("invalid interval format, must n[-n]")
	}

	if err != nil {
		return
	}

	if i.Stop <= i.Start {
		err = fmt.Errorf("invalid interval format, must n[-n] and the end must >= start")
	}

	return
}

func (i Interval) String() string {
	if i.Stop == i.Start+1 {
		return fmt.Sprintf("%d", i.Start)
	} else {
		return fmt.Sprintf("%d-%d", i.Start, i.Stop-1)
	}
}

type IntervalSlice []Interval

func (s IntervalSlice) Len() int {
	return len(s)
}

func (s IntervalSlice) Less(i, j int) bool {
	if s[i].Start < s[j].Start {
		return true
	} else if s[i].Start > s[j].Start {
		return false
	} else {
		return s[i].Stop < s[j].Stop
	}
}

func (s IntervalSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s IntervalSlice) Sort() {
	sort.Sort(s)
}

func (s IntervalSlice) Normalize() IntervalSlice {
	var n IntervalSlice
	if len(s) == 0 {
		return n
	}

	s.Sort()

	n = append(n, s[0])

	for i := 1; i < len(s); i++ {
		last := n[len(n)-1]
		if s[i].Start > last.Stop {
			n = append(n, s[i])
			continue
		} else {
			n[len(n)-1] = Interval{last.Start, s[i].Stop}
		}
	}

	return n
}

// Return true if sub in s
func (s IntervalSlice) Subset(sub IntervalSlice) bool {
	j := 0
	for i := 0; i < len(sub); i++ {
		for ; j < len(s); j++ {
			if sub[i].Start > s[j].Stop {
				continue
			} else {
				break
			}
		}
		if j == len(s) {
			return false
		}

		if sub[i].Start < s[j].Start || sub[i].Stop > s[j].Stop {
			return false
		}
	}

	return true
}

func (s IntervalSlice) Equal(o IntervalSlice) bool {
	if len(s) != len(o) {
		return false
	}

	for i := 0; i < len(s); i++ {
		if s[i].Start != o[i].Start || s[i].Stop != o[i].Stop {
			return false
		}
	}

	return true
}

// Refer http://dev.mysql.com/doc/refman/5.6/en/replication-gtids-concepts.html
type UUIDSet struct {
	SID uuid.UUID

	Intervals IntervalSlice
}

func ParseUUIDSet(str string) (*UUIDSet, error) {
	str = strings.TrimSpace(str)
	sep := strings.Split(str, ":")
	if len(sep) < 2 {
		return nil, fmt.Errorf("invalid GTID format, must UUID:interval[:interval]")
	}

	var err error
	s := new(UUIDSet)
	if s.SID, err = uuid.FromString(sep[0]); err != nil {
		return nil, err
	}

	// Handle interval
	for i := 1; i < len(sep); i++ {
		if in, err := parseInterval(sep[i]); err != nil {
			return nil, err
		} else {
			s.Intervals = append(s.Intervals, in)
		}
	}

	s.Intervals = s.Intervals.Normalize()

	return s, nil
}

func NewUUIDSet(sid uuid.UUID, in ...Interval) *UUIDSet {
	s := new(UUIDSet)
	s.SID = sid

	s.Intervals = in
	s.Intervals = s.Intervals.Normalize()

	return s
}

func (s *UUIDSet) Subset(sub *UUIDSet) bool {
	if !bytes.Equal(s.SID.Bytes(), sub.SID.Bytes()) {
		return false
	}

	return s.Intervals.Subset(sub.Intervals)
}

func (s *UUIDSet) Bytes() []byte {
	var buf bytes.Buffer

	buf.WriteString(s.SID.String())

	for _, i := range s.Intervals {
		buf.WriteString(":")
		buf.WriteString(i.String())
	}

	return buf.Bytes()
}

func (s *UUIDSet) AddInterval(in IntervalSlice) {
	s.Intervals = append(s.Intervals, in...)
	s.Intervals = s.Intervals.Normalize()
}

func (s *UUIDSet) String() string {
	return hack.String(s.Bytes())
}

func (s *UUIDSet) encode(w io.Writer) {
	w.Write(s.SID.Bytes())
	n := int64(len(s.Intervals))

	binary.Write(w, binary.LittleEndian, n)

	for _, i := range s.Intervals {
		binary.Write(w, binary.LittleEndian, i.Start)
		binary.Write(w, binary.LittleEndian, i.Stop)
	}
}

func (s *UUIDSet) Encode() []byte {
	var buf bytes.Buffer

	s.encode(&buf)

	return buf.Bytes()
}

func (s *UUIDSet) decode(data []byte) (int, error) {
	if len(data) < 24 {
		return 0, fmt.Errorf("invalid uuid set buffer, less 24")
	}

	pos := 0
	var err error
	if s.SID, err = uuid.FromBytes(data[0:16]); err != nil {
		return 0, err
	}
	pos += 16

	n := int64(binary.LittleEndian.Uint64(data[pos : pos+8]))
	pos += 8
	if len(data) < int(16*n)+pos {
		return 0, fmt.Errorf("invalid uuid set buffer, must %d, but %d", pos+int(16*n), len(data))
	}

	s.Intervals = make([]Interval, 0, n)

	var in Interval
	for i := int64(0); i < n; i++ {
		in.Start = int64(binary.LittleEndian.Uint64(data[pos : pos+8]))
		pos += 8
		in.Stop = int64(binary.LittleEndian.Uint64(data[pos : pos+8]))
		pos += 8
		s.Intervals = append(s.Intervals, in)
	}

	return pos, nil
}

func (s *UUIDSet) Decode(data []byte) error {
	n, err := s.decode(data)
	if n != len(data) {
		return fmt.Errorf("invalid uuid set buffer, must %d, but %d", n, len(data))
	}
	return err
}

type GTIDSet struct {
	Sets map[string]*UUIDSet
}

func ParseGTIDSet(str string) (*GTIDSet, error) {
	s := new(GTIDSet)

	sp := strings.Split(str, ",")

	s.Sets = make(map[string]*UUIDSet, len(sp))

	//todo, handle redundant same uuid
	for i := 0; i < len(sp); i++ {
		if set, err := ParseUUIDSet(sp[i]); err != nil {
			return nil, err
		} else {
			s.AddSet(set)
		}

	}
	return s, nil
}

func (s *GTIDSet) AddSet(set *UUIDSet) {
	sid := set.SID.String()
	o, ok := s.Sets[sid]
	if ok {
		o.AddInterval(set.Intervals)
	} else {
		s.Sets[sid] = set
	}
}

// Return true if sub in s
func (s *GTIDSet) Subset(sub *GTIDSet) bool {
	for key, set := range sub.Sets {
		o, ok := s.Sets[key]
		if !ok {
			return false
		}

		if !o.Subset(set) {
			return false
		}
	}

	return true
}

func (s *GTIDSet) String() string {
	var buf bytes.Buffer
	sep := ""
	for _, set := range s.Sets {
		buf.WriteString(sep)
		buf.WriteString(set.String())
		sep = ","
	}

	return hack.String(buf.Bytes())
}

func (s *GTIDSet) Encode() []byte {
	var buf bytes.Buffer

	binary.Write(&buf, binary.LittleEndian, uint64(len(s.Sets)))

	for i, _ := range s.Sets {
		s.Sets[i].encode(&buf)
	}

	return buf.Bytes()
}

func (s *GTIDSet) Decode(data []byte) error {
	if len(data) < 8 {
		return fmt.Errorf("invalid gtid set buffer, less 4")
	}

	n := int(binary.LittleEndian.Uint64(data))
	s.Sets = make(map[string]*UUIDSet, n)

	pos := 8

	for i := 0; i < n; i++ {
		set := new(UUIDSet)
		if n, err := set.decode(data[pos:]); err != nil {
			return err
		} else {
			pos += n

			s.AddSet(set)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// creates compressed archives.
//
// Go's `archive` package, supports `tar` and `zip`
// Go's `compress` package supports: bzip2, flate, gzip, lzw, zlib
//
// Carchivum supports zip and tar. For tar, archiver also supports
// the following compression:
//
// When using archiver, compression is not optional.
//
package carchivum

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

const (
	FmtUnsupported Format = iota
	FmtGzip
	FmtTar
	FmtTar1
	FmtTar2
	FmtZip
	FmtZipEmpty
	FmtZipSpanned
	FmtBzip2
	FmtLZH
	FmtLZW
	FmtRAR
	FmtRAROld
)

// Header information for common archive/compression formats.
// Zip includes: zip, jar, odt, ods, odp, docx, xlsx, pptx, apx, odf, ooxml
var (
	headerGzip []byte = []byte{0x1f, 0x8b}
	headerTar1 []byte = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x30, 0x30} // offset: 257
	headerTar2 []byte = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x20, 0x00} // offset: 257
	headerZip []byte = []byte{0x50, 0x4b, 0x03, 0x04}
	headerZipEmpty []byte = []byte{0x50, 0x4b, 0x05, 0x06}
	headerZipSpanned []byte = []byte{0x50, 0x4b, 0x07, 0x08}
	headerBzip2 []byte = []byte{0x42, 0x5a, 0x68}
	headerLZH []byte = []byte{0x1F, 0xa0}
	headerLZW []byte = []byte{0x1F, 0x9d}
	headerRAR []byte = []byte{0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00}
	headerRAROld []byte = []byte{0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00}
)

func GetFileFormat(r io.ReaderAt) (Format, error) {
	h := make([]byte, 8, 8)

	r.ReadAt(h, 0)

	if bytes.Equal(headerGzip, h[0:2]) {
		return FmtGzip, nil
	}

	if bytes.Equal(headerZip, h[0:4]) {
		return FmtZip, nil
	}

	r.ReadAt(h, 257)
	if bytes.Equal(headerTar1, h) || bytes.Equal(headerTar2, h) {
		return FmtTar, nil
	}

	// unsupported
	if bytes.Equal(headerRAROld, h) {
		return FmtUnsupported, FmtRAROld.NotSupportedError()
	}

	if bytes.Equal(headerRAR, h) {
		return FmtUnsupported, FmtRAR.NotSupportedError()
	}

	r.ReadAt(h, 0)
	if bytes.Equal(headerZipEmpty, h[0:4]) {
		return FmtUnsupported, FmtZipEmpty.NotSupportedError()
	}

	if bytes.Equal(headerZipSpanned, h[0:4]) {
		return FmtUnsupported, FmtZipSpanned.NotSupportedError()
	}

	if bytes.Equal(headerBzip2, h[0:3]) {
		return FmtUnsupported, FmtBzip2.NotSupportedError()
	}

	if bytes.Equal(headerLZW, h[0:2]) {
		return FmtUnsupported, FmtLZW.NotSupportedError()
	}

	if bytes.Equal(headerLZH, h[0:2]) {
		return FmtUnsupported, FmtLZH.NotSupportedError()
	}

	return FmtUnsupported, FmtUnsupported.NotSupportedError()
}

type (
	Format int
)

func (f Format) ToString() string {
	switch f {
	case FmtGzip:
		return "gzip"
	case FmtTar1, FmtTar2:
		return "tar"
	case FmtZip:
		return "zip"
	case FmtZipEmpty:
		return "empty zip archive"
	case FmtZipSpanned:
		return "spanned zip archive"
	case FmtBzip2:
		return "bzip2"
	case FmtLZH:
		return "LZH"
	case FmtLZW:
		return "LZW"
	case FmtRAR:
		return "RAR post 5.0"
	case FmtRAROld:
		return  "RAR pre 1.5"
	}
	return "unsupported"
}

func (f Format) NotSupportedError() error {
	switch f {
	case FmtZipEmpty:
		return fmt.Errorf("empty zip archive is not supported")
	case FmtZipSpanned:
		return fmt.Errorf("spanned zip archive is not supported")
	case FmtBzip2 :
		return fmt.Errorf("bzip2 is not supported")
	case FmtLZH:
		return fmt.Errorf("LZH is not supported")
	case FmtLZW:
		return fmt.Errorf("LZW is not supported")
	case FmtRAR:
		return fmt.Errorf("RAR post 5.0 is not supported")
	case FmtRAROld:
		return fmt.Errorf("RAR pre 1.5 is not supported")
	}
	return fmt.Errorf("unsupported format error, more specific information unavailable")
}

var defaultFormat Format = FmtGzip

// Options
var AppendDate bool
var TimeFormat string = time.RFC3339
var UseNano bool
var Separator string = "-"
var MakeUnique bool = false
// default max random number for random number generation.
var MaxRand = 10000

// we assume this count isn't going to change during runtime
var cpu int = runtime.NumCPU()

// Arbitrarily set the multiplier to some default value.
var CPUMultiplier int = 4

// Car is a Compressed Archive. The struct holds information about Cars and
// their processing.
type Car struct {
	// Name of the archive, this includes path information, if any.
	Name string
	UseLongExt bool
	UseFullpath bool

	// Create operation modifiers
	Owner int
	Group int
	Mode os.FileMode

	// Extract operation modifiers

	// Local file selection
	DeleteFiles bool

	Exclude string
	ExcludeExt []string
	ExcludeAnchored string

	Include string
	IncludeExt []string
	IncludeAnchored string

	// Processing queue
	FileCh	chan *os.File

	// Other Counters
	counterLock sync.Mutex
	files int32
	bytes int64
	compressedBytes int64

	// timer
	t0 time.Time
	𝛥t float64
}

func (c *Car) setDelta() {
	c.𝛥t = float64(time.Since(c.t0)) / 1e9
}

func (c *Car) Delta() float64 {
	return c.𝛥t
}

func (c *Car) Message() string {
	return fmt.Sprintf("%q created in %4f seconds\n%d files totalling %d bytes were processed", c.Name, c.𝛥t, c.files, c.bytes)
}

// addFile  reads a file and pipes it to the zipper goroutine.
func (c *Car) AddFile(root, p string, fi os.FileInfo, err error) error {
	logger.Debugf("root: %s, p: %s, fi.Name: %s", root, p, fi.Name())
	// Don't add symlinks, otherwise would have to code some cycle
	// detection amongst other stuff.
	if fi.Mode() & os.ModeSymlink == os.ModeSymlink {
		logger.Debugf("don't follow symlinks: %q", p)
		return nil
	}

	if strings.HasSuffix(root, p) {
		logger.Debugf("%s | %s, don't add if source is the source directory", root, p)
		return nil
	}

	b := c.includeFile(p)
	if !b {
		logger.Debugf("don't include %q", p)
		return nil
	}

	b := c.excludeFile(p)
	if b {
		logger.Debugf("exclude %q", p)
		return nil
	}

	var relPath string
	relPath, err = filepath.Rel(root, p)
	if err != nil {
		logger.Error(err)
		return err
	}

	if relPath == "," {
		logger.Debug("Don't add relative root")
		return nil
	}

	if !c.UseFullpath {
		p = filepath.Join(filepath.Base(root), relPath)
	}

	f, err := os.Open(p)
	if err != nil {
		logger.Error(err)
		return err
	}

	c.counterLock.Lock()
	c.files++
	c.bytes += fi.Size()
	c.counterLock.Unlock()

	c.FileCh <- f

	return nil
}

func (c *Car) includeFile(k string) bool {
	logger.Infof("%s c.IncludeAnchored %s", k, c.IncludeAnchored)
	if c.IncludeAnchored != "" {
		logger.Info(filepath.Base(k))
		if strings.HasPrefix(filepath.Base(k), c.IncludeAnchored) {
			logger.Info("has prefix")
			return true
		}
	}

	return false
}

func (c *Car) excludeFile(k string) bool {
	logger.Infof("%s c.ExcludeAnchored %s", k, c.ExcludeAnchored)
	if c.ExcludeAnchored != "" {
		logger.Info(filepath.Base(k))
		if strings.HasPrefix(filepath.Base(k), c.ExcludeAnchored) {
			logger.Info("has prefix")
			return true
		}
	}

	return false
}

func ParseFormat(s string) (Format, error) {
	switch s {
	case "gzip", "tar.gz", "tgz":
		return FmtGzip, nil
	case "tar":
		return FmtTar, nil
	case "zip":
		return FmtZip, nil
	}

	return FmtUnsupported, FmtUnsupported.NotSupportedError()
}

func formattedNow() string {
	return time.Now().Local().Format(TimeFormat)
}

func getFileParts(s string) (dir, file, ext string, err error) {
	// see if there is path involved, if there is, get the last part of it
	dir, filename := filepath.Split(s)
	parts := strings.Split(filename, ".")
	l := len(parts)
	switch l {
	case 2:
		file := parts[0]
		ext := parts[1]
		return dir, file, ext, nil
	case 1:
		file := parts[0]
		return dir, file, ext, nil
	case 0:
		err := fmt.Errorf("no destination filename found in %s", s)
		logger.Error(err)
		return dir, file, ext, err
	default:
		// join all but the last parts together with a "."
		file := strings.Join(parts[0:l-1], ".")
		ext := parts[l-1]
		return dir, file, ext, nil
	}

	err = fmt.Errorf("unable to determine destination filename and extension")
	logger.Error(err)
	return dir, file, ext, err
}
<commit_msg>add support for exclude-ext<commit_after>// creates compressed archives.
//
// Go's `archive` package, supports `tar` and `zip`
// Go's `compress` package supports: bzip2, flate, gzip, lzw, zlib
//
// Carchivum supports zip and tar. For tar, archiver also supports
// the following compression:
//
// When using archiver, compression is not optional.
//
package carchivum

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

const (
	FmtUnsupported Format = iota
	FmtGzip
	FmtTar
	FmtTar1
	FmtTar2
	FmtZip
	FmtZipEmpty
	FmtZipSpanned
	FmtBzip2
	FmtLZH
	FmtLZW
	FmtRAR
	FmtRAROld
)

// Header information for common archive/compression formats.
// Zip includes: zip, jar, odt, ods, odp, docx, xlsx, pptx, apx, odf, ooxml
var (
	headerGzip []byte = []byte{0x1f, 0x8b}
	headerTar1 []byte = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x30, 0x30} // offset: 257
	headerTar2 []byte = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x20, 0x00} // offset: 257
	headerZip []byte = []byte{0x50, 0x4b, 0x03, 0x04}
	headerZipEmpty []byte = []byte{0x50, 0x4b, 0x05, 0x06}
	headerZipSpanned []byte = []byte{0x50, 0x4b, 0x07, 0x08}
	headerBzip2 []byte = []byte{0x42, 0x5a, 0x68}
	headerLZH []byte = []byte{0x1F, 0xa0}
	headerLZW []byte = []byte{0x1F, 0x9d}
	headerRAR []byte = []byte{0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00}
	headerRAROld []byte = []byte{0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00}
)

func GetFileFormat(r io.ReaderAt) (Format, error) {
	h := make([]byte, 8, 8)

	r.ReadAt(h, 0)

	if bytes.Equal(headerGzip, h[0:2]) {
		return FmtGzip, nil
	}

	if bytes.Equal(headerZip, h[0:4]) {
		return FmtZip, nil
	}

	r.ReadAt(h, 257)
	if bytes.Equal(headerTar1, h) || bytes.Equal(headerTar2, h) {
		return FmtTar, nil
	}

	// unsupported
	if bytes.Equal(headerRAROld, h) {
		return FmtUnsupported, FmtRAROld.NotSupportedError()
	}

	if bytes.Equal(headerRAR, h) {
		return FmtUnsupported, FmtRAR.NotSupportedError()
	}

	r.ReadAt(h, 0)
	if bytes.Equal(headerZipEmpty, h[0:4]) {
		return FmtUnsupported, FmtZipEmpty.NotSupportedError()
	}

	if bytes.Equal(headerZipSpanned, h[0:4]) {
		return FmtUnsupported, FmtZipSpanned.NotSupportedError()
	}

	if bytes.Equal(headerBzip2, h[0:3]) {
		return FmtUnsupported, FmtBzip2.NotSupportedError()
	}

	if bytes.Equal(headerLZW, h[0:2]) {
		return FmtUnsupported, FmtLZW.NotSupportedError()
	}

	if bytes.Equal(headerLZH, h[0:2]) {
		return FmtUnsupported, FmtLZH.NotSupportedError()
	}

	return FmtUnsupported, FmtUnsupported.NotSupportedError()
}

type (
	Format int
)

func (f Format) ToString() string {
	switch f {
	case FmtGzip:
		return "gzip"
	case FmtTar1, FmtTar2:
		return "tar"
	case FmtZip:
		return "zip"
	case FmtZipEmpty:
		return "empty zip archive"
	case FmtZipSpanned:
		return "spanned zip archive"
	case FmtBzip2:
		return "bzip2"
	case FmtLZH:
		return "LZH"
	case FmtLZW:
		return "LZW"
	case FmtRAR:
		return "RAR post 5.0"
	case FmtRAROld:
		return  "RAR pre 1.5"
	}
	return "unsupported"
}

func (f Format) NotSupportedError() error {
	switch f {
	case FmtZipEmpty:
		return fmt.Errorf("empty zip archive is not supported")
	case FmtZipSpanned:
		return fmt.Errorf("spanned zip archive is not supported")
	case FmtBzip2 :
		return fmt.Errorf("bzip2 is not supported")
	case FmtLZH:
		return fmt.Errorf("LZH is not supported")
	case FmtLZW:
		return fmt.Errorf("LZW is not supported")
	case FmtRAR:
		return fmt.Errorf("RAR post 5.0 is not supported")
	case FmtRAROld:
		return fmt.Errorf("RAR pre 1.5 is not supported")
	}
	return fmt.Errorf("unsupported format error, more specific information unavailable")
}

var defaultFormat Format = FmtGzip

// Options
var AppendDate bool
var TimeFormat string = time.RFC3339
var UseNano bool
var Separator string = "-"
var MakeUnique bool = false
// default max random number for random number generation.
var MaxRand = 10000

// we assume this count isn't going to change during runtime
var cpu int = runtime.NumCPU()

// Arbitrarily set the multiplier to some default value.
var CPUMultiplier int = 4

// Car is a Compressed Archive. The struct holds information about Cars and
// their processing.
type Car struct {
	// Name of the archive, this includes path information, if any.
	Name string
	UseLongExt bool
	UseFullpath bool

	// Create operation modifiers
	Owner int
	Group int
	Mode os.FileMode

	// Extract operation modifiers

	// Local file selection
	DeleteFiles bool

	Exclude string
	ExcludeExt []string
	ExcludeExtCount int
	ExcludeAnchored string

	Include string
	IncludeExt []string
	IncludeExtCount int
	IncludeAnchored string

	// Processing queue
	FileCh	chan *os.File

	// Other Counters
	counterLock sync.Mutex
	files int32
	bytes int64
	compressedBytes int64

	// timer
	t0 time.Time
	𝛥t float64
}

func (c *Car) setDelta() {
	c.𝛥t = float64(time.Since(c.t0)) / 1e9
}

func (c *Car) Delta() float64 {
	return c.𝛥t
}

func (c *Car) Message() string {
	return fmt.Sprintf("%q created in %4f seconds\n%d files totalling %d bytes were processed", c.Name, c.𝛥t, c.files, c.bytes)
}

// addFile  reads a file and pipes it to the zipper goroutine.
func (c *Car) AddFile(root, p string, fi os.FileInfo, err error) error {
	logger.Debugf("root: %s, p: %s, fi.Name: %s", root, p, fi.Name())
	// Don't add symlinks, otherwise would have to code some cycle
	// detection amongst other stuff.
	if fi.Mode() & os.ModeSymlink == os.ModeSymlink {
		logger.Debugf("don't follow symlinks: %q", p)
		return nil
	}

	if strings.HasSuffix(root, p) {
		logger.Debugf("%s | %s, don't add if source is the source directory", root, p)
		return nil
	}

	b := c.includeFile(p)
	if !b {
		logger.Debugf("don't include %q", p)
		return nil
	}

	b = c.excludeFile(p)
	if b {
		logger.Debugf("exclude %q", p)
		return nil
	}

	var relPath string
	relPath, err = filepath.Rel(root, p)
	if err != nil {
		logger.Error(err)
		return err
	}

	if relPath == "," {
		logger.Debug("Don't add relative root")
		return nil
	}

	if !c.UseFullpath {
		p = filepath.Join(filepath.Base(root), relPath)
	}

	f, err := os.Open(p)
	if err != nil {
		logger.Error(err)
		return err
	}

	c.counterLock.Lock()
	c.files++
	c.bytes += fi.Size()
	c.counterLock.Unlock()

	c.FileCh <- f

	return nil
}

func (c *Car) includeFile(k string) bool {
	logger.Infof("%s c.IncludeAnchored %s", k, c.IncludeAnchored)
	if c.IncludeAnchored != "" {
		logger.Info(filepath.Base(k))
		if strings.HasPrefix(filepath.Base(k), c.IncludeAnchored) {
			logger.Info("has prefix")
			return true
		}
	}

	return false
}

func (c *Car) excludeFile(k string) bool {
	logger.Infof("%s c.ExcludeAnchored %s", k, c.ExcludeAnchored)
	if c.ExcludeAnchored != "" {
		logger.Info(filepath.Base(k))
		if strings.HasPrefix(filepath.Base(k), c.ExcludeAnchored) {
			logger.Info("has prefix")
			return true
		}
	}

	if c.ExcludeExtCount == 0 {
		return false
	}

	for _, ext := range c.ExcludeExt {
		if strings.HasSuffix(filepath.Base(k), "." + ext) {
			return true
		}
	}

	return false
}

func ParseFormat(s string) (Format, error) {
	switch s {
	case "gzip", "tar.gz", "tgz":
		return FmtGzip, nil
	case "tar":
		return FmtTar, nil
	case "zip":
		return FmtZip, nil
	}

	return FmtUnsupported, FmtUnsupported.NotSupportedError()
}

func formattedNow() string {
	return time.Now().Local().Format(TimeFormat)
}

func getFileParts(s string) (dir, file, ext string, err error) {
	// see if there is path involved, if there is, get the last part of it
	dir, filename := filepath.Split(s)
	parts := strings.Split(filename, ".")
	l := len(parts)
	switch l {
	case 2:
		file := parts[0]
		ext := parts[1]
		return dir, file, ext, nil
	case 1:
		file := parts[0]
		return dir, file, ext, nil
	case 0:
		err := fmt.Errorf("no destination filename found in %s", s)
		logger.Error(err)
		return dir, file, ext, err
	default:
		// join all but the last parts together with a "."
		file := strings.Join(parts[0:l-1], ".")
		ext := parts[l-1]
		return dir, file, ext, nil
	}

	err = fmt.Errorf("unable to determine destination filename and extension")
	logger.Error(err)
	return dir, file, ext, err
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package instance

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/ernestio/ernestaws"
	"github.com/ernestio/ernestaws/credentials"
)

var (
	// ErrDatacenterIDInvalid ...
	ErrDatacenterIDInvalid = errors.New("Datacenter VPC ID invalid")
	// ErrDatacenterRegionInvalid ...
	ErrDatacenterRegionInvalid = errors.New("Datacenter Region invalid")
	// ErrDatacenterCredentialsInvalid ...
	ErrDatacenterCredentialsInvalid = errors.New("Datacenter credentials invalid")
	// ErrInstanceAWSIDInvalid ...
	ErrInstanceAWSIDInvalid = errors.New("Instance aws id invalid")
	// ErrNetworkInvalid ...
	ErrNetworkInvalid = errors.New("Network invalid")
	// ErrInstanceNameInvalid ...
	ErrInstanceNameInvalid = errors.New("Instance name invalid")
	// ErrInstanceImageInvalid ...
	ErrInstanceImageInvalid = errors.New("Instance image invalid")
	// ErrInstanceTypeInvalid ...
	ErrInstanceTypeInvalid = errors.New("Instance type invalid")
)

// Volume stores ebs volume data
type Volume struct {
	Name        string `json:"name"`
	Device      string `json:"device"`
	VolumeAWSID string `json:"volume_aws_id"`
}

// Event stores the template data
type Event struct {
	UUID                  string   `json:"_uuid"`
	BatchID               string   `json:"_batch_id"`
	ProviderType          string   `json:"_type"`
	VPCID                 string   `json:"vpc_id"`
	DatacenterRegion      string   `json:"datacenter_region"`
	DatacenterAccessKey   string   `json:"datacenter_secret"`
	DatacenterAccessToken string   `json:"datacenter_token"`
	NetworkAWSID          string   `json:"network_aws_id"`
	NetworkIsPublic       bool     `json:"network_is_public"`
	SecurityGroupAWSIDs   []string `json:"security_group_aws_ids"`
	InstanceAWSID         string   `json:"instance_aws_id,omitempty"`
	Name                  string   `json:"name"`
	Image                 string   `json:"image"`
	InstanceType          string   `json:"instance_type"`
	IP                    string   `json:"ip"`
	KeyPair               string   `json:"key_pair"`
	UserData              string   `json:"user_data"`
	PublicIP              string   `json:"public_ip"`
	ElasticIP             string   `json:"elastic_ip"`
	AssignElasticIP       bool     `json:"assign_elastic_ip"`
	Volumes               []Volume `json:"volumes"`
	ErrorMessage          string   `json:"error,omitempty"`
	Subject               string   `json:"-"`
	Body                  []byte   `json:"-"`
	CryptoKey             []byte   `json:"-"`
}

// New : Constructor
func New(subject string, body, cryptoKey []byte) ernestaws.Event {
	n := Event{Subject: subject, Body: body, CryptoKey: cryptoKey}

	return &n
}

// GetBody : Gets the body for this event
func (ev *Event) GetBody() []byte {
	var err error
	if ev.Body, err = json.Marshal(ev); err != nil {
		log.Println(err.Error())
	}
	return ev.Body
}

// GetSubject : Gets the subject for this event
func (ev *Event) GetSubject() string {
	return ev.Subject
}

// Process : starts processing the current message
func (ev *Event) Process() (err error) {
	if err := json.Unmarshal(ev.Body, &ev); err != nil {
		ev.Error(err)
		return err
	}

	if err := ev.Validate(); err != nil {
		ev.Error(err)
		return err
	}

	return nil
}

// Error : Will respond the current event with an error
func (ev *Event) Error(err error) {
	log.Printf("Error: %s", err.Error())
	ev.ErrorMessage = err.Error()

	ev.Body, err = json.Marshal(ev)
}

// Validate checks if all criteria are met
func (ev *Event) Validate() error {
	if ev.VPCID == "" {
		return ErrDatacenterIDInvalid
	}

	if ev.DatacenterRegion == "" {
		return ErrDatacenterRegionInvalid
	}

	if ev.DatacenterAccessKey == "" || ev.DatacenterAccessToken == "" {
		return ErrDatacenterCredentialsInvalid
	}

	if ev.Subject != "instance.create.aws" {
		if ev.InstanceAWSID == "" {
			return ErrInstanceAWSIDInvalid
		}
	}

	if ev.Subject != "instance.delete.aws" {
		if ev.NetworkAWSID == "" {
			return ErrNetworkInvalid
		}
	}

	if ev.Name == "" {
		return ErrInstanceNameInvalid
	}

	if ev.Image == "" {
		return ErrInstanceImageInvalid
	}

	if ev.InstanceType == "" {
		return ErrInstanceTypeInvalid
	}

	return nil
}

// Create : Creates a instance object on aws
func (ev *Event) Create() error {
	svc := ev.getEC2Client()

	req := ec2.RunInstancesInput{
		SubnetId:         aws.String(ev.NetworkAWSID),
		ImageId:          aws.String(ev.Image),
		InstanceType:     aws.String(ev.InstanceType),
		PrivateIpAddress: aws.String(ev.IP),
		KeyName:          aws.String(ev.KeyPair),
		MaxCount:         aws.Int64(1),
		MinCount:         aws.Int64(1),
	}

	for _, sg := range ev.SecurityGroupAWSIDs {
		req.SecurityGroupIds = append(req.SecurityGroupIds, aws.String(sg))
	}

	if ev.UserData != "" {
		data := ev.encodeUserData(ev.UserData)
		req.UserData = aws.String(data)
	}

	resp, err := svc.RunInstances(&req)
	if err != nil {
		return err
	}

	builtInstance := ec2.DescribeInstancesInput{
		InstanceIds: []*string{resp.Instances[0].InstanceId},
	}

	err = svc.WaitUntilInstanceRunning(&builtInstance)
	if err != nil {
		return err
	}

	if ev.AssignElasticIP {
		ev.ElasticIP, err = ev.assignElasticIP(svc, *resp.Instances[0].InstanceId)
		if err != nil {
			return err
		}
	}

	ev.InstanceAWSID = *resp.Instances[0].InstanceId

	instance, err := ev.getInstanceByID(resp.Instances[0].InstanceId)
	if err != nil {
		return err
	}

	if instance.PublicIpAddress != nil {
		ev.PublicIP = *instance.PublicIpAddress
	}

	return ev.attachVolumes()
}

// Update : Updates a instance object on aws
func (ev *Event) Update() error {
	svc := ev.getEC2Client()

	builtInstance := ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	okInstance := ec2.DescribeInstanceStatusInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	err := svc.WaitUntilInstanceStatusOk(&okInstance)
	if err != nil {
		return err
	}

	stopreq := ec2.StopInstancesInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	// power off the instance
	_, err = svc.StopInstances(&stopreq)
	if err != nil {
		return err
	}

	err = svc.WaitUntilInstanceStopped(&builtInstance)
	if err != nil {
		return err
	}

	// resize the instance
	req := ec2.ModifyInstanceAttributeInput{
		InstanceId: aws.String(ev.InstanceAWSID),
		InstanceType: &ec2.AttributeValue{
			Value: aws.String(ev.InstanceType),
		},
	}

	_, err = svc.ModifyInstanceAttribute(&req)
	if err != nil {
		return err
	}

	// update instance security groups
	req = ec2.ModifyInstanceAttributeInput{
		InstanceId: aws.String(ev.InstanceAWSID),
		Groups:     []*string{},
	}

	for _, sg := range ev.SecurityGroupAWSIDs {
		req.Groups = append(req.Groups, aws.String(sg))
	}

	_, err = svc.ModifyInstanceAttribute(&req)
	if err != nil {
		return err
	}

	// power the instance back on
	startreq := ec2.StartInstancesInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	_, err = svc.StartInstances(&startreq)
	if err != nil {
		return err
	}

	err = svc.WaitUntilInstanceRunning(&builtInstance)
	if err != nil {
		return err
	}

	instance, err := ev.getInstanceByID(&ev.InstanceAWSID)
	if err != nil {
		return err
	}

	if instance.PublicIpAddress != nil {
		ev.PublicIP = *instance.PublicIpAddress
	}

	return nil
}

// Delete : Deletes a instance object on aws
func (ev *Event) Delete() error {
	svc := ev.getEC2Client()

	req := ec2.TerminateInstancesInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	_, err := svc.TerminateInstances(&req)
	if err != nil {
		return err
	}

	termreq := ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	err = svc.WaitUntilInstanceTerminated(&termreq)
	if err != nil {
		return err
	}

	return nil
}

// Get : Gets a instance object on aws
func (ev *Event) Get() error {
	return errors.New(ev.Subject + " not supported")
}

func (ev *Event) getEC2Client() *ec2.EC2 {
	creds, _ := credentials.NewStaticCredentials(ev.DatacenterAccessKey, ev.DatacenterAccessToken, ev.CryptoKey)
	return ec2.New(session.New(), &aws.Config{
		Region:      aws.String(ev.DatacenterRegion),
		Credentials: creds,
	})
}

func (ev *Event) assignElasticIP(svc *ec2.EC2, instanceID string) (string, error) {
	// Create Elastic IP
	resp, err := svc.AllocateAddress(nil)
	if err != nil {
		return "", err
	}

	req := ec2.AssociateAddressInput{
		InstanceId:   aws.String(instanceID),
		AllocationId: resp.AllocationId,
	}
	_, err = svc.AssociateAddress(&req)
	if err != nil {
		return "", err
	}

	return *resp.PublicIp, nil
}

func (ev *Event) getInstanceByID(id *string) (*ec2.Instance, error) {
	svc := ev.getEC2Client()

	req := ec2.DescribeInstancesInput{
		InstanceIds: []*string{id},
	}

	resp, err := svc.DescribeInstances(&req)
	if err != nil {
		return nil, err
	}

	if len(resp.Reservations) != 1 {
		return nil, errors.New("Could not find any instance reservations")
	}

	if len(resp.Reservations[0].Instances) != 1 {
		return nil, errors.New("Could not find an instance with that ID")
	}

	return resp.Reservations[0].Instances[0], nil
}

func (ev *Event) encodeUserData(data string) string {
	return base64.StdEncoding.EncodeToString([]byte(data))
}

func (ev *Event) attachVolumes() error {
	svc := ev.getEC2Client()

	instance, err := ev.getInstanceByID(&ev.InstanceAWSID)
	if err != nil {
		return err
	}

	for _, vol := range ev.Volumes {
		// check volume doesn't exist
		if hasVolumeAttached(instance.BlockDeviceMappings, vol) {
			continue
		}

		req := &ec2.AttachVolumeInput{
			Device:     aws.String(vol.Device),
			VolumeId:   aws.String(vol.VolumeAWSID),
			InstanceId: aws.String(ev.InstanceAWSID),
		}

		_, err = svc.AttachVolume(req)
		if err != nil {
			return err
		}
	}

	return nil
}

func hasVolumeAttached(bdms []*ec2.InstanceBlockDeviceMapping, vol Volume) bool {
	for _, bdm := range bdms {
		if *bdm.Ebs.VolumeId == vol.VolumeAWSID || *bdm.DeviceName == vol.Device {
			return true
		}
	}

	return false
}
<commit_msg>add instance volumes on update<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package instance

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/ernestio/ernestaws"
	"github.com/ernestio/ernestaws/credentials"
)

var (
	// ErrDatacenterIDInvalid ...
	ErrDatacenterIDInvalid = errors.New("Datacenter VPC ID invalid")
	// ErrDatacenterRegionInvalid ...
	ErrDatacenterRegionInvalid = errors.New("Datacenter Region invalid")
	// ErrDatacenterCredentialsInvalid ...
	ErrDatacenterCredentialsInvalid = errors.New("Datacenter credentials invalid")
	// ErrInstanceAWSIDInvalid ...
	ErrInstanceAWSIDInvalid = errors.New("Instance aws id invalid")
	// ErrNetworkInvalid ...
	ErrNetworkInvalid = errors.New("Network invalid")
	// ErrInstanceNameInvalid ...
	ErrInstanceNameInvalid = errors.New("Instance name invalid")
	// ErrInstanceImageInvalid ...
	ErrInstanceImageInvalid = errors.New("Instance image invalid")
	// ErrInstanceTypeInvalid ...
	ErrInstanceTypeInvalid = errors.New("Instance type invalid")
)

// Volume stores ebs volume data
type Volume struct {
	Name        string `json:"name"`
	Device      string `json:"device"`
	VolumeAWSID string `json:"volume_aws_id"`
}

// Event stores the template data
type Event struct {
	UUID                  string   `json:"_uuid"`
	BatchID               string   `json:"_batch_id"`
	ProviderType          string   `json:"_type"`
	VPCID                 string   `json:"vpc_id"`
	DatacenterRegion      string   `json:"datacenter_region"`
	DatacenterAccessKey   string   `json:"datacenter_secret"`
	DatacenterAccessToken string   `json:"datacenter_token"`
	NetworkAWSID          string   `json:"network_aws_id"`
	NetworkIsPublic       bool     `json:"network_is_public"`
	SecurityGroupAWSIDs   []string `json:"security_group_aws_ids"`
	InstanceAWSID         string   `json:"instance_aws_id,omitempty"`
	Name                  string   `json:"name"`
	Image                 string   `json:"image"`
	InstanceType          string   `json:"instance_type"`
	IP                    string   `json:"ip"`
	KeyPair               string   `json:"key_pair"`
	UserData              string   `json:"user_data"`
	PublicIP              string   `json:"public_ip"`
	ElasticIP             string   `json:"elastic_ip"`
	AssignElasticIP       bool     `json:"assign_elastic_ip"`
	Volumes               []Volume `json:"volumes"`
	ErrorMessage          string   `json:"error,omitempty"`
	Subject               string   `json:"-"`
	Body                  []byte   `json:"-"`
	CryptoKey             []byte   `json:"-"`
}

// New : Constructor
func New(subject string, body, cryptoKey []byte) ernestaws.Event {
	n := Event{Subject: subject, Body: body, CryptoKey: cryptoKey}

	return &n
}

// GetBody : Gets the body for this event
func (ev *Event) GetBody() []byte {
	var err error
	if ev.Body, err = json.Marshal(ev); err != nil {
		log.Println(err.Error())
	}
	return ev.Body
}

// GetSubject : Gets the subject for this event
func (ev *Event) GetSubject() string {
	return ev.Subject
}

// Process : starts processing the current message
func (ev *Event) Process() (err error) {
	if err := json.Unmarshal(ev.Body, &ev); err != nil {
		ev.Error(err)
		return err
	}

	if err := ev.Validate(); err != nil {
		ev.Error(err)
		return err
	}

	return nil
}

// Error : Will respond the current event with an error
func (ev *Event) Error(err error) {
	log.Printf("Error: %s", err.Error())
	ev.ErrorMessage = err.Error()

	ev.Body, err = json.Marshal(ev)
}

// Validate checks if all criteria are met
func (ev *Event) Validate() error {
	if ev.VPCID == "" {
		return ErrDatacenterIDInvalid
	}

	if ev.DatacenterRegion == "" {
		return ErrDatacenterRegionInvalid
	}

	if ev.DatacenterAccessKey == "" || ev.DatacenterAccessToken == "" {
		return ErrDatacenterCredentialsInvalid
	}

	if ev.Subject != "instance.create.aws" {
		if ev.InstanceAWSID == "" {
			return ErrInstanceAWSIDInvalid
		}
	}

	if ev.Subject != "instance.delete.aws" {
		if ev.NetworkAWSID == "" {
			return ErrNetworkInvalid
		}
	}

	if ev.Name == "" {
		return ErrInstanceNameInvalid
	}

	if ev.Image == "" {
		return ErrInstanceImageInvalid
	}

	if ev.InstanceType == "" {
		return ErrInstanceTypeInvalid
	}

	return nil
}

// Create : Creates a instance object on aws
func (ev *Event) Create() error {
	svc := ev.getEC2Client()

	req := ec2.RunInstancesInput{
		SubnetId:         aws.String(ev.NetworkAWSID),
		ImageId:          aws.String(ev.Image),
		InstanceType:     aws.String(ev.InstanceType),
		PrivateIpAddress: aws.String(ev.IP),
		KeyName:          aws.String(ev.KeyPair),
		MaxCount:         aws.Int64(1),
		MinCount:         aws.Int64(1),
	}

	for _, sg := range ev.SecurityGroupAWSIDs {
		req.SecurityGroupIds = append(req.SecurityGroupIds, aws.String(sg))
	}

	if ev.UserData != "" {
		data := ev.encodeUserData(ev.UserData)
		req.UserData = aws.String(data)
	}

	resp, err := svc.RunInstances(&req)
	if err != nil {
		return err
	}

	builtInstance := ec2.DescribeInstancesInput{
		InstanceIds: []*string{resp.Instances[0].InstanceId},
	}

	err = svc.WaitUntilInstanceRunning(&builtInstance)
	if err != nil {
		return err
	}

	if ev.AssignElasticIP {
		ev.ElasticIP, err = ev.assignElasticIP(svc, *resp.Instances[0].InstanceId)
		if err != nil {
			return err
		}
	}

	ev.InstanceAWSID = *resp.Instances[0].InstanceId

	instance, err := ev.getInstanceByID(resp.Instances[0].InstanceId)
	if err != nil {
		return err
	}

	if instance.PublicIpAddress != nil {
		ev.PublicIP = *instance.PublicIpAddress
	}

	return ev.attachVolumes()
}

// Update : Updates a instance object on aws
func (ev *Event) Update() error {
	svc := ev.getEC2Client()

	builtInstance := ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	okInstance := ec2.DescribeInstanceStatusInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	err := svc.WaitUntilInstanceStatusOk(&okInstance)
	if err != nil {
		return err
	}

	stopreq := ec2.StopInstancesInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	// power off the instance
	_, err = svc.StopInstances(&stopreq)
	if err != nil {
		return err
	}

	err = svc.WaitUntilInstanceStopped(&builtInstance)
	if err != nil {
		return err
	}

	// resize the instance
	req := ec2.ModifyInstanceAttributeInput{
		InstanceId: aws.String(ev.InstanceAWSID),
		InstanceType: &ec2.AttributeValue{
			Value: aws.String(ev.InstanceType),
		},
	}

	_, err = svc.ModifyInstanceAttribute(&req)
	if err != nil {
		return err
	}

	// update instance security groups
	req = ec2.ModifyInstanceAttributeInput{
		InstanceId: aws.String(ev.InstanceAWSID),
		Groups:     []*string{},
	}

	for _, sg := range ev.SecurityGroupAWSIDs {
		req.Groups = append(req.Groups, aws.String(sg))
	}

	_, err = svc.ModifyInstanceAttribute(&req)
	if err != nil {
		return err
	}

	// power the instance back on
	startreq := ec2.StartInstancesInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	_, err = svc.StartInstances(&startreq)
	if err != nil {
		return err
	}

	err = svc.WaitUntilInstanceRunning(&builtInstance)
	if err != nil {
		return err
	}

	instance, err := ev.getInstanceByID(&ev.InstanceAWSID)
	if err != nil {
		return err
	}

	if instance.PublicIpAddress != nil {
		ev.PublicIP = *instance.PublicIpAddress
	}

	return ev.attachVolumes()
}

// Delete : Deletes a instance object on aws
func (ev *Event) Delete() error {
	svc := ev.getEC2Client()

	req := ec2.TerminateInstancesInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	_, err := svc.TerminateInstances(&req)
	if err != nil {
		return err
	}

	termreq := ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(ev.InstanceAWSID)},
	}

	err = svc.WaitUntilInstanceTerminated(&termreq)
	if err != nil {
		return err
	}

	return nil
}

// Get : Gets a instance object on aws
func (ev *Event) Get() error {
	return errors.New(ev.Subject + " not supported")
}

func (ev *Event) getEC2Client() *ec2.EC2 {
	creds, _ := credentials.NewStaticCredentials(ev.DatacenterAccessKey, ev.DatacenterAccessToken, ev.CryptoKey)
	return ec2.New(session.New(), &aws.Config{
		Region:      aws.String(ev.DatacenterRegion),
		Credentials: creds,
	})
}

func (ev *Event) assignElasticIP(svc *ec2.EC2, instanceID string) (string, error) {
	// Create Elastic IP
	resp, err := svc.AllocateAddress(nil)
	if err != nil {
		return "", err
	}

	req := ec2.AssociateAddressInput{
		InstanceId:   aws.String(instanceID),
		AllocationId: resp.AllocationId,
	}
	_, err = svc.AssociateAddress(&req)
	if err != nil {
		return "", err
	}

	return *resp.PublicIp, nil
}

func (ev *Event) getInstanceByID(id *string) (*ec2.Instance, error) {
	svc := ev.getEC2Client()

	req := ec2.DescribeInstancesInput{
		InstanceIds: []*string{id},
	}

	resp, err := svc.DescribeInstances(&req)
	if err != nil {
		return nil, err
	}

	if len(resp.Reservations) != 1 {
		return nil, errors.New("Could not find any instance reservations")
	}

	if len(resp.Reservations[0].Instances) != 1 {
		return nil, errors.New("Could not find an instance with that ID")
	}

	return resp.Reservations[0].Instances[0], nil
}

func (ev *Event) encodeUserData(data string) string {
	return base64.StdEncoding.EncodeToString([]byte(data))
}

func (ev *Event) attachVolumes() error {
	svc := ev.getEC2Client()

	instance, err := ev.getInstanceByID(&ev.InstanceAWSID)
	if err != nil {
		return err
	}

	for _, vol := range ev.Volumes {
		// check volume doesn't exist
		if hasVolumeAttached(instance.BlockDeviceMappings, vol) {
			continue
		}

		req := &ec2.AttachVolumeInput{
			Device:     aws.String(vol.Device),
			VolumeId:   aws.String(vol.VolumeAWSID),
			InstanceId: aws.String(ev.InstanceAWSID),
		}

		_, err = svc.AttachVolume(req)
		if err != nil {
			return err
		}
	}

	return nil
}

func hasVolumeAttached(bdms []*ec2.InstanceBlockDeviceMapping, vol Volume) bool {
	for _, bdm := range bdms {
		if *bdm.Ebs.VolumeId == vol.VolumeAWSID || *bdm.DeviceName == vol.Device {
			return true
		}
	}

	return false
}
<|endoftext|>