<commit_before>package aralog

import (
    "io"
    "os"
    "runtime"
    "sync"
    "time"
)

// These flags define which text to prefix to each log entry generated by the Logger.
const (
// Bits or'ed together to control what's printed. There is no control over the
// order they appear (the order listed here) or the format they present (as
// described in the comments).  A colon appears after these items:
//	2009/01/23 01:23:23.123123 /a/b/c/d.go:23: message
    Ldate = 1 << iota     // the date: 2009/01/23
    Ltime                         // the time: 01:23:23
    Lmicroseconds                 // microsecond resolution: 01:23:23.123123.  assumes Ltime.
    Llongfile                     // full file name and line number: /a/b/c/d.go:23
    Lshortfile                    // final file name element and line number: d.go:23. overrides Llongfile
    LstdFlags = Ldate | Ltime // initial values for the standard logger
)

// A Logger represents an active logging object that generates lines of
// output to an io.Writer.  Each logging operation makes a single call to
// the Writer's Write method.  A Logger can be used simultaneously from
// multiple goroutines; it guarantees to serialize access to the Writer.
type Logger struct {
    mu      sync.Mutex // ensures atomic writes; protects the following fields
    prefix  string     // prefix to write at beginning of each line
    flag    int        // properties
    out     io.Writer  // destination for output
    buf     []byte     // for accumulating text to write
    size    uint       // current size of log file
    path    string     // file path if output to a file
    maxsize uint       // minimal maxsize should >= 1MB
}

var currentOutFile *os.File

// New creates a new Logger.   The out variable sets the
// destination to which log data will be written.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func New(out io.Writer, prefix string, flag int) *Logger {
    return &Logger{out: out, prefix: prefix, flag: flag}
}

// NewFileLogger create a new Logger which output to a file specified
func NewFileLogger(path string, flag int) (*Logger, error) {
    return NewRollFileLogger(path, 1024*1024*10, flag)
}

// NewRollFileLogger create a new Logger which output to a file specified path,
// and roll at specified size
func NewRollFileLogger(path string, maxsize uint, flag int) (*Logger, error) {
    out, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600)
    if err != nil {
        return nil, err
    }

    currentOutFile = out

    // minimal maxsize should >= 1MB
    if maxsize < 1024 * 1024 {
        maxsize = 1024 * 1024 * 10
    }

    return &Logger{out: out, prefix: "", flag: flag, path: path, maxsize: maxsize}, nil
}

//var std = New(os.Stderr, "", LstdFlags)

// Cheap integer to fixed-width decimal ASCII.  Give a negative width to avoid zero-padding.
// Knows the buffer has capacity.
func itoa(buf *[]byte, i int, wid int) {
    var u uint = uint(i)
    if u == 0 && wid <= 1 {
        *buf = append(*buf, '0')
        return
    }

    // Assemble decimal in reverse order.
    var b [32]byte
    bp := len(b)
    for ; u > 0 || wid > 0; u /= 10 {
        bp--
        wid--
        b[bp] = byte(u%10) + '0'
    }
    *buf = append(*buf, b[bp:]...)
}

func (l *Logger) formatHeader(buf *[]byte, t time.Time, file string, line int) {
    *buf = append(*buf, l.prefix...)
    if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 {
        if l.flag&Ldate != 0 {
            year, month, day := t.Date()
            itoa(buf, year, 4)
            *buf = append(*buf, '/')
            itoa(buf, int(month), 2)
            *buf = append(*buf, '/')
            itoa(buf, day, 2)
            *buf = append(*buf, ' ')
        }
        if l.flag&(Ltime|Lmicroseconds) != 0 {
            hour, min, sec := t.Clock()
            itoa(buf, hour, 2)
            *buf = append(*buf, ':')
            itoa(buf, min, 2)
            *buf = append(*buf, ':')
            itoa(buf, sec, 2)
            if l.flag&Lmicroseconds != 0 {
                *buf = append(*buf, '.')
                itoa(buf, t.Nanosecond()/1e3, 6)
            }
            *buf = append(*buf, ' ')
        }
    }
    if l.flag&(Lshortfile|Llongfile) != 0 {
        if l.flag&Lshortfile != 0 {
            short := file
            for i := len(file) - 1; i > 0; i-- {
                if file[i] == '/' {
                    short = file[i+1:]
                    break
                }
            }
            file = short
        }
        *buf = append(*buf, file...)
        *buf = append(*buf, ':')
        itoa(buf, line, -1)
        *buf = append(*buf, ": "...)
    }
}

// Output writes the output for a logging event.  The string s contains
// the text to print after the prefix specified by the flags of the
// Logger.  A newline is appended if the last character of s is not
// already a newline.  Calldepth is used to recover the PC and is
// provided for generality, although at the moment on all pre-defined
// paths it will be 2.
func (l *Logger) output(calldepth int, s string) error {
    now := time.Now() // get this early.
    var file string
    var line int
    l.mu.Lock()
    defer l.mu.Unlock()
    if l.flag&(Lshortfile|Llongfile) != 0 {
        // release lock while getting caller info - it's expensive.
        l.mu.Unlock()
        var ok bool
        _, file, line, ok = runtime.Caller(calldepth)
        if !ok {
            file = "???"
            line = 0
        }
        l.mu.Lock()
    }
    l.buf = l.buf[:0]
    l.formatHeader(&l.buf, now, file, line)
    l.buf = append(l.buf, s...)
    if len(s) > 0 && s[len(s)-1] != '\n' {
        l.buf = append(l.buf, '\n')
    }

    if len(l.path) > 0 {
        err := l.rollFile(now)
        if err != nil {
            return err
        }
    }
    _, err := l.out.Write(l.buf)
    return err
}

func (l *Logger) rollFile(now time.Time) error {
    l.size += uint(len(l.buf))

    if l.size < l.maxsize {
        return nil
    }

    // file rotation if size > maxsize
    // close file before rename it
    if currentOutFile != nil {
        // ignore if Close() failed
        err := currentOutFile.Close()
        if err != nil {
            l.buf = append(l.buf, ("[XXX] ARALOGGER ERROR: Close current output file failed, " + err.Error())...)
            l.buf = append(l.buf, '\n')
        }
    }

    newPath := l.path

    // rename l.path to nameYYYYMMDDhhmmss
    err := os.Rename(l.path,
        l.path + string(now.Year()) + string(now.Month()) + string(now.Day()) +
        string(now.Hour()) + string(now.Minute()) + string(now.Second()))
    if err == nil {
        // TODO zip it
    } else {
        l.buf = append(l.buf, ("[XXX] ARALOGGER ERROR: Rolling file failed, " + err.Error())...)
        l.buf = append(l.buf, '\n')

        // if rename failed, start a new log file with different name
        newPath = l.path + string(now.Unix())
    }

    newOut, err := os.OpenFile(newPath, os.O_APPEND|os.O_WRONLY, 0600)
    if err != nil {
        return err
    }

    currentOutFile = newOut
    l.out = newOut
    l.size = uint(len(l.buf))

    return nil
}

func (l *Logger) Debug(s string) error {
    err := l.output(2, s)
    return err
}
<commit_msg>test case<commit_after>package aralog

import (
    "io"
    "os"
    "runtime"
    "sync"
    "time"
    "strings"
    "path/filepath"
)

// These flags define which text to prefix to each log entry generated by the Logger.
const (
// Bits or'ed together to control what's printed. There is no control over the
// order they appear (the order listed here) or the format they present (as
// described in the comments).  A colon appears after these items:
//	2009/01/23 01:23:23.123123 /a/b/c/d.go:23: message
    Ldate = 1 << iota     // the date: 2009/01/23
    Ltime                         // the time: 01:23:23
    Lmicroseconds                 // microsecond resolution: 01:23:23.123123.  assumes Ltime.
    Llongfile                     // full file name and line number: /a/b/c/d.go:23
    Lshortfile                    // final file name element and line number: d.go:23. overrides Llongfile
    LstdFlags = Ldate | Ltime // initial values for the standard logger
)

// A Logger represents an active logging object that generates lines of
// output to an io.Writer.  Each logging operation makes a single call to
// the Writer's Write method.  A Logger can be used simultaneously from
// multiple goroutines; it guarantees to serialize access to the Writer.
type Logger struct {
    mu      sync.Mutex // ensures atomic writes; protects the following fields
    prefix  string     // prefix to write at beginning of each line
    flag    int        // properties
    out     io.Writer  // destination for output
    buf     []byte     // for accumulating text to write
    size    uint       // current size of log file
    path    string     // file path if output to a file
    maxsize uint       // minimal maxsize should >= 1MB
}

var currentOutFile *os.File

// New creates a new Logger.   The out variable sets the
// destination to which log data will be written.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func New(out io.Writer, prefix string, flag int) *Logger {
    return &Logger{out: out, prefix: prefix, flag: flag}
}

// NewFileLogger create a new Logger which output to a file specified
func NewFileLogger(path string, flag int) (*Logger, error) {
    return NewRollFileLogger(path, 1024 * 1024 * 10, flag)
}

// NewRollFileLogger create a new Logger which output to a file specified path,
// and roll at specified size
func NewRollFileLogger(path string, maxsize uint, flag int) (*Logger, error) {
    if strings.ContainsAny(path, string(filepath.Separator)) {
        dir := path
        // not ended by "/", ex: abc/d/e/x.log
        if strings.HasSuffix(path, string(filepath.Separator)) {
            path = path + "aralog.log" // the default log file name if not provided
        } else {
            i := strings.LastIndex(path, string(filepath.Separator))
            paths := strings.SplitAfterN(path, string(filepath.Separator), i)
            dir = paths[0]
        }

        os.MkdirAll(dir, 0600)
    }

    out, err := os.OpenFile(path, os.O_APPEND | os.O_CREATE | os.O_WRONLY, 0600)
    if err != nil {
        return nil, err
    }

    currentOutFile = out

    // minimal maxsize should >= 1MB
    if maxsize < 1024 * 1024 {
        maxsize = 1024 * 1024 * 10
    }

    return &Logger{out: out, prefix: "", flag: flag, path: path, maxsize: maxsize}, nil
}

//var std = New(os.Stderr, "", LstdFlags)

// Cheap integer to fixed-width decimal ASCII.  Give a negative width to avoid zero-padding.
// Knows the buffer has capacity.
func itoa(buf *[]byte, i int, wid int) {
    var u uint = uint(i)
    if u == 0 && wid <= 1 {
        *buf = append(*buf, '0')
        return
    }

    // Assemble decimal in reverse order.
    var b [32]byte
    bp := len(b)
    for ; u > 0 || wid > 0; u /= 10 {
        bp--
        wid--
        b[bp] = byte(u % 10) + '0'
    }
    *buf = append(*buf, b[bp:]...)
}

func (l *Logger) formatHeader(buf *[]byte, t time.Time, file string, line int) {
    *buf = append(*buf, l.prefix...)
    if l.flag & (Ldate | Ltime | Lmicroseconds) != 0 {
        if l.flag & Ldate != 0 {
            year, month, day := t.Date()
            itoa(buf, year, 4)
            *buf = append(*buf, '/')
            itoa(buf, int(month), 2)
            *buf = append(*buf, '/')
            itoa(buf, day, 2)
            *buf = append(*buf, ' ')
        }
        if l.flag & (Ltime | Lmicroseconds) != 0 {
            hour, min, sec := t.Clock()
            itoa(buf, hour, 2)
            *buf = append(*buf, ':')
            itoa(buf, min, 2)
            *buf = append(*buf, ':')
            itoa(buf, sec, 2)
            if l.flag & Lmicroseconds != 0 {
                *buf = append(*buf, '.')
                itoa(buf, t.Nanosecond() / 1e3, 6)
            }
            *buf = append(*buf, ' ')
        }
    }
    if l.flag & (Lshortfile | Llongfile) != 0 {
        if l.flag & Lshortfile != 0 {
            short := file
            for i := len(file) - 1; i > 0; i-- {
                if file[i] == '/' {
                    short = file[i + 1:]
                    break
                }
            }
            file = short
        }
        *buf = append(*buf, file...)
        *buf = append(*buf, ':')
        itoa(buf, line, -1)
        *buf = append(*buf, ": "...)
    }
}

// Output writes the output for a logging event.  The string s contains
// the text to print after the prefix specified by the flags of the
// Logger.  A newline is appended if the last character of s is not
// already a newline.  Calldepth is used to recover the PC and is
// provided for generality, although at the moment on all pre-defined
// paths it will be 2.
func (l *Logger) output(calldepth int, s string) error {
    now := time.Now() // get this early.
    var file string
    var line int
    l.mu.Lock()
    defer l.mu.Unlock()
    if l.flag & (Lshortfile | Llongfile) != 0 {
        // release lock while getting caller info - it's expensive.
        l.mu.Unlock()
        var ok bool
        _, file, line, ok = runtime.Caller(calldepth)
        if !ok {
            file = "???"
            line = 0
        }
        l.mu.Lock()
    }
    l.buf = l.buf[:0]
    l.formatHeader(&l.buf, now, file, line)
    l.buf = append(l.buf, s...)
    if len(s) > 0 && s[len(s) - 1] != '\n' {
        l.buf = append(l.buf, '\n')
    }

    if len(l.path) > 0 {
        err := l.rollFile(now)
        if err != nil {
            return err
        }
    }
    _, err := l.out.Write(l.buf)
    return err
}

func (l *Logger) rollFile(now time.Time) error {
    l.size += uint(len(l.buf))

    if l.size < l.maxsize {
        return nil
    }

    // file rotation if size > maxsize
    // close file before rename it
    if currentOutFile != nil {
        // ignore if Close() failed
        err := currentOutFile.Close()
        if err != nil {
            l.buf = append(l.buf, ("[XXX] ARALOGGER ERROR: Close current output file failed, " + err.Error())...)
            l.buf = append(l.buf, '\n')
        }
    }

    newPath := l.path

    // rename l.path to nameYYYYMMDDhhmmss
    err := os.Rename(l.path,
        l.path + string(now.Year()) + string(now.Month()) + string(now.Day()) +
        string(now.Hour()) + string(now.Minute()) + string(now.Second()))
    if err == nil {
        // TODO zip it
    } else {
        l.buf = append(l.buf, ("[XXX] ARALOGGER ERROR: Rolling file failed, " + err.Error())...)
        l.buf = append(l.buf, '\n')

        // if rename failed, start a new log file with different name
        newPath = l.path + string(now.Unix())
    }

    newOut, err := os.OpenFile(newPath, os.O_APPEND | os.O_WRONLY, 0600)
    if err != nil {
        return err
    }

    currentOutFile = newOut
    l.out = newOut
    l.size = uint(len(l.buf))

    return nil
}

func (l *Logger) Debug(s string) error {
    err := l.output(2, s)
    return err
}
<|endoftext|>


<commit_before>// Package mongodb is a parser for mongodb logs
package mongodb

import (
	"errors"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/honeycombio/mongodbtools/logparser"

	"github.com/honeycombio/honeytail/event"
)

const (
	// https://github.com/rueckstiess/mongodb-log-spec#timestamps
	ctimeNoMSTimeFormat    = "Mon Jan _2 15:04:05"
	ctimeTimeFormat        = "Mon Jan _2 15:04:05.000"
	iso8601UTCTimeFormat   = "2006-01-02T15:04:05.000Z"
	iso8601LocalTimeFormat = "2006-01-02T15:04:05.000-0700"

	timestampFieldName  = "timestamp"
	namespaceFieldName  = "namespace"
	databaseFieldName   = "database"
	collectionFieldName = "collection"
	locksFieldName      = "locks"
)

var timestampFormats = []string{
	iso8601LocalTimeFormat,
	iso8601UTCTimeFormat,
	ctimeTimeFormat,
	ctimeNoMSTimeFormat,
}

type Options struct {
	LogPartials bool `long:"log_partials" description:"Send what was successfully parsed from a line (only if the error occured in the log line's message)."`
}

type Parser struct {
	conf       Options
	lineParser LineParser
	nower      Nower
}

type LineParser interface {
	ParseLogLine(line string) (map[string]interface{}, error)
}

type MongoLineParser struct {
}

func (m *MongoLineParser) ParseLogLine(line string) (map[string]interface{}, error) {
	return logparser.ParseLogLine(line)
}

func (p *Parser) Init(options interface{}) error {
	p.conf = *options.(*Options)
	p.nower = &RealNower{}
	p.lineParser = &MongoLineParser{}
	return nil
}

func (p *Parser) ProcessLines(lines <-chan string, send chan<- event.Event) {
	for line := range lines {
		values, err := p.lineParser.ParseLogLine(line)
		// we get a bunch of errors from the parser on mongo logs, skip em
		if err == nil || (p.conf.LogPartials && logparser.IsPartialLogLine(err)) {
			timestamp, err := p.parseTimestamp(values)
			if err != nil {
				logFailure(line, err, "couldn't parse logline timestamp, skipping")
				continue
			}
			if err = p.decomposeNamespace(values); err != nil {
				logFailure(line, err, "couldn't decompose logline namespace, skipping")
				continue
			}
			if err = p.decomposeLocks(values); err != nil {
				logFailure(line, err, "couldn't decompose logline locks, skipping")
				continue
			}

			logrus.WithFields(logrus.Fields{
				"line":   line,
				"values": values,
			}).Debug("Successfully parsed line")

			// we'll be putting the timestamp in the Event
			// itself, no need to also have it in the Data
			delete(values, timestampFieldName)

			send <- event.Event{
				Timestamp: timestamp,
				Data:      values,
			}
		} else {
			logFailure(line, err, "logline didn't parse, skipping.")
		}
	}
	logrus.Debug("lines channel is closed, ending mongo processor")
}

func (p *Parser) parseTimestamp(values map[string]interface{}) (time.Time, error) {
	now := p.nower.Now()
	timestamp_value, ok := values[timestampFieldName].(string)
	if ok {
		var err error
		for _, f := range timestampFormats {
			var timestamp time.Time
			timestamp, err = time.Parse(f, timestamp_value)
			if err == nil {
				if f == ctimeTimeFormat || f == ctimeNoMSTimeFormat {
					// these formats lacks the year, so we check
					// if adding Now().Year causes the date to be
					// after today.  if it's after today, we
					// decrement year by 1.  if it's not after, we
					// use it.
					ts := timestamp.AddDate(now.Year(), 0, 0)
					if now.After(ts) {
						return ts, nil
					}

					return timestamp.AddDate(now.Year()-1, 0, 0), nil
				}
				return timestamp, nil
			}
		}
		return time.Time{}, err
	}

	return time.Time{}, errors.New("timestamp missing from logline")
}

func (p *Parser) decomposeNamespace(values map[string]interface{}) error {
	ns_value, ok := values[namespaceFieldName]
	if !ok {
		return nil
	}

	decomposed := strings.SplitN(ns_value.(string), ".", 2)
	if len(decomposed) < 2 {
		return nil
	}
	values[databaseFieldName] = decomposed[0]
	values[collectionFieldName] = decomposed[1]
	return nil
}

func (p *Parser) decomposeLocks(values map[string]interface{}) error {
	locks_value, ok := values[locksFieldName]
	if !ok {
		return nil
	}
	locks_map, ok := locks_value.(map[string]interface{})
	if !ok {
		return nil
	}
	for scope, v := range locks_map {
		v_map, ok := v.(map[string]interface{})
		if !ok {
			continue
		}
		for attrKey, attrVal := range v_map {
			attrVal_map, ok := attrVal.(map[string]interface{})
			if !ok {
				continue
			}
			for lockType, lockCount := range attrVal_map {
				if lockType == "r" {
					lockType = "read"
				} else if lockType == "w" {
					lockType = "write"
				}

				if attrKey == "acquireCount" {
					values[strings.ToLower(scope)+"_"+lockType+"_lock"] = lockCount
				} else if attrKey == "acquireWaitCount" {
					values[strings.ToLower(scope)+"_"+lockType+"_lock_wait"] = lockCount
				}
			}
		}
	}
	return nil
}

func logFailure(line string, err error, msg string) {
	logrus.WithFields(logrus.Fields{"line": line}).WithError(err).Debugln(msg)
}

type Nower interface {
	Now() time.Time
}

type RealNower struct{}

func (r *RealNower) Now() time.Time {
	return time.Now().UTC()
}
<commit_msg>move query normalization and (expanded) command read-or-write classification here from the mongodbtools repo<commit_after>// Package mongodb is a parser for mongodb logs
package mongodb

import (
	"errors"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/honeycombio/mongodbtools/logparser"
	queryshape "github.com/honeycombio/mongodbtools/queryshape"

	"github.com/honeycombio/honeytail/event"
)

const (
	// https://github.com/rueckstiess/mongodb-log-spec#timestamps
	ctimeNoMSTimeFormat    = "Mon Jan _2 15:04:05"
	ctimeTimeFormat        = "Mon Jan _2 15:04:05.000"
	iso8601UTCTimeFormat   = "2006-01-02T15:04:05.000Z"
	iso8601LocalTimeFormat = "2006-01-02T15:04:05.000-0700"

	timestampFieldName  = "timestamp"
	namespaceFieldName  = "namespace"
	databaseFieldName   = "database"
	collectionFieldName = "collection"
	locksFieldName      = "locks"
)

var timestampFormats = []string{
	iso8601LocalTimeFormat,
	iso8601UTCTimeFormat,
	ctimeTimeFormat,
	ctimeNoMSTimeFormat,
}

type Options struct {
	LogPartials bool `long:"log_partials" description:"Send what was successfully parsed from a line (only if the error occured in the log line's message)."`
}

type Parser struct {
	conf       Options
	lineParser LineParser
	nower      Nower
}

type LineParser interface {
	ParseLogLine(line string) (map[string]interface{}, error)
}

type MongoLineParser struct {
}

func (m *MongoLineParser) ParseLogLine(line string) (map[string]interface{}, error) {
	return logparser.ParseLogLine(line)
}

func (p *Parser) Init(options interface{}) error {
	p.conf = *options.(*Options)
	p.nower = &RealNower{}
	p.lineParser = &MongoLineParser{}
	return nil
}

func (p *Parser) ProcessLines(lines <-chan string, send chan<- event.Event) {
	for line := range lines {
		values, err := p.lineParser.ParseLogLine(line)
		// we get a bunch of errors from the parser on mongo logs, skip em
		if err == nil || (p.conf.LogPartials && logparser.IsPartialLogLine(err)) {
			timestamp, err := p.parseTimestamp(values)
			if err != nil {
				logFailure(line, err, "couldn't parse logline timestamp, skipping")
				continue
			}
			if err = p.decomposeNamespace(values); err != nil {
				logFailure(line, err, "couldn't decompose logline namespace, skipping")
				continue
			}
			if err = p.decomposeLocks(values); err != nil {
				logFailure(line, err, "couldn't decompose logline locks, skipping")
				continue
			}

			if q, ok := values["query"].(map[string]interface{}); ok {
				if _, ok = values["normalized_query"]; !ok {
					// also calculate the query_shape if we can
					values["normalized_query"] = queryshape.GetQueryShape(q)
				}
			}

			p.classifyReadOrWrite(values)

			logrus.WithFields(logrus.Fields{
				"line":   line,
				"values": values,
			}).Debug("Successfully parsed line")

			// we'll be putting the timestamp in the Event
			// itself, no need to also have it in the Data
			delete(values, timestampFieldName)

			send <- event.Event{
				Timestamp: timestamp,
				Data:      values,
			}
		} else {
			logFailure(line, err, "logline didn't parse, skipping.")
		}
	}
	logrus.Debug("lines channel is closed, ending mongo processor")
}

var commandReadOrWrite map[string]string = map[string]string{
	"aggregate":         "read",
	"bulkWrite":         "write",
	"count":             "read",
	"copyTo":            "read-write",
	"deleteOne":         "write",
	"deleteMany":        "write",
	"distinct":          "read",
	"find":              "read",
	"findAndModify":     "read-write",
	"findOne":           "read",
	"findOneAndDelete":  "read-write",
	"findOneAndReplace": "read-write",
	"findOneAndUpdate":  "read-write",
	"group":             "read",
	"insert":            "write",
	"insertOne":         "write",
	"insertMany":        "write",
	"mapReduce":         "read", /* can target a document, so read-write? */
	"replaceOne":        "write",
	"remove":            "write",
	"update":            "write",
	"updateOne":         "write",
	"updateMany":        "write",
}

func (p *Parser) classifyReadOrWrite(values map[string]interface{}) {
	// determine if this log line represents a read or write
	// operation.  not "operation" in the sense of the "operation"
	// field, but in the data direction.
	if operation, ok := values["operation"].(string); ok {
		readOrWrite := ""
		if operation == "query" {
			readOrWrite = "read"
		} else if operation == "insert" {
			readOrWrite = "write"
		} else if operation == "command" {
			if commandType, ok := values["command_type"].(string); ok {
				if commandRW, ok := commandReadOrWrite[commandType]; ok {
					readOrWrite = commandRW
				}
			}
		}
		values["read_or_write"] = readOrWrite
	}
}

func (p *Parser) parseTimestamp(values map[string]interface{}) (time.Time, error) {
	now := p.nower.Now()
	timestamp_value, ok := values[timestampFieldName].(string)
	if ok {
		var err error
		for _, f := range timestampFormats {
			var timestamp time.Time
			timestamp, err = time.Parse(f, timestamp_value)
			if err == nil {
				if f == ctimeTimeFormat || f == ctimeNoMSTimeFormat {
					// these formats lacks the year, so we check
					// if adding Now().Year causes the date to be
					// after today.  if it's after today, we
					// decrement year by 1.  if it's not after, we
					// use it.
					ts := timestamp.AddDate(now.Year(), 0, 0)
					if now.After(ts) {
						return ts, nil
					}

					return timestamp.AddDate(now.Year()-1, 0, 0), nil
				}
				return timestamp, nil
			}
		}
		return time.Time{}, err
	}

	return time.Time{}, errors.New("timestamp missing from logline")
}

func (p *Parser) decomposeNamespace(values map[string]interface{}) error {
	ns_value, ok := values[namespaceFieldName]
	if !ok {
		return nil
	}

	decomposed := strings.SplitN(ns_value.(string), ".", 2)
	if len(decomposed) < 2 {
		return nil
	}
	values[databaseFieldName] = decomposed[0]
	values[collectionFieldName] = decomposed[1]
	return nil
}

func (p *Parser) decomposeLocks(values map[string]interface{}) error {
	locks_value, ok := values[locksFieldName]
	if !ok {
		return nil
	}
	locks_map, ok := locks_value.(map[string]interface{})
	if !ok {
		return nil
	}
	for scope, v := range locks_map {
		v_map, ok := v.(map[string]interface{})
		if !ok {
			continue
		}
		for attrKey, attrVal := range v_map {
			attrVal_map, ok := attrVal.(map[string]interface{})
			if !ok {
				continue
			}
			for lockType, lockCount := range attrVal_map {
				if lockType == "r" {
					lockType = "read"
				} else if lockType == "w" {
					lockType = "write"
				}

				if attrKey == "acquireCount" {
					values[strings.ToLower(scope)+"_"+lockType+"_lock"] = lockCount
				} else if attrKey == "acquireWaitCount" {
					values[strings.ToLower(scope)+"_"+lockType+"_lock_wait"] = lockCount
				}
			}
		}
	}
	return nil
}

func logFailure(line string, err error, msg string) {
	logrus.WithFields(logrus.Fields{"line": line}).WithError(err).Debugln(msg)
}

type Nower interface {
	Now() time.Time
}

type RealNower struct{}

func (r *RealNower) Now() time.Time {
	return time.Now().UTC()
}
<|endoftext|>


<commit_before>package shellwords

import (
	"encoding/json"
	"testing"
)

func TestSimpleWord(t *testing.T) {
	var (
		words []string
		err   error
	)

	words, err = Split(`a-word`)
	equal(t, err, []string{"a-word"}, words)

	words, err = Split(`a-word foo bar`)
	equal(t, err, []string{"a-word", "foo", "bar"}, words)
}

func TestSingleQuoteWord(t *testing.T) {
	var (
		words []string
		err   error
	)

	words, err = Split(`'a-word'`)
	equal(t, err, []string{"a-word"}, words)

	words, err = Split(`'a-word' 'foo' 'bar'`)
	equal(t, err, []string{"a-word", "foo", "bar"}, words)

	words, err = Split(`'a-word' 'foo' cool'bar'`)
	equal(t, err, []string{"a-word", "foo", "coolbar"}, words)
}

func TestDoubleQuoteWord(t *testing.T) {
	var (
		words []string
		err   error
	)

	words, err = Split(`"a-word"`)
	equal(t, err, []string{"a-word"}, words)

	words, err = Split(`"a-word" "foo" "bar"`)
	equal(t, err, []string{"a-word", "foo", "bar"}, words)

	words, err = Split(`"a \"word\"" "foo \\ baz" cool"bar"`)
	equal(t, err, []string{"a \"word\"", "foo \\ baz", "coolbar"}, words)
}

func equal(t *testing.T, e error, exp, act []string) {
	if e != nil {
		t.Fatalf("err: %s", e)
	}

	err := false

	if len(exp) != len(act) {
		err = true
	} else {
		for i, w := range exp {
			if w != act[i] {
				err = true
				break
			}
		}
	}

	if err {
		exp_json, _ := json.Marshal(exp)
		act_json, _ := json.Marshal(act)
		t.Fatalf("expected %s but got %s", exp_json, act_json)
	}
}
<commit_msg>Added an example for Join<commit_after>package shellwords

import (
	"encoding/json"
	"fmt"
	"testing"
)

func ExampleSplit() {
	words, err := Split(`a-word 'cool' foo'bar'`)
	if err != nil {
		panic(err)
	}
	for _, word := range words {
		fmt.Println(word)
	}

	// Output:
	// a-word
	// cool
	// foobar
}

func TestSimpleWord(t *testing.T) {
	var (
		words []string
		err   error
	)

	words, err = Split(`a-word`)
	equal(t, err, []string{"a-word"}, words)

	words, err = Split(`a-word foo bar`)
	equal(t, err, []string{"a-word", "foo", "bar"}, words)
}

func TestSingleQuoteWord(t *testing.T) {
	var (
		words []string
		err   error
	)

	words, err = Split(`'a-word'`)
	equal(t, err, []string{"a-word"}, words)

	words, err = Split(`'a-word' 'foo' 'bar'`)
	equal(t, err, []string{"a-word", "foo", "bar"}, words)

	words, err = Split(`'a-word' 'foo' cool'bar'`)
	equal(t, err, []string{"a-word", "foo", "coolbar"}, words)
}

func TestDoubleQuoteWord(t *testing.T) {
	var (
		words []string
		err   error
	)

	words, err = Split(`"a-word"`)
	equal(t, err, []string{"a-word"}, words)

	words, err = Split(`"a-word" "foo" "bar"`)
	equal(t, err, []string{"a-word", "foo", "bar"}, words)

	words, err = Split(`"a \"word\"" "foo \\ baz" cool"bar"`)
	equal(t, err, []string{"a \"word\"", "foo \\ baz", "coolbar"}, words)
}

func equal(t *testing.T, e error, exp, act []string) {
	if e != nil {
		t.Fatalf("err: %s", e)
	}

	err := false

	if len(exp) != len(act) {
		err = true
	} else {
		for i, w := range exp {
			if w != act[i] {
				err = true
				break
			}
		}
	}

	if err {
		exp_json, _ := json.Marshal(exp)
		act_json, _ := json.Marshal(act)
		t.Fatalf("expected %s but got %s", exp_json, act_json)
	}
}
<|endoftext|>


<commit_before>package model

type RepositoryItemType byte

const (
	NONE                                     = iota
	REPOSITORYDESCRIPTION RepositoryItemType = iota
	DOCUMENT              RepositoryItemType = iota
	COMMENT               RepositoryItemType = iota
	LOCATION              RepositoryItemType = iota
	MESSAGE               RepositoryItemType = iota
	TAG                   RepositoryItemType = iota
)

func (itemType RepositoryItemType) String() string {
	switch itemType {

	case REPOSITORYDESCRIPTION:
		{
			return "Repository Description"
		}

	case DOCUMENT:
		{
			return "Document"
		}

	case COMMENT:
		{
			return "Comment"
		}

	case LOCATION:
		{
			return "Location"
		}

	case MESSAGE:
		{
			return "Message"
		}

	case TAG:
		{
			return "Tag"
		}

	}

	return "Unidentified Repository Item Type"
}
<commit_msg>The iota identifier is only required once per const declaration<commit_after>package model

type RepositoryItemType byte

const (
	NONE = iota
	REPOSITORYDESCRIPTION
	DOCUMENT
	COMMENT
	LOCATION
	MESSAGE
	TAG
)

func (itemType RepositoryItemType) String() string {
	switch itemType {

	case REPOSITORYDESCRIPTION:
		{
			return "Repository Description"
		}

	case DOCUMENT:
		{
			return "Document"
		}

	case COMMENT:
		{
			return "Comment"
		}

	case LOCATION:
		{
			return "Location"
		}

	case MESSAGE:
		{
			return "Message"
		}

	case TAG:
		{
			return "Tag"
		}

	}

	return "Unidentified Repository Item Type"
}
<|endoftext|>


<commit_before>package models

import "github.com/antihax/evedata/null"

func AddLPOffer(offerID int64, corporationID int64, typeID int64, quantity int64, lpCost int64, akCost, iskCost int64) error {
	if _, err := database.Exec(`
		INSERT INTO lpOffers
			(offerID,corporationID,typeID,quantity,lpCost,akCost,iskCost)
			VALUES(?,?,?,?,?,?,?);
	`, offerID, corporationID, typeID, quantity, lpCost, akCost, iskCost); err != nil {
		return err
	}
	return nil
}

func AddLPOfferRequirements(offerID int64, typeID int64, quantity int64) error {
	if _, err := database.Exec(`INSERT INTO lpOfferRequirements (offerID,typeID,quantity) VALUES(?,?,?);`,
		offerID, typeID, quantity); err != nil {
		return err
	}
	return nil
}

type IskPerLP struct {
	ItemName     string      `db:"itemName" json:"itemName"`
	TypeID       int64       `db:"typeID" json:"typeID"`
	TypeName     string      `db:"typeName" json:"typeName"`
	JitaPrice    float64     `db:"JitaPrice" json:"jitaPrice"`
	ItemCost     float64     `db:"itemCost" json:"itemCost"`
	IskPerLP     int64       `db:"iskPerLP" json:"iskPerLP"`
	JitaVolume   int64       `db:"JitaVolume" json:"jitaVolume"`
	IskVolume    float64     `db:"iskVolume" json:"iskVolume"`
	Requirements null.String `db:"requirements" json:"requirements"`
}

// [BENCHMARK] 0.016 sec / 0.000 sec
func GetISKPerLP(corporationName string) ([]IskPerLP, error) {
	s := []IskPerLP{}
	if err := database.Select(&s, `
		SELECT itemName, Lp.typeID, Lp.typeName, JitaPrice, itemCost, iskPerLP, JitaVolume, JitaVolume*JitaPrice AS iskVolume, GROUP_CONCAT(quantity, " x ", T.typeName SEPARATOR '
\n') AS requirements
			FROM evedata.iskPerLp Lp
			LEFT JOIN lpOfferRequirements R ON Lp.offerID = R.offerID
			LEFT JOIN invTypes T ON R.typeID = T.typeID
			WHERE itemName = ?
			GROUP BY Lp.typeName
			ORDER BY ISKperLP DESC;
	;`, corporationName); err != nil {

		return nil, err
	}
	return s, nil
}

type IskPerLPCorporation struct {
	ItemName string `db:"itemName" json:"itemName" `
}

func GetISKPerLPCorporations() ([]IskPerLPCorporation, error) {
	s := []IskPerLPCorporation{}
	if err := database.Select(&s, `SELECT DISTINCT itemName FROM evedata.iskPerLp ORDER BY itemName ASC;`); err != nil {
		return nil, err
	}
	return s, nil
}
<commit_msg>move to new schema<commit_after>package models

import "github.com/antihax/evedata/null"

func AddLPOffer(offerID int64, corporationID int64, typeID int64, quantity int64, lpCost int64, akCost, iskCost int64) error {
	if _, err := database.Exec(`
		INSERT INTO evedata.lpOffers
			(offerID,corporationID,typeID,quantity,lpCost,akCost,iskCost)
			VALUES(?,?,?,?,?,?,?);
	`, offerID, corporationID, typeID, quantity, lpCost, akCost, iskCost); err != nil {
		return err
	}
	return nil
}

func AddLPOfferRequirements(offerID int64, typeID int64, quantity int64) error {
	if _, err := database.Exec(`INSERT INTO evedata.lpOfferRequirements (offerID,typeID,quantity) VALUES(?,?,?);`,
		offerID, typeID, quantity); err != nil {
		return err
	}
	return nil
}

type IskPerLP struct {
	ItemName     string      `db:"itemName" json:"itemName"`
	TypeID       int64       `db:"typeID" json:"typeID"`
	TypeName     string      `db:"typeName" json:"typeName"`
	JitaPrice    float64     `db:"JitaPrice" json:"jitaPrice"`
	ItemCost     float64     `db:"itemCost" json:"itemCost"`
	IskPerLP     int64       `db:"iskPerLP" json:"iskPerLP"`
	JitaVolume   int64       `db:"JitaVolume" json:"jitaVolume"`
	IskVolume    float64     `db:"iskVolume" json:"iskVolume"`
	Requirements null.String `db:"requirements" json:"requirements"`
}

// [BENCHMARK] 0.016 sec / 0.000 sec
func GetISKPerLP(corporationName string) ([]IskPerLP, error) {
	s := []IskPerLP{}
	if err := database.Select(&s, `
		SELECT itemName, Lp.typeID, Lp.typeName, JitaPrice, itemCost, iskPerLP, JitaVolume, JitaVolume*JitaPrice AS iskVolume, GROUP_CONCAT(quantity, " x ", T.typeName SEPARATOR '
\n') AS requirements
			FROM evedata.iskPerLp Lp
			LEFT JOIN lpOfferRequirements R ON Lp.offerID = R.offerID
			LEFT JOIN invTypes T ON R.typeID = T.typeID
			WHERE itemName = ?
			GROUP BY Lp.typeName
			ORDER BY ISKperLP DESC;
	;`, corporationName); err != nil {

		return nil, err
	}
	return s, nil
}

type IskPerLPCorporation struct {
	ItemName string `db:"itemName" json:"itemName" `
}

func GetISKPerLPCorporations() ([]IskPerLPCorporation, error) {
	s := []IskPerLPCorporation{}
	if err := database.Select(&s, `SELECT DISTINCT itemName FROM evedata.iskPerLp ORDER BY itemName ASC;`); err != nil {
		return nil, err
	}
	return s, nil
}
<|endoftext|>


<commit_before>package cycletimer

import (
	"fmt"
	"testing"
	"time"
)

func Test_Timer(t *testing.T) {
	go Start(10)
	for i := 0; i < 10; i++ {
		//time.Sleep(time.Duration(3) * time.Second)
		go tickerTest()
	}
	time.Sleep(time.Duration(120) * time.Second)
}

func tickerTest() {
	c := NewTicker()
	res, ok := <-c
	if !ok {
		fmt.Println("close c")
		return
	}
	fmt.Println("c ", res)
}
<commit_msg>Modify test<commit_after>package cycletimer

import (
	"fmt"
	"testing"
	"time"
)

func Test_Timer(t *testing.T) {
	go Start(10)
	time.Sleep(time.Duration(1) * time.Second)
	for i := 0; i < 10; i++ {

		go tickerTest()
	}
	time.Sleep(time.Duration(120) * time.Second)
}

func tickerTest() {
	c := NewTicker()
	res, ok := <-c
	if !ok {
		fmt.Println("close c")
		return
	}
	fmt.Println("c ", res)
}
<|endoftext|>


<commit_before>package parse

import (
	"strings"

	"github.com/jimmyfrasche/etlite/internal/ast"
	"github.com/jimmyfrasche/etlite/internal/parse/internal/interpolate"
	"github.com/jimmyfrasche/etlite/internal/token"
)

type sqlParser struct {
	*parser
	sql *ast.SQL
}

func newSqlParser(p *parser) *sqlParser {
	return &sqlParser{
		parser: p,
		sql: &ast.SQL{
			Tokens: make([]token.Value, 0, 256),
		},
	}
}

func (p *sqlParser) extend(ts ...token.Value) {
	p.sql.Tokens = append(p.sql.Tokens, ts...)
}

func (p *sqlParser) push(t token.Value) {
	p.extend(t)
}

//synthesize a token and push it.
//Takes the current token as we use that as a rough proxy
//for the position in error messages.
func (p *sqlParser) synth(t token.Value, k token.Kind) {
	p.push(token.Value{
		Position: t.Position,
		Kind:     k,
	})
}

func digital(s string) bool {
	for i := 0; i < len(s); i++ {
		if b := s[i]; b < '0' || b > '9' {
			return false
		}
	}
	return true
}

//maybeRun eats possible runs of lits such as "IF", "NOT", "EXISTS".
func (p *sqlParser) maybeRun(t token.Value, lits ...string) token.Value {
	if !t.Literal(lits[0]) {
		return t
	}
	p.push(t) //lits[0]
	for _, lit := range lits[1:] {
		t = p.expectLit(lit)
		p.push(t)
	}
	return p.next()
}

//top does the statement level parsing
func (p *sqlParser) top(t token.Value, subq, etl bool) {
	//TODO recognize WITH so we can ban imports in DELETE with a better error message
	//and so that we can special case UPDATE
	//Forbidden statements
	if t.AnyLiteral("ANALYZE", "EXPLAIN", "ROLLBACK") {
		panic(p.errMsg(t, "ANALYZE and EXPLAIN and ROLLBACK are not allowed"))
	}

	//Savepoint and release need a check to see they don't step on
	//reserved savepoints
	if t.AnyLiteral("SAVEPOINT", "RELEASE") {
		if subq {
			panic(p.unexpected(t))
		}
		p.saverelease(t)
		return
	}

	//These are very simple and we just need to make sure nothing's obviously wrong
	//while seeking ;
	if t.AnyLiteral("DROP", "BEGIN", "END", "VACCUM", "REINDEX") {
		if subq {
			panic(p.unexpected(t))
		}
		p.slurp(t)
		return
	}

	if t.Literal("CREATE") {
		if subq {
			panic(p.unexpected(t))
		}
		p.push(t)
		t = p.next()
		temp := false
		if t.AnyLiteral("TEMP", "TEMPORARY") {
			p.push(t)
			t = p.next()
			temp = true
		}
		if t.Literal("TRIGGER") {
			p.trigger(t)
			return
		} else if t.Literal("TABLE") {
			p.table(t, temp)
			return
		}
	}
	//the stutter is not an accident: except for some special cases these are the same
	_ = p.regular(t, subq, etl, etl)
}

func (p *sqlParser) saverelease(t token.Value) {
	p.push(t)
	t = p.next()
	s, ok := t.Unescape()
	if !ok {
		panic(p.unexpected(t))
	}
	if digital(s) {
		panic(p.errMsg(t, "digital savepoint names are reserved by etlite"))
	}
	p.push(t)
	p.expect(token.Semicolon)
	return
}

//slurp simple statements until semicolon, making sure nothing untoward happens.
func (p *sqlParser) slurp(t token.Value) {
	for t.Kind != token.Semicolon {
		p.push(t)
		t = p.cantBe(token.Argument, token.LParen, token.RParen)
	}
	p.push(t) //the ;
}

//trigger handles triggers which have a special structure
//requiring them to be handled separately.
func (p *sqlParser) trigger(t token.Value) {
	//skip till begin
	for !t.Literal("BEGIN") {
		p.push(t)
		t = p.cantBe(token.Argument, token.LParen, token.RParen, token.Semicolon)
	}

	p.push(t) //BEGIN

	stmts := 0
	for {
		t = p.next()
		//end at the END
		if t.Literal("END") {
			if stmts == 0 {
				panic(p.errMsg(t, "trigger has no actions"))
			}
			p.push(t)
			t = p.expect(token.Semicolon)
			p.push(t)
			return
		}

		//otherwise, make sure we have a valid head
		if !t.AnyLiteral("INSERT", "UPDATE", "DELETE", "REPLACE", "SELECT", "WITH") {
			panic(p.unexpected(t))
		}
		stmts++
		t = p.regular(t, false, false, false)
	}
}

//table parses create table statements to handle CREATE TABLE FROM special form.
func (p *sqlParser) table(t token.Value, temp bool) {
	p.push(t)
	t = p.maybeRun(p.next(), "IF", "NOT", "EXISTS")

	//get the name in case this is CREATE TABLE FROM
	//and validate that it's not reserved.
	var name []token.Value
	t, name = p.name(t)
	p.extend(name...)
	if len(name) == 3 {
		s, _ := name[0].Unescape()
		if strings.ToUpper(s) == "TEMP" {
			if temp {
				panic(p.unexpected(name[0]))
			}
			temp = true
		}
	}
	if temp {
		last := name[len(name)-1]
		s, _ := last.Unescape()
		if digital(s) {
			panic(p.errMsg(last, "digital temporary table names are reserved by etlite"))
		}
	}

	if t.Literal("AS") {
		p.push(t)
		_ = p.regular(p.next(), false, false, true)
		//XXX is this fair? safe to do subimport in create table?
		//XXX it wouldn't respect the usual rules and if it failed
		//XXX the table wouldn't exist, unlike with import statement. Must think.
		return
	}

	//we're at the column definitions now, just need to make sure we handle
	//(()) and catch anything that's obviously wrong.
	if t.Kind != token.LParen {
		panic(p.unexpected(t))
	}
	p.push(t)
	depth := 1
loop:
	for {
		t = p.cantBe(token.Semicolon, token.Argument)
		switch t.Kind {
		case token.LParen:
			depth++
		case token.RParen:
			depth--
			if depth == 0 {
				p.push(t)
				break loop
			}
		case token.Literal:
			if t.Head(false) {
				panic(p.unexpected(t))
			}
		}
		p.push(t)
	}

	t = p.maybeRun(t, "WITHOUT", "ROWID")
	switch {
	case t.Kind == token.Semicolon: //done
		p.push(t)
		return
	case t.Literal("FROM"):
		//we do not push "FROM" since this is fake syntax
		//instead, we insert a synthetic semicolon
		p.synth(t, token.Semicolon)

		i := p.importStmt(p.expectLit("IMPORT"), false)
		p.sql.Name = name
		p.sql.Subqueries = []*ast.Import{i}
	}
}

//The regular parser mops up everything else.
func (p *sqlParser) regular(t token.Value, subq, etl, arg bool) token.Value {
	//TODO check for cases where we recognize arguments but not imports, like DELETE, UPDATE

	//This handles all sql we don't explicitly recognize.
	//It ensures that parens are balanced and finds the end of the statement
	//or subquery,
	//handling some special cases along the way.
	depth := 0
	for {
		switch t.Kind {
		case token.Semicolon:
			if !subq {
				p.push(t)
				return t //leave on last token for trigger parser
			} else {
				panic(p.unexpected(t))
			}

		case token.RParen:
			depth--
			if depth < 0 {
				panic(p.errMsg(t, "unbalanced parentheses: ())"))
			}
			p.push(t)
			if depth == 0 && subq {
				return t //leave on last token for trigger parser
			}
			t = p.next()

		case token.LParen:
			depth++
			p.push(t)
			t = p.next()

			if !t.Literal("IMPORT") {
				//quick sanity check while we're here,
				//we have a non-subquery head in a subquery position:
				//definite error.
				if t.Head(false) && !t.Head(true) {
					panic(p.unexpected(t))
				}

				//we could even end up back in this case if t = (,
				//but that is the correct behavior, handling the case (((etc.
				//for regular sql subqueries we just slurp em up
				continue
			}

			if !etl {
				panic(p.errMsg(t, "illegal IMPORT subquery"))
			}

			//handle nested import
			p.sql.Subqueries = append(p.sql.Subqueries, p.importStmt(t, true))

			depth-- // ) consumed by import

			//add placeholder that the compiler rewrites
			p.synth(t, token.Placeholder)
			t = p.next()
			p.synth(t, token.RParen)

		case token.Argument:
			if !arg {
				panic(p.errMsg(t, "illegal @ substitution"))
			}
			//TODO move all sql rewriting to compiler, and just fallthrough

			ts, err := interpolate.Desugar(t)
			if err != nil {
				panic(p.mkErr(t, err))
			}
			p.synth(t, token.LParen)
			p.extend(ts...)
			p.synth(t, token.RParen)
			t = p.next()

		default:
			p.push(t)
			t = p.next()
		}
	}
}
<commit_msg>greatly extend sql recognizer<commit_after>package parse

import (
	"strings"

	"github.com/jimmyfrasche/etlite/internal/ast"
	"github.com/jimmyfrasche/etlite/internal/parse/internal/interpolate"
	"github.com/jimmyfrasche/etlite/internal/token"
)

//TODO should recognize imports in compound selects (IMPORT ... UNION SELECT ... and vice versa) would just need to compile to subq import
//just need to make sure no () are put around placeholder tokens

type sqlParser struct {
	*parser
	sql *ast.SQL
}

func newSqlParser(p *parser) *sqlParser {
	return &sqlParser{
		parser: p,
		sql: &ast.SQL{
			Tokens: make([]token.Value, 0, 256),
		},
	}
}

func (p *sqlParser) extend(ts ...token.Value) {
	p.sql.Tokens = append(p.sql.Tokens, ts...)
}

func (p *sqlParser) push(t token.Value) {
	p.extend(t)
}

//synthesize a token and push it.
//Takes the current token as we use that as a rough proxy
//for the position in error messages.
func (p *sqlParser) synth(t token.Value, k token.Kind) {
	p.push(token.Value{
		Position: t.Position,
		Kind:     k,
	})
}

func digital(s string) bool {
	for i := 0; i < len(s); i++ {
		if b := s[i]; b < '0' || b > '9' {
			return false
		}
	}
	return true
}

func (p *sqlParser) chkDigTmp(name []token.Value) {
	if len(name) == 3 {
		s, _ := name[0].Unescape()
		if strings.ToUpper(s) == "TEMP" {
			s, _ = name[2].Unescape()
			if digital(s) {
				panic(p.errMsg(name[0], "digital temporary table names are reserved by etlite"))
			}
		}
	}
}

func (p *sqlParser) tmpCheck(t token.Value) token.Value {
	t, name := p.name(t)
	p.chkDigTmp(name)
	p.extend(name...)
	return t
}

func (p *sqlParser) chkSysReserved(name []token.Value) {
	if len(name) == 3 {
		s, _ := name[0].Unescape()
		//this fails if it relies on object resolution but catches some misuse.
		//TODO could insert a flag in the AST to check sys names exist if length 1 and name[0] ∈ {args, env}?
		if strings.ToUpper(s) == "SYS" {
			s, _ = name[2].Unescape()
			s = strings.ToUpper(s)
			switch s {
			case "ARGS", "ENV":
				panic(p.errMsg(name[0], "sys.args and sys.env are reserved by etlite"))
			}
		}
	}
}

func (p *sqlParser) tmpOrSysCheck(t token.Value) token.Value {
	t, name := p.name(t)
	p.chkDigTmp(name)
	p.chkSysReserved(name)
	p.extend(name...)
	return t
}

//maybeRun eats possible runs of lits such as "IF", "NOT", "EXISTS".
func (p *sqlParser) maybeRun(t token.Value, lits ...string) token.Value {
	if !t.Literal(lits[0]) {
		return t
	}
	p.push(t) //lits[0]
	for _, lit := range lits[1:] {
		t = p.expectLit(lit)
		p.push(t)
	}
	return p.next()
}

//top does the statement level parsing
func (p *sqlParser) top(t token.Value, subq, etl bool) {
	if t.Kind != token.Literal {
		panic(p.unexpected(t))
	}

	//Forbidden statements
	if t.AnyLiteral("ANALYZE", "EXPLAIN", "ROLLBACK") {
		panic(p.errMsg(t, "ANALYZE and EXPLAIN and ROLLBACK are not allowed"))
	}

	//Savepoint and release need a check to see they don't step on
	//reserved savepoints
	if t.AnyLiteral("SAVEPOINT", "RELEASE") {
		if subq {
			panic(p.unexpected(t))
		}
		p.saverelease(t)
		return
	}

	//These are very simple and we just need to make sure nothing's obviously wrong
	//while seeking ;
	if t.AnyLiteral("BEGIN", "END", "VACCUM", "REINDEX") {
		if subq {
			panic(p.unexpected(t))
		}
		p.slurp(t)
		return
	}

	//for these two we validate no reserved names are injured.
	if t.Literal("ALTER") {
		if subq {
			panic(p.unexpected(t))
		}
		p.alterTable(t)
		return
	}
	if t.Literal("DROP") {
		if subq {
			panic(p.unexpected(t))
		}
		p.drop(t)
		return
	}

	if t.Literal("CREATE") {
		if subq {
			panic(p.unexpected(t))
		}
		p.push(t)
		t = p.next()
		temp := false
		if t.AnyLiteral("TEMP", "TEMPORARY") {
			p.push(t)
			t = p.next()
			temp = true
		}
		if t.Literal("TRIGGER") {
			p.trigger(t)
			return
		} else if t.Literal("TABLE") {
			p.table(t, temp)
			return
		}
	}

	//the stutter is not an accident: except for some special cases these are the same
	switch t.Canon {
	case "INSERT", "REPLACE":
		_ = p.insert(t, subq, etl, etl)
	case "DELETE":
		_ = p.delete(t, subq, etl, etl)
	case "UPDATE":
		_ = p.update(t, subq, etl, etl)
	case "WITH":
		_ = p.with(t, subq, etl, etl)
	default:
		_ = p.regular(t, 0, subq, etl, etl)
	}
}

func (p *sqlParser) alterTable(t token.Value) {
	p.push(t)
	t = p.expectLit("TABLE")
	p.push(t)
	t = p.tmpOrSysCheck(t)
	switch t.Canon {
	default:
		panic(p.unexpected(t))
	case "RENAME":
		p.push(t)
		t = p.expectLit("TO")
		p.push(t)
		t = p.tmpOrSysCheck(t)
		if t.Kind != token.Semicolon {
			panic(p.unexpected(t))
		}
		p.push(t)
	case "ADD":
		p.push(t)
		_ = p.regular(t, 0, false, false, false)
	}
}

func (p *sqlParser) drop(t token.Value) {
	p.push(t)
	t = p.expect(token.Literal)
	if !t.Literal("TABLE") {
		p.slurp(t)
	}
	p.push(t)
	t = p.maybeRun(p.next(), "IF", "EXISTS")
	t = p.tmpOrSysCheck(t)
	if t.Kind != token.Semicolon {
		panic(p.unexpected(t))
	}
	p.push(t)
}

func (p *sqlParser) saverelease(t token.Value) {
	p.push(t)
	t = p.next()
	s, ok := t.Unescape()
	if !ok {
		panic(p.unexpected(t))
	}
	if digital(s) {
		panic(p.errMsg(t, "digital savepoint names are reserved by etlite"))
	}
	p.push(t)
	p.expect(token.Semicolon)
}

//slurp simple statements until semicolon, making sure nothing untoward happens.
func (p *sqlParser) slurp(t token.Value) {
	for t.Kind != token.Semicolon {
		p.push(t)
		t = p.cantBe(token.Argument, token.LParen, token.RParen)
	}
	p.push(t) //the ;
}

func (p *sqlParser) with(t token.Value, subq, etl, arg bool) token.Value {
	p.push(t)
	first := true
	for {
		t = p.expectLitOrStr() // name or possibly RECURSIVE if first time through

		if t.Literal("RECURSIVE") {
			if first {
				p.push(t)
				t = p.expectLitOrStr()
			} else {
				panic(p.unexpected(t))
			}
		}
		first = false

		p.push(t)
		t = p.next()
		if t.Kind == token.LParen { //optional column names
			p.push(t)
			for t.Kind != token.RParen {
				t = p.next()
				p.push(t)
			}
			t = p.next()
		}
		if t.Literal("AS") {
			p.push(t)
		} else {
			panic(p.unexpected(t))
		}

		//The table expression
		t = p.expect(token.LParen)
		p.push(t)
		t = p.expect(token.Literal)
		if t.Literal("WITH") {
			t = p.with(t, true, etl, arg)
		} else {
			t = p.regular(t, 1, true, etl, arg)
		}

		t = p.expect(token.Literal)
		if !t.Literal(",") {
			break
		}
	}

	switch t.Canon {
	default:
		panic(p.unexpected(t))
	case "INSERT", "REPLACE":
		return p.insert(t, subq, etl, arg)
	case "UPDATE":
		//probably too complicated to do any special checking here but should at least check table name
		return p.update(t, subq, etl, arg)
	case "DELETE":
		return p.delete(t, subq, etl, arg)
	case "SELECT":
		depth := 0
		if subq {
			depth++
		}
		return p.regular(t, depth, subq, etl, arg)
	}
}

func (p *sqlParser) delete(t token.Value, subq, etl, arg bool) token.Value {
	if subq {
		panic(p.unexpected(t))
	}
	p.push(t)
	t = p.expectLit("FROM")
	p.push(t)
	t = p.tmpCheck(p.next())
	return p.regular(t, 0, subq, etl, arg)
}

func (p *sqlParser) update(t token.Value, subq, etl, arg bool) token.Value {
	if subq {
		panic(p.unexpected(t))
	}
	p.push(t)
	t = p.expectLitOrStr()
	if t.Literal("OR") {
		p.push(t)
		t = p.expect(token.Literal) //ROLLBACK, etc.
		p.push(t)
		t = p.next()
	}
	t = p.tmpCheck(t)
	if !t.Literal("SET") {
		panic(p.unexpected(t))
	}
	return p.regular(t, 0, subq, etl, arg)
}

func (p *sqlParser) insert(t token.Value, subq, etl, arg bool) token.Value {
	if subq {
		panic(p.unexpected(t))
	}
	replace := t.Literal("REPLACE")
	p.push(t)

	t = p.expect(token.Literal)
	if t.Literal("OR") {
		if replace {
			panic(p.unexpected(t))
		}
		p.push(t)
		t = p.expect(token.Literal) //ROLLBACK, etc.
		p.push(t)
		t = p.next()
	}
	if !t.Literal("INTO") {
		panic(p.unexpected(t))
	}
	p.push(t)

	t = p.tmpCheck(p.next())
	if t.Kind != token.LParen {
		panic(p.unexpected(t))
	}
	p.push(t)

loop:
	for {
		t = p.expectLitOrStr()
		p.push(t)
		t = p.next()
		switch {
		default:
			panic(p.unexpected(t))
		case t.Literal(","):
			p.push(t)
		case t.Kind == token.RParen:
			p.push(t)
			break loop
		}
	}

	t = p.expect(token.Literal)
	switch t.Canon {
	default:
		panic(p.unexpected(t))
	case "DEFAULT": //not to be confused with the above
		p.push(t)
		t = p.expectLit("VALUES")
		p.push(t)
		t = p.expect(token.Semicolon)
		p.push(t)
		return t //this isn't used anywhere, but needed for symmetry
	case "IMPORT":
		//TODO we could add a special FROM IMPORT here, with a little work
		panic(p.errMsg(t, "INSERT ... IMPORT is currently unsupported"))
	case "WITH": //XXX is this legal? Seems like it should be
		return p.with(t, subq, etl, arg)
	case "VALUES", "SELECT":
		return p.regular(t, 0, subq, etl, arg)
	}
}

//trigger handles triggers which have a special structure
//requiring them to be handled separately.
func (p *sqlParser) trigger(t token.Value) {
	//skip till begin
	for !t.Literal("BEGIN") {
		p.push(t)
		t = p.cantBe(token.Argument, token.LParen, token.RParen, token.Semicolon)
	}

	p.push(t) //BEGIN

	stmts := 0
	for {
		t = p.next()
		//end at the END
		if t.Literal("END") {
			if stmts == 0 {
				panic(p.errMsg(t, "trigger has no actions"))
			}
			p.push(t)
			t = p.expect(token.Semicolon)
			p.push(t)
			return
		}

		//otherwise, make sure we have a valid head
		switch t.Canon {
		default:
			panic(p.unexpected(t))
		case "INSERT", "REPLACE":
			t = p.insert(t, false, false, false)
		case "UPDATE":
			t = p.update(t, false, false, false)
		case "DELETE":
			t = p.delete(t, false, false, false)
		case "SELECT":
			t = p.regular(t, 0, false, false, false)
		}
		stmts++
	}
}

//table parses create table statements to handle CREATE TABLE FROM special form.
func (p *sqlParser) table(t token.Value, temp bool) {
	p.push(t)
	t = p.maybeRun(p.next(), "IF", "NOT", "EXISTS")

	//get the name in case this is CREATE TABLE FROM
	//and validate that it's not reserved.
	var name []token.Value
	t, name = p.name(t)
	p.extend(name...)
	if len(name) == 3 {
		s, _ := name[0].Unescape()
		if strings.ToUpper(s) == "TEMP" {
			if temp {
				panic(p.unexpected(name[0]))
			}
			temp = true
		}
	}
	if temp {
		last := name[len(name)-1]
		s, _ := last.Unescape()
		if digital(s) {
			panic(p.errMsg(last, "digital temporary table names are reserved by etlite"))
		}
	}

	if t.Literal("AS") {
		p.push(t)
		_ = p.regular(p.next(), 0, false, false, true)
		//XXX is this fair? safe to do subimport in create table?
		//XXX it wouldn't respect the usual rules and if it failed
		//XXX the table wouldn't exist, unlike with import statement. Must think.
		//XXX note in ast, special case compiler to treat like normal subquery import
		//XXX but release savepoint before create and drop tables after.
		return
	}

	//we're at the column definitions now, just need to make sure we handle
	//(()) and catch anything that's obviously wrong.
	if t.Kind != token.LParen {
		panic(p.unexpected(t))
	}
	p.push(t)
	depth := 1
loop:
	for {
		t = p.cantBe(token.Semicolon, token.Argument)
		switch t.Kind {
		case token.LParen:
			depth++
		case token.RParen:
			depth--
			if depth == 0 {
				p.push(t)
				break loop
			}
		case token.Literal:
			if t.Head(false) {
				panic(p.unexpected(t))
			}
		}
		p.push(t)
	}

	t = p.maybeRun(t, "WITHOUT", "ROWID")
	switch {
	case t.Kind == token.Semicolon: //done
		p.push(t)
		return
	case t.Literal("FROM"):
		//we do not push "FROM" since this is fake syntax
		//instead, we insert a synthetic semicolon
		p.synth(t, token.Semicolon)

		i := p.importStmt(p.expectLit("IMPORT"), false)
		p.sql.Name = name
		p.sql.Subqueries = []*ast.Import{i}
	}
}

//The regular parser mops up everything else.
func (p *sqlParser) regular(t token.Value, depth int, subq, etl, arg bool) token.Value {
	//This handles all sql we don't explicitly recognize.
	//It ensures that parens are balanced and finds the end of the statement
	//or subquery,
	//handling some special cases along the way.
	for {
		switch t.Kind {
		case token.Semicolon:
			if !subq {
				p.push(t)
				return t //leave on last token for trigger parser
			} else {
				panic(p.unexpected(t))
			}

		case token.RParen:
			depth--
			if depth < 0 {
				panic(p.errMsg(t, "unbalanced parentheses: ())"))
			}
			p.push(t)
			if depth == 0 && subq {
				return t //leave on last token for trigger parser
			}
			t = p.next()

		case token.LParen:
			depth++
			p.push(t)
			t = p.next()
			if t.Kind != token.Literal {
				continue
			}

			switch t.Canon {
			default:
				if t.Head(false) && !t.Head(true) {
					panic(p.unexpected(t))
				}

			case "WITH":
				t = p.with(t, true, etl, arg)

			case "IMPORT":
				if !etl {
					panic(p.errMsg(t, "illegal IMPORT subquery"))
				}

				//handle nested import
				p.sql.Subqueries = append(p.sql.Subqueries, p.importStmt(t, true))

				depth-- // ) consumed by import

				//add placeholder that the compiler rewrites
				p.synth(t, token.Placeholder)
				t = p.next()
				p.synth(t, token.RParen)
			}

		case token.Argument:
			if !arg {
				panic(p.errMsg(t, "illegal @ substitution"))
			}
			//TODO move all sql rewriting to compiler, and just fallthrough

			ts, err := interpolate.Desugar(t)
			if err != nil {
				panic(p.mkErr(t, err))
			}
			p.synth(t, token.LParen)
			p.extend(ts...)
			p.synth(t, token.RParen)
			t = p.next()

		default:
			p.push(t)
			t = p.next()
		}
	}
}
<|endoftext|>


<commit_before>package uvm

import (
	"fmt"
	"strconv"

	"github.com/Microsoft/hcsshim/internal/guestrequest"
	"github.com/Microsoft/hcsshim/internal/requesttype"
	"github.com/Microsoft/hcsshim/internal/schema2"
	"github.com/sirupsen/logrus"
)

// allocateVPMEM finds the next available VPMem slot. The lock MUST be held
// when calling this function.
func (uvm *UtilityVM) allocateVPMEM(hostPath string) (uint32, error) {
	for index, vi := range uvm.vpmemDevices {
		if vi.hostPath == "" {
			vi.hostPath = hostPath
			logrus.Debugf("uvm::allocateVPMEM %d %q", index, hostPath)
			return uint32(index), nil
		}
	}
	return 0, fmt.Errorf("no free VPMEM locations")
}

func (uvm *UtilityVM) deallocateVPMEM(deviceNumber uint32) error {
	uvm.m.Lock()
	defer uvm.m.Unlock()
	uvm.vpmemDevices[deviceNumber] = vpmemInfo{}
	return nil
}

// Lock must be held when calling this function
func (uvm *UtilityVM) findVPMEMDevice(findThisHostPath string) (uint32, string, error) {
	for deviceNumber, vi := range uvm.vpmemDevices {
		if vi.hostPath == findThisHostPath {
			logrus.Debugf("uvm::findVPMEMDeviceNumber %d %s", deviceNumber, findThisHostPath)
			return uint32(deviceNumber), vi.uvmPath, nil
		}
	}
	return 0, "", fmt.Errorf("%s is not attached to VPMEM", findThisHostPath)
}

// AddVPMEM adds a VPMEM disk to a utility VM at the next available location.
//
// Returns the location(0..MaxVPMEM-1) where the device is attached, and if exposed,
// the utility VM path which will be /tmp/p<location>//
func (uvm *UtilityVM) AddVPMEM(hostPath string, expose bool) (uint32, string, error) {
	if uvm.operatingSystem != "linux" {
		return 0, "", errNotSupported
	}

	logrus.Debugf("uvm::AddVPMEM id:%s hostPath:%s expose:%t", uvm.id, hostPath, expose)

	uvm.m.Lock()
	defer uvm.m.Unlock()

	var deviceNumber uint32
	var err error
	uvmPath := ""

	deviceNumber, uvmPath, err = uvm.findVPMEMDevice(hostPath)
	if err != nil {
		// It doesn't exist, so we're going to allocate and hot-add it
		deviceNumber, err = uvm.allocateVPMEM(hostPath)
		if err != nil {
			return 0, "", err
		}
		controller := hcsschema.VirtualPMemController{}
		controller.Devices = make(map[string]hcsschema.VirtualPMemDevice)
		controller.Devices[strconv.Itoa(int(deviceNumber))] = hcsschema.VirtualPMemDevice{
			HostPath:    hostPath,
			ReadOnly:    true,
			ImageFormat: "Vhd1",
		}

		modification := &hcsschema.ModifySettingRequest{
			RequestType:  requesttype.Add,
			Settings:     controller,
			ResourcePath: fmt.Sprintf("VirtualMachine/Devices/VirtualPMem/%d", deviceNumber),
		}

		if expose {
			uvmPath = fmt.Sprintf("/tmp/p%d", deviceNumber)
			modification.GuestRequest = guestrequest.GuestRequest{
				ResourceType: guestrequest.ResourceTypeVPMemDevice,
				RequestType:  requesttype.Add,
				Settings: guestrequest.LCOWMappedVPMemDevice{
					DeviceNumber: deviceNumber,
					MountPath:    uvmPath,
				},
			}
		}

		if err := uvm.Modify(modification); err != nil {
			uvm.vpmemDevices[deviceNumber] = vpmemInfo{}
			return 0, "", fmt.Errorf("uvm::AddVPMEM: failed to modify utility VM configuration: %s", err)
		}

		uvm.vpmemDevices[deviceNumber] = vpmemInfo{
			hostPath: hostPath,
			refCount: 1,
			uvmPath:  uvmPath}
	} else {
		pmemi := vpmemInfo{
			hostPath: hostPath,
			refCount: uvm.vpmemDevices[deviceNumber].refCount + 1,
			uvmPath:  uvmPath}
		uvm.vpmemDevices[deviceNumber] = pmemi
	}
	logrus.Debugf("hcsshim::AddVPMEM id:%s Success %+v", uvm.id, uvm.vpmemDevices[deviceNumber])
	return deviceNumber, uvmPath, nil

}

// RemoveVPMEM removes a VPMEM disk from a utility VM. As an external API, it
// is "safe". Internal use can call removeVPMEM.
func (uvm *UtilityVM) RemoveVPMEM(hostPath string) error {
	if uvm.operatingSystem != "linux" {
		return errNotSupported
	}

	uvm.m.Lock()
	defer uvm.m.Unlock()

	// Make sure is actually attached
	deviceNumber, uvmPath, err := uvm.findVPMEMDevice(hostPath)
	if err != nil {
		return fmt.Errorf("cannot remove VPMEM %s as it is not attached to utility VM %s: %s", hostPath, uvm.id, err)
	}

	if err := uvm.removeVPMEM(hostPath, uvmPath, deviceNumber); err != nil {
		return fmt.Errorf("failed to remove VPMEM %s from utility VM %s: %s", hostPath, uvm.id, err)
	}
	return nil
}

// removeVPMEM is the internally callable "unsafe" version of RemoveVPMEM. The mutex
// MUST be held when calling this function.
func (uvm *UtilityVM) removeVPMEM(hostPath string, uvmPath string, deviceNumber uint32) error {
	logrus.Debugf("uvm::RemoveVPMEM id:%s hostPath:%s device:%d", uvm.id, hostPath, deviceNumber)

	if uvm.vpmemDevices[deviceNumber].refCount == 1 {
		modification := &hcsschema.ModifySettingRequest{
			RequestType:  requesttype.Remove,
			ResourcePath: fmt.Sprintf("VirtualMachine/Devices/VirtualPMem/%d", deviceNumber),
		}

		modification.GuestRequest = guestrequest.GuestRequest{
			ResourceType: guestrequest.ResourceTypeVPMemDevice,
			RequestType:  requesttype.Remove,
			Settings: guestrequest.LCOWMappedVPMemDevice{
				DeviceNumber: deviceNumber,
				MountPath:    uvmPath,
			},
		}

		if err := uvm.Modify(modification); err != nil {
			return err
		}
		uvm.vpmemDevices[deviceNumber] = vpmemInfo{}
		logrus.Debugf("uvm::RemoveVPMEM: Success id:%s hostPath:%s device:%d", uvm.id, hostPath, deviceNumber)
		return nil
	}
	uvm.vpmemDevices[deviceNumber].refCount--
	logrus.Debugf("uvm::RemoveVPMEM: Success id:%s hostPath:%s device:%d refCount:%d", uvm.id, hostPath, deviceNumber, uvm.vpmemDevices[deviceNumber].refCount)
	return nil

}
<commit_msg>Fix vpmem add/remove<commit_after>package uvm

import (
	"fmt"
	"strconv"

	"github.com/Microsoft/hcsshim/internal/guestrequest"
	"github.com/Microsoft/hcsshim/internal/requesttype"
	"github.com/Microsoft/hcsshim/internal/schema2"
	"github.com/sirupsen/logrus"
)

// allocateVPMEM finds the next available VPMem slot. The lock MUST be held
// when calling this function.
func (uvm *UtilityVM) allocateVPMEM(hostPath string) (uint32, error) {
	for index, vi := range uvm.vpmemDevices {
		if vi.hostPath == "" {
			vi.hostPath = hostPath
			logrus.Debugf("uvm::allocateVPMEM %d %q", index, hostPath)
			return uint32(index), nil
		}
	}
	return 0, fmt.Errorf("no free VPMEM locations")
}

func (uvm *UtilityVM) deallocateVPMEM(deviceNumber uint32) error {
	uvm.m.Lock()
	defer uvm.m.Unlock()
	uvm.vpmemDevices[deviceNumber] = vpmemInfo{}
	return nil
}

// Lock must be held when calling this function
func (uvm *UtilityVM) findVPMEMDevice(findThisHostPath string) (uint32, string, error) {
	for deviceNumber, vi := range uvm.vpmemDevices {
		if vi.hostPath == findThisHostPath {
			logrus.Debugf("uvm::findVPMEMDeviceNumber %d %s", deviceNumber, findThisHostPath)
			return uint32(deviceNumber), vi.uvmPath, nil
		}
	}
	return 0, "", fmt.Errorf("%s is not attached to VPMEM", findThisHostPath)
}

// AddVPMEM adds a VPMEM disk to a utility VM at the next available location.
//
// Returns the location(0..MaxVPMEM-1) where the device is attached, and if exposed,
// the utility VM path which will be /tmp/p<location>//
func (uvm *UtilityVM) AddVPMEM(hostPath string, expose bool) (uint32, string, error) {
	if uvm.operatingSystem != "linux" {
		return 0, "", errNotSupported
	}

	logrus.Debugf("uvm::AddVPMEM id:%s hostPath:%s expose:%t", uvm.id, hostPath, expose)

	uvm.m.Lock()
	defer uvm.m.Unlock()

	var deviceNumber uint32
	var err error
	uvmPath := ""

	deviceNumber, uvmPath, err = uvm.findVPMEMDevice(hostPath)
	if err != nil {
		// It doesn't exist, so we're going to allocate and hot-add it
		deviceNumber, err = uvm.allocateVPMEM(hostPath)
		if err != nil {
			return 0, "", err
		}
		controller := hcsschema.VirtualPMemController{}
		controller.Devices = make(map[string]hcsschema.VirtualPMemDevice)
		controller.Devices[strconv.Itoa(int(deviceNumber))] = hcsschema.VirtualPMemDevice{
			HostPath:    hostPath,
			ReadOnly:    true,
			ImageFormat: "Vhd1",
		}

		modification := &hcsschema.ModifySettingRequest{
			RequestType:  requesttype.Add,
			Settings:     controller,
			ResourcePath: fmt.Sprintf("VirtualMachine/Devices/VirtualPMem"),
		}

		if expose {
			uvmPath = fmt.Sprintf("/tmp/p%d", deviceNumber)
			modification.GuestRequest = guestrequest.GuestRequest{
				ResourceType: guestrequest.ResourceTypeVPMemDevice,
				RequestType:  requesttype.Add,
				Settings: guestrequest.LCOWMappedVPMemDevice{
					DeviceNumber: deviceNumber,
					MountPath:    uvmPath,
				},
			}
		}

		if err := uvm.Modify(modification); err != nil {
			uvm.vpmemDevices[deviceNumber] = vpmemInfo{}
			return 0, "", fmt.Errorf("uvm::AddVPMEM: failed to modify utility VM configuration: %s", err)
		}

		uvm.vpmemDevices[deviceNumber] = vpmemInfo{
			hostPath: hostPath,
			refCount: 1,
			uvmPath:  uvmPath}
	} else {
		pmemi := vpmemInfo{
			hostPath: hostPath,
			refCount: uvm.vpmemDevices[deviceNumber].refCount + 1,
			uvmPath:  uvmPath}
		uvm.vpmemDevices[deviceNumber] = pmemi
	}
	logrus.Debugf("hcsshim::AddVPMEM id:%s Success %+v", uvm.id, uvm.vpmemDevices[deviceNumber])
	return deviceNumber, uvmPath, nil

}

// RemoveVPMEM removes a VPMEM disk from a utility VM. As an external API, it
// is "safe". Internal use can call removeVPMEM.
func (uvm *UtilityVM) RemoveVPMEM(hostPath string) error {
	if uvm.operatingSystem != "linux" {
		return errNotSupported
	}

	uvm.m.Lock()
	defer uvm.m.Unlock()

	// Make sure is actually attached
	deviceNumber, uvmPath, err := uvm.findVPMEMDevice(hostPath)
	if err != nil {
		return fmt.Errorf("cannot remove VPMEM %s as it is not attached to utility VM %s: %s", hostPath, uvm.id, err)
	}

	if err := uvm.removeVPMEM(hostPath, uvmPath, deviceNumber); err != nil {
		return fmt.Errorf("failed to remove VPMEM %s from utility VM %s: %s", hostPath, uvm.id, err)
	}
	return nil
}

// removeVPMEM is the internally callable "unsafe" version of RemoveVPMEM. The mutex
// MUST be held when calling this function.
func (uvm *UtilityVM) removeVPMEM(hostPath string, uvmPath string, deviceNumber uint32) error {
	logrus.Debugf("uvm::RemoveVPMEM id:%s hostPath:%s device:%d", uvm.id, hostPath, deviceNumber)

	if uvm.vpmemDevices[deviceNumber].refCount == 1 {
		modification := &hcsschema.ModifySettingRequest{
			RequestType:  requesttype.Remove,
			ResourcePath: fmt.Sprintf("VirtualMachine/Devices/VirtualPMem"),
			GuestRequest: guestrequest.GuestRequest{
				ResourceType: guestrequest.ResourceTypeVPMemDevice,
				RequestType:  requesttype.Remove,
				Settings: guestrequest.LCOWMappedVPMemDevice{
					DeviceNumber: deviceNumber,
					MountPath:    uvmPath,
				},
			},
		}

		if err := uvm.Modify(modification); err != nil {
			return err
		}
		uvm.vpmemDevices[deviceNumber] = vpmemInfo{}
		logrus.Debugf("uvm::RemoveVPMEM: Success id:%s hostPath:%s device:%d", uvm.id, hostPath, deviceNumber)
		return nil
	}
	uvm.vpmemDevices[deviceNumber].refCount--
	logrus.Debugf("uvm::RemoveVPMEM: Success id:%s hostPath:%s device:%d refCount:%d", uvm.id, hostPath, deviceNumber, uvm.vpmemDevices[deviceNumber].refCount)
	return nil

}
<|endoftext|>


<commit_before>package rncryptor

import (
	"bytes"
	"crypto/aes"
	"crypto/cipher"
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"hash"
)

func Encrypt(data, password []byte, v int) string {

	version := byte(3)
	options := byte(1)
	encryptionSalt, err := RandomBytes(8)
	hmacSalt, err := RandomBytes(8)
	iv, err := RandomBytes(16)
	cipherText := Pad(data)

	hmacKey := Key(password, hmacSalt, 10000, 32, sha1.New)
	cipherKey := Key(password, encryptionSalt, 10000, 32, sha1.New)

	block, err := aes.NewCipher(cipherKey)
	if err != nil {
		panic(err)
	}

	cbc := cipher.NewCBCEncrypter(block, iv)
	cbc.CryptBlocks(cipherText, cipherText)

	buf := bytes.NewBuffer([]byte{version, options})
	buf.Write(encryptionSalt)
	buf.Write(hmacSalt)
	buf.Write(iv)
	buf.Write(cipherText)

	mac := hmac.New(sha256.New, hmacKey)
	mac.Write(buf.Bytes())
	expectedMAC := mac.Sum(nil)

	buf.Write(expectedMAC)

	result := base64.StdEncoding.EncodeToString(buf.Bytes())

	return result
}

// Pad applies the PKCS #7 padding scheme on the buffer.
func Pad(in []byte) []byte {
	padding := 16 - (len(in) % 16)
	if padding == 0 {
		padding = 16
	}
	for i := 0; i < padding; i++ {
		in = append(in, byte(padding))
	}
	return in
}

func RandomBytes(length int) ([]byte, error) {
	b := make([]byte, length)
	_, err := rand.Read(b)
	if err != nil {
		return nil, err
	}
	return b, nil
}

func Key(password, salt []byte, iter, keyLen int, h func() hash.Hash) []byte {
	prf := hmac.New(h, password)
	hashLen := prf.Size()
	numBlocks := (keyLen + hashLen - 1) / hashLen

	var buf [4]byte
	dk := make([]byte, 0, numBlocks*hashLen)
	U := make([]byte, hashLen)
	for block := 1; block <= numBlocks; block++ {
		// N.B.: || means concatenation, ^ means XOR
		// for each block T_i = U_1 ^ U_2 ^ ... ^ U_iter
		// U_1 = PRF(password, salt || uint(i))
		prf.Reset()
		prf.Write(salt)
		buf[0] = byte(block >> 24)
		buf[1] = byte(block >> 16)
		buf[2] = byte(block >> 8)
		buf[3] = byte(block)
		prf.Write(buf[:4])
		dk = prf.Sum(dk)
		T := dk[len(dk)-hashLen:]
		copy(U, T)

		// U_n = PRF(password, U_(n-1))
		for n := 2; n <= iter; n++ {
			prf.Reset()
			prf.Write(U)
			U = U[:0]
			U = prf.Sum(U)
			for x := range U {
				T[x] ^= U[x]
			}
		}
	}
	return dk[:keyLen]
}
<commit_msg>Remove fmt<commit_after>package rncryptor

import (
	"bytes"
	"crypto/aes"
	"crypto/cipher"
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/base64"
	"hash"
)

func Encrypt(data, password []byte, v int) string {

	version := byte(3)
	options := byte(1)
	encryptionSalt, err := RandomBytes(8)
	hmacSalt, err := RandomBytes(8)
	iv, err := RandomBytes(16)
	cipherText := Pad(data)

	hmacKey := Key(password, hmacSalt, 10000, 32, sha1.New)
	cipherKey := Key(password, encryptionSalt, 10000, 32, sha1.New)

	block, err := aes.NewCipher(cipherKey)
	if err != nil {
		panic(err)
	}

	cbc := cipher.NewCBCEncrypter(block, iv)
	cbc.CryptBlocks(cipherText, cipherText)

	buf := bytes.NewBuffer([]byte{version, options})
	buf.Write(encryptionSalt)
	buf.Write(hmacSalt)
	buf.Write(iv)
	buf.Write(cipherText)

	mac := hmac.New(sha256.New, hmacKey)
	mac.Write(buf.Bytes())
	expectedMAC := mac.Sum(nil)

	buf.Write(expectedMAC)

	result := base64.StdEncoding.EncodeToString(buf.Bytes())

	return result
}

// Pad applies the PKCS #7 padding scheme on the buffer.
func Pad(in []byte) []byte {
	padding := 16 - (len(in) % 16)
	if padding == 0 {
		padding = 16
	}
	for i := 0; i < padding; i++ {
		in = append(in, byte(padding))
	}
	return in
}

func RandomBytes(length int) ([]byte, error) {
	b := make([]byte, length)
	_, err := rand.Read(b)
	if err != nil {
		return nil, err
	}
	return b, nil
}

func Key(password, salt []byte, iter, keyLen int, h func() hash.Hash) []byte {
	prf := hmac.New(h, password)
	hashLen := prf.Size()
	numBlocks := (keyLen + hashLen - 1) / hashLen

	var buf [4]byte
	dk := make([]byte, 0, numBlocks*hashLen)
	U := make([]byte, hashLen)
	for block := 1; block <= numBlocks; block++ {
		// N.B.: || means concatenation, ^ means XOR
		// for each block T_i = U_1 ^ U_2 ^ ... ^ U_iter
		// U_1 = PRF(password, salt || uint(i))
		prf.Reset()
		prf.Write(salt)
		buf[0] = byte(block >> 24)
		buf[1] = byte(block >> 16)
		buf[2] = byte(block >> 8)
		buf[3] = byte(block)
		prf.Write(buf[:4])
		dk = prf.Sum(dk)
		T := dk[len(dk)-hashLen:]
		copy(U, T)

		// U_n = PRF(password, U_(n-1))
		for n := 2; n <= iter; n++ {
			prf.Reset()
			prf.Write(U)
			U = U[:0]
			U = prf.Sum(U)
			for x := range U {
				T[x] ^= U[x]
			}
		}
	}
	return dk[:keyLen]
}
<|endoftext|>


<commit_before>package passivation

import (
	"github.com/Sirupsen/logrus"
	"github.com/arkenio/goarken/model"
	"time"
)

var log = logrus.New()

type PassivationHandler struct {
	arkenModel *model.Model
	Stop       chan interface{}
}

func NewHandler(model *model.Model) *PassivationHandler {
	return &PassivationHandler{
		arkenModel: model,
		Stop:       make(chan interface{}),
	}
}

func (p *PassivationHandler) Start() {
	ticker := time.NewTicker(time.Minute)
	updateChannel := p.arkenModel.Listen()

	for {
		select {
		case <-p.Stop:
			return
		case <-ticker.C:
			// Check every minute which service has to be passivated
			for _, serviceCluster := range p.arkenModel.Services {
				p.passivateServiceIfNeeded(serviceCluster)
			}
		case event := <-updateChannel:
			// When a service changes, check if it has to be started
			if sc, ok := event.Model.(*model.ServiceCluster); ok {
				for _, service := range p.arkenModel.Services[sc.Name].Instances {
					p.restartIfNeeded(service)
				}
			}
		}
	}
}

func (p *PassivationHandler) passivateServiceIfNeeded(serviceCluster *model.ServiceCluster) {

	service, err := serviceCluster.Next()
	if err != nil {
		//No active instance, no need to passivate
		return
	}

	// Checking if the service should be passivated or not
	if p.hasToBePassivated(service) {
		log.Infof("Service %s enters passivation", service.Name)
		if "destroy" == service.Config.Passivation.Action {
			p.arkenModel.DestroyService(service)
		} else if "stop" == service.Config.Passivation.Action {
			p.arkenModel.StopService(service)
		} else {
			// By default passivate
			p.arkenModel.PassivateService(service)
		}

	}

}

func (p *PassivationHandler) hasToBePassivated(service *model.Service) bool {

	config := service.Config.Passivation
	if config.Enabled {
		passiveLimitDuration := time.Duration(config.DelayInSeconds) * time.Second

		return service.StartedSince() != nil &&
			time.Now().After(service.StartedSince().Add(passiveLimitDuration))
	}
	return false
}

func (p *PassivationHandler) restartIfNeeded(service *model.Service) {

	if p.hasToBeRestarted(service) {
		service, err := p.arkenModel.StartService(service)
		if err != nil {
			log.Errorf("Service "+service.Name+" restart has failed: %s", err)
			return
		}
		log.Infof("Service %s restarted", service.Name)
	}
}

func (p *PassivationHandler) hasToBeRestarted(service *model.Service) bool {
	return service.Config.Passivation.Enabled &&
		service.LastAccess != nil &&
		service.Status != nil &&
		service.Status.Expected == model.STARTED_STATUS &&
		service.Status.Current == model.PASSIVATED_STATUS

}
<commit_msg>NXIO-529 prevent npe when cluster is destroyed<commit_after>package passivation

import (
	"github.com/Sirupsen/logrus"
	"github.com/arkenio/goarken/model"
	"time"
)

var log = logrus.New()

type PassivationHandler struct {
	arkenModel *model.Model
	Stop       chan interface{}
}

func NewHandler(model *model.Model) *PassivationHandler {
	return &PassivationHandler{
		arkenModel: model,
		Stop:       make(chan interface{}),
	}
}

func (p *PassivationHandler) Start() {
	ticker := time.NewTicker(time.Minute)
	updateChannel := p.arkenModel.Listen()

	for {
		select {
		case <-p.Stop:
			return
		case <-ticker.C:
			// Check every minute which service has to be passivated
			for _, serviceCluster := range p.arkenModel.Services {
				p.passivateServiceIfNeeded(serviceCluster)
			}
		case event := <-updateChannel:
			// When a service changes, check if it has to be started

			if sc, ok := event.Model.(*model.ServiceCluster); ok {
				cluster := p.arkenModel.Services[sc.Name]
				//Cluster may be nil if event was a delete
				if cluster != nil {
					for _, service := range p.arkenModel.Services[sc.Name].Instances {
						p.restartIfNeeded(service)
					}
				}
			}
		}
	}
}

func (p *PassivationHandler) passivateServiceIfNeeded(serviceCluster *model.ServiceCluster) {

	service, err := serviceCluster.Next()
	if err != nil {
		//No active instance, no need to passivate
		return
	}

	// Checking if the service should be passivated or not
	if p.hasToBePassivated(service) {
		log.Infof("Service %s enters passivation", service.Name)
		if "destroy" == service.Config.Passivation.Action {
			p.arkenModel.DestroyService(service)
		} else if "stop" == service.Config.Passivation.Action {
			p.arkenModel.StopService(service)
		} else {
			// By default passivate
			p.arkenModel.PassivateService(service)
		}

	}

}

func (p *PassivationHandler) hasToBePassivated(service *model.Service) bool {

	config := service.Config.Passivation
	if config.Enabled {
		passiveLimitDuration := time.Duration(config.DelayInSeconds) * time.Second

		return service.StartedSince() != nil &&
			time.Now().After(service.StartedSince().Add(passiveLimitDuration))
	}
	return false
}

func (p *PassivationHandler) restartIfNeeded(service *model.Service) {

	if p.hasToBeRestarted(service) {
		service, err := p.arkenModel.StartService(service)
		if err != nil {
			log.Errorf("Service "+service.Name+" restart has failed: %s", err)
			return
		}
		log.Infof("Service %s restarted", service.Name)
	}
}

func (p *PassivationHandler) hasToBeRestarted(service *model.Service) bool {
	return service.Config.Passivation.Enabled &&
		service.LastAccess != nil &&
		service.Status != nil &&
		service.Status.Expected == model.STARTED_STATUS &&
		service.Status.Current == model.PASSIVATED_STATUS

}
<|endoftext|>


<commit_before>package env

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/pkg/errors"
	"gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
)

type Git struct {
	Branch      string `json:"branch" structs:"branch"`
	CommitSHA   string `json:"commit_sha" structs:"commit_sha"`
	CommittedAt int    `json:"committed_at" structs:"committed_at"`
}

func (g Git) String() string {
	out := &bytes.Buffer{}
	out.WriteString("GIT_BRANCH=")
	out.WriteString(g.Branch)
	out.WriteString("\nGIT_COMMIT_SHA=")
	out.WriteString(g.CommitSHA)
	out.WriteString("\nGIT_COMMITTED_AT=")
	out.WriteString(fmt.Sprint(g.CommittedAt))
	return out.String()
}

func GetHead() (*object.Commit, error) {
	r, err := git.PlainOpen(".")
	if err != nil {
		return nil, errors.WithStack(err)
	}

	ref, err := r.Head()
	if err != nil {
		return nil, errors.WithStack(err)
	}

	commit, err := r.CommitObject(ref.Hash())
	if err != nil {
		return nil, errors.WithStack(err)
	}

	return commit, nil
}

func findGitInfo() (Git, error) {
	_, err := exec.LookPath("git")
	if err != nil {
		// git isn't present, so load from ENV vars:
		return loadGitFromENV()
	}

	g := Git{}

	cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD")
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		return g, errors.WithStack(err)
	}
	g.Branch = strings.TrimSpace(string(out))

	cmd = exec.Command("git", "log", "-1", "--pretty=format:%H")
	cmd.Stderr = os.Stderr
	out, err = cmd.Output()
	if err != nil {
		return g, errors.WithStack(err)
	}
	g.CommitSHA = strings.TrimSpace(string(out))

	cmd = exec.Command("git", "log", "-1", "--pretty=format:%ct")
	cmd.Stderr = os.Stderr
	out, err = cmd.Output()
	if err != nil {
		return g, errors.WithStack(err)
	}
	g.CommittedAt, err = strconv.Atoi(strings.TrimSpace(string(out)))
	if err != nil {
		return g, errors.WithStack(err)
	}
	return g, nil
}

func GitSHA(path string) (string, error) {
	args := []string{"log", "-1", "--follow", "--pretty=format:%H"}
	if path != "" {
		if pwd, err := os.Getwd(); err == nil {
			path = strings.TrimPrefix(path, pwd)
			path = filepath.Join(".", path)
		}
		args = append(args, path)
	}
	cmd := exec.Command("git", args...)
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		return "", errors.WithStack(err)
	}
	return strings.TrimSpace(string(out)), nil
}

var GitBlob = func(path string, commit *object.Commit) (string, error) {
	if commit == nil {
		blob, err := fallbackBlob(path)

		if err != nil {
			return "", errors.WithStack(err)
		}

		return blob, nil
	}

	file, err := commit.File(path)

	if err != nil {
		return "", errors.WithStack(err)
	}

	blob := strings.TrimSpace(file.Hash.String())

	return blob, nil
}

func fallbackBlob(path string) (string, error) {
	file, err := ioutil.ReadFile(path)

	if err != nil {
		return "", errors.WithStack(err)
	}

	hash := plumbing.ComputeHash(plumbing.BlobObject, []byte(file))
	res := hash.String()
	return res, nil
}

func loadGitFromENV() (Git, error) {
	g := Git{
		Branch:    findVar(gitBranchVars),
		CommitSHA: findVar(gitCommitShaVars),
	}
	var err error
	g.CommittedAt, err = strconv.Atoi(findVar(gitCommittedAtVars))
	return g, err
}

var gitBranchVars = []string{"GIT_BRANCH", "APPVEYOR_REPO_BRANCH", "BRANCH_NAME", "BUILDKITE_BRANCH", "CIRCLE_BRANCH", "CI_BRANCH", "CI_BUILD_REF_NAME", "TRAVIS_BRANCH", "WERCKER_GIT_BRANCH"}

var gitCommitShaVars = []string{"GIT_COMMIT_SHA", "APPVEYOR_REPO_COMMIT", "BUILDKITE_COMMIT", "CIRCLE_SHA1", "CI_BUILD_REF", "CI_BUILD_SHA", "CI_COMMIT", "CI_COMMIT_ID", "GIT_COMMIT", "WERCKER_GIT_COMMIT"}

var gitCommittedAtVars = []string{"GIT_COMMITTED_AT", "GIT_COMMITED_AT", "CI_COMMITTED_AT", "CI_COMMITED_AT"}

var blobRegex = regexp.MustCompile(`^\d.+\s+blob\s(\w+)`)
<commit_msg>Add debug logs to git commands<commit_after>package env

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/pkg/errors"
	"gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
)

type Git struct {
	Branch      string `json:"branch" structs:"branch"`
	CommitSHA   string `json:"commit_sha" structs:"commit_sha"`
	CommittedAt int    `json:"committed_at" structs:"committed_at"`
}

func (g Git) String() string {
	out := &bytes.Buffer{}
	out.WriteString("GIT_BRANCH=")
	out.WriteString(g.Branch)
	out.WriteString("\nGIT_COMMIT_SHA=")
	out.WriteString(g.CommitSHA)
	out.WriteString("\nGIT_COMMITTED_AT=")
	out.WriteString(fmt.Sprint(g.CommittedAt))
	return out.String()
}

func GetHead() (*object.Commit, error) {
	r, err := git.PlainOpen(".")
	if err != nil {
		return nil, errors.WithStack(err)
	}

	ref, err := r.Head()
	if err != nil {
		return nil, errors.WithStack(err)
	}

	commit, err := r.CommitObject(ref.Hash())
	if err != nil {
		return nil, errors.WithStack(err)
	}

	return commit, nil
}

func findGitInfo() (Git, error) {
	_, err := exec.LookPath("git")
	if err != nil {
		// git isn't present, so load from ENV vars:
		logrus.Debug("Loading GIT info from ENV")
		return loadGitFromENV()
	}

	g := Git{}

	cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD")
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		return g, errors.WithStack(err)
	}
	g.Branch = strings.TrimSpace(string(out))

	cmd = exec.Command("git", "log", "-1", "--pretty=format:%H")
	cmd.Stderr = os.Stderr
	out, err = cmd.Output()
	if err != nil {
		return g, errors.WithStack(err)
	}
	g.CommitSHA = strings.TrimSpace(string(out))

	cmd = exec.Command("git", "log", "-1", "--pretty=format:%ct")
	cmd.Stderr = os.Stderr
	out, err = cmd.Output()
	if err != nil {
		return g, errors.WithStack(err)
	}
	g.CommittedAt, err = strconv.Atoi(strings.TrimSpace(string(out)))
	if err != nil {
		return g, errors.WithStack(err)
	}
	return g, nil
}

func GitSHA(path string) (string, error) {
	args := []string{"log", "-1", "--follow", "--pretty=format:%H"}
	if path != "" {
		if pwd, err := os.Getwd(); err == nil {
			path = strings.TrimPrefix(path, pwd)
			path = filepath.Join(".", path)
		}
		args = append(args, path)
	}
	cmd := exec.Command("git", args...)
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		return "", errors.WithStack(err)
	}
	return strings.TrimSpace(string(out)), nil
}

var GitBlob = func(path string, commit *object.Commit) (string, error) {
	if commit == nil {
		blob, err := fallbackBlob(path)

		if err != nil {
			return "", errors.WithStack(err)
		}

		return blob, nil
	}

	logrus.Debugf("getting git blob_id for source file %s", path)
	file, err := commit.File(path)

	if err != nil {
		logrus.Errorf("failed to find file %s\n%s", path, err)
		return "", errors.WithStack(err)
	}

	blob := strings.TrimSpace(file.Hash.String())

	return blob, nil
}

func fallbackBlob(path string) (string, error) {
	logrus.Debugf("getting fallback blob_id for source file %s", path)
	file, err := ioutil.ReadFile(path)

	if err != nil {
		logrus.Errorf("failed to read file %s\n%s", path, err)
		return "", errors.WithStack(err)
	}

	hash := plumbing.ComputeHash(plumbing.BlobObject, []byte(file))
	res := hash.String()
	return res, nil
}

func loadGitFromENV() (Git, error) {
	g := Git{
		Branch:    findVar(gitBranchVars),
		CommitSHA: findVar(gitCommitShaVars),
	}
	var err error
	g.CommittedAt, err = strconv.Atoi(findVar(gitCommittedAtVars))
	return g, err
}

var gitBranchVars = []string{"GIT_BRANCH", "APPVEYOR_REPO_BRANCH", "BRANCH_NAME", "BUILDKITE_BRANCH", "CIRCLE_BRANCH", "CI_BRANCH", "CI_BUILD_REF_NAME", "TRAVIS_BRANCH", "WERCKER_GIT_BRANCH"}

var gitCommitShaVars = []string{"GIT_COMMIT_SHA", "APPVEYOR_REPO_COMMIT", "BUILDKITE_COMMIT", "CIRCLE_SHA1", "CI_BUILD_REF", "CI_BUILD_SHA", "CI_COMMIT", "CI_COMMIT_ID", "GIT_COMMIT", "WERCKER_GIT_COMMIT"}

var gitCommittedAtVars = []string{"GIT_COMMITTED_AT", "GIT_COMMITED_AT", "CI_COMMITTED_AT", "CI_COMMITED_AT"}

var blobRegex = regexp.MustCompile(`^\d.+\s+blob\s(\w+)`)
<|endoftext|>