<commit_before>package jwt

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"net/http"
	"strings"
	"time"
)

// Parse methods use this callback function to supply
// the key for verification.  The function receives the parsed,
// but unverified Token.  This allows you to use propries in the
// Header of the token (such as `kid`) to identify which key to use.
type Keyfunc func(*Token) ([]byte, error)

// A JWT Token
type Token struct {
	Header map[string]interface{}
	Claims map[string]interface{}
	Method SigningMethod
	// This is only populated when you Parse a token
	Signature string
	// This is only populated when you Parse/Verify a token
	Valid bool
}

func New(method SigningMethod) *Token {
	return &Token{
		Header: map[string]interface{}{
			"typ": "JWT",
			"alg": method.Alg(),
		},
		Claims: make(map[string]interface{}),
		Method: method,
	}
}

// Get the complete, signed token
func (t *Token) SignedString(key []byte) (string, error) {
	var sig, sstr string
	var err error
	if sstr, err = t.SigningString(); err != nil {
		return "", err
	}
	if sig, err = t.Method.Sign(sstr, key); err != nil {
		return "", err
	}
	return strings.Join([]string{sstr, sig}, "."), nil
}

// Generate the signing string.  This is the
// most expensive part of the whole deal.  Unless you
// need this for something special, just go straight for
// the SignedString.
func (t *Token) SigningString() (string, error) {
	var err error
	parts := make([]string, 2)
	for i, _ := range parts {
		var source map[string]interface{}
		if i == 0 {
			source = t.Header
		} else {
			source = t.Claims
		}

		var jsonValue []byte
		if jsonValue, err = json.Marshal(source); err != nil {
			return "", err
		}

		parts[i] = EncodeSegment(jsonValue)
	}
	return strings.Join(parts, "."), nil
}

// Parse, validate, and return a token.
// keyFunc will receive the parsed token and should return the key for validating.
// If everything is kosher, err will be nil
func Parse(tokenString string, keyFunc Keyfunc) (token *Token, err error) {
	parts := strings.Split(tokenString, ".")
	if len(parts) == 3 {
		token = new(Token)
		// parse Header
		var headerBytes []byte
		if headerBytes, err = DecodeSegment(parts[0]); err != nil {
			return
		}
		if err = json.Unmarshal(headerBytes, &token.Header); err != nil {
			return
		}

		// parse Claims
		var claimBytes []byte
		if claimBytes, err = DecodeSegment(parts[1]); err != nil {
			return
		}
		if err = json.Unmarshal(claimBytes, &token.Claims); err != nil {
			return
		}

		// Lookup signature method
		if method, ok := token.Header["alg"].(string); ok {
			if token.Method = GetSigningMethod(method); token.Method == nil {
				err = errors.New("Signing method (alg) is unavailable.")
				return
			}
		} else {
			err = errors.New("Signing method (alg) is unspecified.")
			return
		}

		// Check expiry times
		if exp, ok := token.Claims["exp"].(float64); ok {
			if time.Now().Unix() > int64(exp) {
				err = errors.New("Token is expired")
			}
		}

		// Lookup key
		var key []byte
		if key, err = keyFunc(token); err != nil {
			return
		}

		// Perform validation
		if err = token.Method.Verify(strings.Join(parts[0:2], "."), parts[2], key); err == nil {
			token.Valid = true
		}

	} else {
		err = errors.New("Token contains an invalid number of segments")
	}
	return
}

// Try to find the token in an http.Request.
// This method will call ParseMultipartForm if there's no token in the header.
// Currently, it looks in the Authorization header as well as
// looking for an 'access_token' request parameter in req.Form.
func ParseFromRequest(req *http.Request, keyFunc Keyfunc) (token *Token, err error) {

	// Look for an Authorization header
	if ah := req.Header.Get("Authorization"); ah != "" {
		// Should be a bearer token
		if len(ah) > 6 && strings.ToUpper(ah[0:6]) == "BEARER" {
			return Parse(ah[7:], keyFunc)
		}
	}

	// Look for "access_token" parameter
	req.ParseMultipartForm(10e6)
	if tokStr := req.Form.Get("access_token"); tokStr != "" {
		return Parse(tokStr, keyFunc)
	}

	return nil, errors.New("No token present in request.")

}

// Encode JWT specific base64url encoding with padding stripped
func EncodeSegment(seg []byte) string {
	return strings.TrimRight(base64.URLEncoding.EncodeToString(seg), "=")
}

// Decode JWT specific base64url encoding with padding stripped
func DecodeSegment(seg string) ([]byte, error) {
	// len % 4
	switch len(seg) % 4 {
	case 2:
		seg = seg + "=="
	case 3:
		seg = seg + "==="
	}

	return base64.URLEncoding.DecodeString(seg)
}
<commit_msg>capture raw string when parsing a token<commit_after>package jwt

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"net/http"
	"strings"
	"time"
)

// Parse methods use this callback function to supply
// the key for verification.  The function receives the parsed,
// but unverified Token.  This allows you to use propries in the
// Header of the token (such as `kid`) to identify which key to use.
type Keyfunc func(*Token) ([]byte, error)

// A JWT Token
type Token struct {
	Raw    string
	Header map[string]interface{}
	Claims map[string]interface{}
	Method SigningMethod
	// This is only populated when you Parse a token
	Signature string
	// This is only populated when you Parse/Verify a token
	Valid bool
}

func New(method SigningMethod) *Token {
	return &Token{
		Header: map[string]interface{}{
			"typ": "JWT",
			"alg": method.Alg(),
		},
		Claims: make(map[string]interface{}),
		Method: method,
	}
}

// Get the complete, signed token
func (t *Token) SignedString(key []byte) (string, error) {
	var sig, sstr string
	var err error
	if sstr, err = t.SigningString(); err != nil {
		return "", err
	}
	if sig, err = t.Method.Sign(sstr, key); err != nil {
		return "", err
	}
	return strings.Join([]string{sstr, sig}, "."), nil
}

// Generate the signing string.  This is the
// most expensive part of the whole deal.  Unless you
// need this for something special, just go straight for
// the SignedString.
func (t *Token) SigningString() (string, error) {
	var err error
	parts := make([]string, 2)
	for i, _ := range parts {
		var source map[string]interface{}
		if i == 0 {
			source = t.Header
		} else {
			source = t.Claims
		}

		var jsonValue []byte
		if jsonValue, err = json.Marshal(source); err != nil {
			return "", err
		}

		parts[i] = EncodeSegment(jsonValue)
	}
	return strings.Join(parts, "."), nil
}

// Parse, validate, and return a token.
// keyFunc will receive the parsed token and should return the key for validating.
// If everything is kosher, err will be nil
func Parse(tokenString string, keyFunc Keyfunc) (token *Token, err error) {
	parts := strings.Split(tokenString, ".")
	if len(parts) == 3 {
		token = &Token{Raw: tokenString}
		// parse Header
		var headerBytes []byte
		if headerBytes, err = DecodeSegment(parts[0]); err != nil {
			return
		}
		if err = json.Unmarshal(headerBytes, &token.Header); err != nil {
			return
		}

		// parse Claims
		var claimBytes []byte
		if claimBytes, err = DecodeSegment(parts[1]); err != nil {
			return
		}
		if err = json.Unmarshal(claimBytes, &token.Claims); err != nil {
			return
		}

		// Lookup signature method
		if method, ok := token.Header["alg"].(string); ok {
			if token.Method = GetSigningMethod(method); token.Method == nil {
				err = errors.New("Signing method (alg) is unavailable.")
				return
			}
		} else {
			err = errors.New("Signing method (alg) is unspecified.")
			return
		}

		// Check expiry times
		if exp, ok := token.Claims["exp"].(float64); ok {
			if time.Now().Unix() > int64(exp) {
				err = errors.New("Token is expired")
			}
		}

		// Lookup key
		var key []byte
		if key, err = keyFunc(token); err != nil {
			return
		}

		// Perform validation
		if err = token.Method.Verify(strings.Join(parts[0:2], "."), parts[2], key); err == nil {
			token.Valid = true
		}

	} else {
		err = errors.New("Token contains an invalid number of segments")
	}
	return
}

// Try to find the token in an http.Request.
// This method will call ParseMultipartForm if there's no token in the header.
// Currently, it looks in the Authorization header as well as
// looking for an 'access_token' request parameter in req.Form.
func ParseFromRequest(req *http.Request, keyFunc Keyfunc) (token *Token, err error) {

	// Look for an Authorization header
	if ah := req.Header.Get("Authorization"); ah != "" {
		// Should be a bearer token
		if len(ah) > 6 && strings.ToUpper(ah[0:6]) == "BEARER" {
			return Parse(ah[7:], keyFunc)
		}
	}

	// Look for "access_token" parameter
	req.ParseMultipartForm(10e6)
	if tokStr := req.Form.Get("access_token"); tokStr != "" {
		return Parse(tokStr, keyFunc)
	}

	return nil, errors.New("No token present in request.")

}

// Encode JWT specific base64url encoding with padding stripped
func EncodeSegment(seg []byte) string {
	return strings.TrimRight(base64.URLEncoding.EncodeToString(seg), "=")
}

// Decode JWT specific base64url encoding with padding stripped
func DecodeSegment(seg string) ([]byte, error) {
	// len % 4
	switch len(seg) % 4 {
	case 2:
		seg = seg + "=="
	case 3:
		seg = seg + "==="
	}

	return base64.URLEncoding.DecodeString(seg)
}
<|endoftext|>


<commit_before>package fernet

import (
	"bytes"
	"crypto/aes"
	"crypto/rand"
	"errors"
	"io"
	"strconv"
	"time"
)

type Key [32]byte

// Decodes base-64 data from s into k.
func DecodeKey(s string) (*Key, error) {
	var k Key
	var b [(len(k) + 2) / 3 * 3]byte
	if encoding.DecodedLen(len(s)) != len(b) {
		return nil, keyLenError(len(s))
	}
	n, err := encoding.Decode(b[:], []byte(s))
	if err != nil {
		return nil, err
	}
	if n != len(k) {
		return nil, keyLenError(len(s))
	}
	copy(k[:], b[:])
	return &k, nil
}

// MustDecodeKey is like DecodeKey, but panics if an error occurs.
func MustDecodeKey(s string) *Key {
	k, err := DecodeKey(s)
	if err != nil {
		panic(err)
	}
	return k
}

// Generates a random key using crypto/rand.
func GenKey() (*Key, error) {
	var k Key
	if _, err := io.ReadFull(rand.Reader, k[:]); err != nil {
		return nil, err
	}
	return &k, nil
}

func (k *Key) cryptBytes() []byte {
	return k[len(k)/2:]
}

func (k *Key) signBytes() []byte {
	return k[:len(k)/2]
}

// Returns the base-64 encoding of k.
func (k *Key) Encode() string {
	b := make([]byte, encoding.EncodedLen(len(k)))
	encoding.Encode(b, k[:])
	return string(b)
}

// Generates an encrypted fernet token containing msg as its message.
func (k *Key) Generate(msg []byte) (tok []byte, err error) {
	if *k == (Key{}) {
		return nil, errors.New("zero key")
	}
	iv := make([]byte, aes.BlockSize)
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		return nil, err
	}
	return gen(msg, iv, time.Now(), k), nil
}

// Verifies that tok is a valid fernet token that was signed at most
// ttl time ago, and returns the plaintext message contained in it.
// Returns nil if tok is invalid.
func (k *Key) Verify(tok []byte, ttl time.Duration) (msg []byte) {
	if !bytes.Contains(tok, pipe) {
		return jsonVerify(tok, ttl, time.Now(), k)
	}
	return verify(tok, ttl, time.Now(), k)
}

type keyLenError int

func (n keyLenError) Error() string {
	return "fernet: key decodes to " + strconv.Itoa(int(n)) + " bytes"
}
<commit_msg>fix doc<commit_after>package fernet

import (
	"bytes"
	"crypto/aes"
	"crypto/rand"
	"errors"
	"io"
	"strconv"
	"time"
)

type Key [32]byte

// Decodes a base64-encoded key from s and returns it.
func DecodeKey(s string) (*Key, error) {
	var k Key
	var b [(len(k) + 2) / 3 * 3]byte
	if encoding.DecodedLen(len(s)) != len(b) {
		return nil, keyLenError(len(s))
	}
	n, err := encoding.Decode(b[:], []byte(s))
	if err != nil {
		return nil, err
	}
	if n != len(k) {
		return nil, keyLenError(len(s))
	}
	copy(k[:], b[:])
	return &k, nil
}

// MustDecodeKey is like DecodeKey, but panics if an error occurs.
func MustDecodeKey(s string) *Key {
	k, err := DecodeKey(s)
	if err != nil {
		panic(err)
	}
	return k
}

// Generates a random key using crypto/rand.
func GenKey() (*Key, error) {
	var k Key
	if _, err := io.ReadFull(rand.Reader, k[:]); err != nil {
		return nil, err
	}
	return &k, nil
}

func (k *Key) cryptBytes() []byte {
	return k[len(k)/2:]
}

func (k *Key) signBytes() []byte {
	return k[:len(k)/2]
}

// Returns the base64 encoding of k.
func (k *Key) Encode() string {
	b := make([]byte, encoding.EncodedLen(len(k)))
	encoding.Encode(b, k[:])
	return string(b)
}

// Generates an encrypted fernet token containing msg as its message.
func (k *Key) Generate(msg []byte) (tok []byte, err error) {
	if *k == (Key{}) {
		return nil, errors.New("zero key")
	}
	iv := make([]byte, aes.BlockSize)
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		return nil, err
	}
	return gen(msg, iv, time.Now(), k), nil
}

// Verifies that tok is a valid fernet token that was signed at most
// ttl time ago, and returns the plaintext message contained in it.
// Returns nil if tok is invalid.
func (k *Key) Verify(tok []byte, ttl time.Duration) (msg []byte) {
	if !bytes.Contains(tok, pipe) {
		return jsonVerify(tok, ttl, time.Now(), k)
	}
	return verify(tok, ttl, time.Now(), k)
}

type keyLenError int

func (n keyLenError) Error() string {
	return "fernet: key decodes to " + strconv.Itoa(int(n)) + " bytes"
}
<|endoftext|>


<commit_before>package lm2

import (
	"bytes"
	"encoding/binary"
	"errors"
	"math/rand"
	"os"
	"sort"
)

const sentinelMagic = 0xDEAD10CC

const cacheSize = 100000

type Collection struct {
	fileHeader
	f     *os.File
	wal   *WAL
	cache *recordCache
}

type fileHeader struct {
	Head          int64
	LastCommit    int64 // Unused for now
	LastLogCommit int64 // Unused for now
}

func (h fileHeader) Bytes() []byte {
	buf := bytes.NewBuffer(nil)
	binary.Write(buf, binary.LittleEndian, h)
	return buf.Bytes()
}

type recordHeader struct {
	Next    int64
	Deleted int64
	KeyLen  uint16
	ValLen  uint32
}

func (h recordHeader) Bytes() []byte {
	buf := bytes.NewBuffer(nil)
	binary.Write(buf, binary.LittleEndian, h)
	return buf.Bytes()
}

type sentinelRecord struct {
	Magic  uint32 // some fixed pattern
	Offset int64  // this record's offset
}

type record struct {
	recordHeader
	Offset int64
	Key    string
	Value  string
}

type recordCache struct {
	cache        map[int64]*record
	dirty        map[int64]bool
	maxKeyRecord *record
	size         int
	preventPurge bool

	c *Collection
}

func newCache(size int) *recordCache {
	return &recordCache{
		cache:        map[int64]*record{},
		dirty:        map[int64]bool{},
		maxKeyRecord: nil,
		size:         size,
	}
}

func (rc *recordCache) findLastLessThan(key string) int64 {
	if rc.maxKeyRecord != nil {
		if rc.maxKeyRecord.Key < key {
			return rc.maxKeyRecord.Offset
		}
	}
	max := ""
	maxOffset := int64(0)

	for offset, record := range rc.cache {
		if record.Key >= key {
			continue
		}
		if record.Key > max {
			max = record.Key
			maxOffset = offset
		}
	}
	return maxOffset
}

func (rc *recordCache) push(rec *record) {
	if rc.maxKeyRecord == nil || rc.maxKeyRecord.Key < rec.Key {
		rc.maxKeyRecord = rec
	} else if rand.Float32() >= 0.04 {
		return
	}
	rc.cache[rec.Offset] = rec
	if len(rc.cache) > rc.size && !rc.preventPurge {
		deletedKey := int64(0)
		for k := range rc.cache {
			if k == rc.maxKeyRecord.Offset {
				continue
			}
			deletedKey = k
			return
		}
		if rc.dirty[deletedKey] {
			// This is a dirty record. Flush changes to disk.
			rc.c.updateRecordHeader(deletedKey, rc.cache[deletedKey].recordHeader)
			delete(rc.dirty, deletedKey)
		}
		delete(rc.cache, deletedKey)
	}
}

func (rc *recordCache) forcePush(rec *record) {
	if rc.maxKeyRecord == nil || rc.maxKeyRecord.Key < rec.Key {
		rc.maxKeyRecord = rec
	}
	rc.cache[rec.Offset] = rec
}

func (c *Collection) readRecord(offset int64) (*record, error) {
	if offset == 0 {
		return nil, errors.New("lm2: invalid record offset 0")
	}

	if rec := c.cache.cache[offset]; rec != nil {
		return rec, nil
	}

	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return nil, err
	}

	header := recordHeader{}
	err = binary.Read(c.f, binary.LittleEndian, &header)
	if err != nil {
		return nil, err
	}

	keyValBuf := make([]byte, int(header.KeyLen)+int(header.ValLen))
	_, err = c.f.Read(keyValBuf)
	if err != nil {
		return nil, err
	}

	key := string(keyValBuf[:int(header.KeyLen)])
	value := string(keyValBuf[int(header.KeyLen):])

	rec := &record{
		recordHeader: header,
		Offset:       offset,
		Key:          key,
		Value:        value,
	}

	c.cache.push(rec)

	return rec, nil
}

func (c *Collection) setRecordNext(offset int64, next int64) error {
	if rec := c.cache.cache[offset]; rec != nil {
		rec.recordHeader.Next = next
		c.cache.dirty[offset] = true
		return nil
	}
	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return err
	}
	return binary.Write(c.f, binary.LittleEndian, next)
}

func (c *Collection) nextRecord(rec *record) *record {
	if rec == nil {
		return nil
	}
	nextRec, err := c.readRecord(rec.Next)
	if err != nil {
		return nil
	}
	return nextRec
}

func (c *Collection) writeRecord(rec *record) (int64, error) {
	offset, err := c.f.Seek(0, 2)
	if err != nil {
		return 0, err
	}

	rec.KeyLen = uint16(len(rec.Key))
	rec.ValLen = uint32(len(rec.Value))

	err = binary.Write(c.f, binary.LittleEndian, rec.recordHeader)
	if err != nil {
		return 0, err
	}

	_, err = c.f.WriteString(rec.Key)
	if err != nil {
		return 0, err
	}
	_, err = c.f.WriteString(rec.Value)
	if err != nil {
		return 0, err
	}

	rec.Offset = offset
	return offset, nil
}

func (c *Collection) writeSentinel() (int64, error) {
	offset, err := c.f.Seek(0, 2)
	if err != nil {
		return 0, err
	}
	sentinel := sentinelRecord{
		Magic:  sentinelMagic,
		Offset: offset,
	}
	err = binary.Write(c.f, binary.LittleEndian, sentinel)
	if err != nil {
		return 0, err
	}
	return offset + 12, nil
}

func (c *Collection) updateRecordHeader(offset int64, header recordHeader) error {
	if rec := c.cache.cache[offset]; rec != nil {
		rec.recordHeader = header
		c.cache.dirty[offset] = true
		return nil
	}
	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return err
	}
	return binary.Write(c.f, binary.LittleEndian, header)
}

func (c *Collection) findLastLessThanOrEqual(key string) (int64, error) {
	offset := int64(0)

	if c.Head == 0 {
		// Empty collection.
		return 0, nil
	}

	// read the head
	rec, err := c.readRecord(c.Head)
	if err != nil {
		return 0, err
	}
	if rec.Key > key { // we have a new head
		return 0, nil
	}

	cacheResult := c.cache.findLastLessThan(key)
	if cacheResult != 0 {
		rec, err = c.readRecord(cacheResult)
		if err != nil {
			return 0, err
		}
	}

	offset = rec.Offset

	for rec != nil {
		if rec.Key > key {
			break
		}
		offset = rec.Offset
		rec = c.nextRecord(rec)
	}

	return offset, nil
}

// Update atomically and durably applies a WriteBatch (a set of updates) to the collection.
func (c *Collection) Update(wb *WriteBatch) error {
	// Clean up WriteBatch.
	wb.cleanup()

	// Find and load records that will be modified into the cache.
	recordsToLoad := []int64{}

	keys := []string{}
	for key := range wb.sets {
		keys = append(keys, key)
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		if offset > 0 {
			recordsToLoad = append(recordsToLoad, offset)
		}
	}

	// Prevent cache purges.
	c.cache.preventPurge = true
	defer func() { c.cache.preventPurge = false }()

	for _, offset := range recordsToLoad {
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		c.cache.forcePush(rec)
	}

	// Sort keys to be inserted.
	sort.Strings(keys)

	// NOTE: we shouldn't be reading any more records after this point.
	// TODO: assert it.

	walEntry := newWALEntry()

	// Append new records with the appropriate "next" pointers.

	overwrittenRecords := []int64{}
	for _, key := range keys {
		value := wb.sets[key]

		// Find last less than.
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		if offset == 0 {
			// Head.
			rec := &record{
				recordHeader: recordHeader{
					Next: c.Head,
				},
				Key:   key,
				Value: value,
			}
			newRecordOffset, err := c.writeRecord(rec)
			if err != nil {
				return err
			}
			c.Head = newRecordOffset
			walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))
			continue
		}
		prevRec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		rec := &record{
			recordHeader: recordHeader{
				Next: prevRec.Next,
			},
			Key:   key,
			Value: value,
		}
		newRecordOffset, err := c.writeRecord(rec)
		prevRec.Next = newRecordOffset
		walEntry.Push(newWALRecord(prevRec.Offset, prevRec.recordHeader.Bytes()))
		if prevRec.Key == key {
			overwrittenRecords = append(overwrittenRecords, prevRec.Offset)
		}
		c.cache.forcePush(rec)
		c.cache.forcePush(prevRec)
	}

	// Write sentinel record.

	currentOffset, err := c.writeSentinel()
	if err != nil {
		return err
	}

	// fsync data file.

	// Mark deleted and overwritten records as "deleted" at sentinel offset.
	// (This happens in memory.)

	for key := range wb.deletes {
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		if rec.Deleted == 0 {
			rec.Deleted = currentOffset
			walEntry.Push(newWALRecord(rec.Offset, rec.recordHeader.Bytes()))
		}
	}

	for _, offset := range overwrittenRecords {
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		rec.Deleted = currentOffset
		walEntry.Push(newWALRecord(rec.Offset, rec.recordHeader.Bytes()))
	}

	// ^ record changes should have been serialized + buffered. Write those entries
	// out to the WAL.
	logCommit, err := c.wal.Append(walEntry)
	if err != nil {
		return err
	}

	c.LastLogCommit = logCommit
	walEntry = newWALEntry()
	walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))
	logCommit, err = c.wal.Append(walEntry)
	if err != nil {
		return err
	}

	// Update + fsync data file header.
	c.LastCommit = currentOffset
	c.LastLogCommit = logCommit
	return c.f.Sync()
}

func NewCollection(file string) (*Collection, error) {
	f, err := os.OpenFile(file, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		return nil, err
	}
	err = f.Truncate(0)
	if err != nil {
		f.Close()
		return nil, err
	}

	wal, err := newWAL(file + ".wal")
	if err != nil {
		f.Close()
		return nil, err
	}

	c := &Collection{
		f:     f,
		wal:   wal,
		cache: newCache(cacheSize),
	}
	c.cache.c = c

	// write file header
	c.fileHeader.Head = 0
	c.f.Seek(0, 0)
	err = binary.Write(c.f, binary.LittleEndian, c.fileHeader)
	if err != nil {
		c.f.Close()
		c.wal.Close()
		return nil, err
	}
	return c, nil
}

func OpenCollection(file string) (*Collection, error) {
	f, err := os.OpenFile(file, os.O_RDWR, 0666)
	if err != nil {
		return nil, err
	}

	wal, err := newWAL(file + ".wal")
	if err != nil {
		f.Close()
		return nil, err
	}

	c := &Collection{
		f:     f,
		wal:   wal,
		cache: newCache(cacheSize),
	}
	c.cache.c = c

	// read file header
	c.f.Seek(0, 0)
	err = binary.Read(c.f, binary.LittleEndian, &c.fileHeader)
	if err != nil {
		c.f.Close()
		c.wal.Close()
		return nil, err
	}
	return c, nil
}

func (c *Collection) Close() {
	c.f.Close()
	c.wal.Close()
}
<commit_msg>write WAL changes to data file<commit_after>package lm2

import (
	"bytes"
	"encoding/binary"
	"errors"
	"math/rand"
	"os"
	"sort"
)

const sentinelMagic = 0xDEAD10CC

const cacheSize = 100000

type Collection struct {
	fileHeader
	f     *os.File
	wal   *WAL
	cache *recordCache
}

type fileHeader struct {
	Head          int64
	LastCommit    int64 // Unused for now
	LastLogCommit int64 // Unused for now
}

func (h fileHeader) Bytes() []byte {
	buf := bytes.NewBuffer(nil)
	binary.Write(buf, binary.LittleEndian, h)
	return buf.Bytes()
}

type recordHeader struct {
	Next    int64
	Deleted int64
	KeyLen  uint16
	ValLen  uint32
}

func (h recordHeader) Bytes() []byte {
	buf := bytes.NewBuffer(nil)
	binary.Write(buf, binary.LittleEndian, h)
	return buf.Bytes()
}

type sentinelRecord struct {
	Magic  uint32 // some fixed pattern
	Offset int64  // this record's offset
}

type record struct {
	recordHeader
	Offset int64
	Key    string
	Value  string
}

type recordCache struct {
	cache        map[int64]*record
	dirty        map[int64]bool
	maxKeyRecord *record
	size         int
	preventPurge bool

	c *Collection
}

func newCache(size int) *recordCache {
	return &recordCache{
		cache:        map[int64]*record{},
		dirty:        map[int64]bool{},
		maxKeyRecord: nil,
		size:         size,
	}
}

func (rc *recordCache) findLastLessThan(key string) int64 {
	if rc.maxKeyRecord != nil {
		if rc.maxKeyRecord.Key < key {
			return rc.maxKeyRecord.Offset
		}
	}
	max := ""
	maxOffset := int64(0)

	for offset, record := range rc.cache {
		if record.Key >= key {
			continue
		}
		if record.Key > max {
			max = record.Key
			maxOffset = offset
		}
	}
	return maxOffset
}

func (rc *recordCache) push(rec *record) {
	if rc.maxKeyRecord == nil || rc.maxKeyRecord.Key < rec.Key {
		rc.maxKeyRecord = rec
	} else if rand.Float32() >= 0.04 {
		return
	}
	rc.cache[rec.Offset] = rec
	if len(rc.cache) > rc.size && !rc.preventPurge {
		deletedKey := int64(0)
		for k := range rc.cache {
			if k == rc.maxKeyRecord.Offset {
				continue
			}
			deletedKey = k
			return
		}
		if rc.dirty[deletedKey] {
			// This is a dirty record. Flush changes to disk.
			rc.c.updateRecordHeader(deletedKey, rc.cache[deletedKey].recordHeader)
			delete(rc.dirty, deletedKey)
		}
		delete(rc.cache, deletedKey)
	}
}

func (rc *recordCache) forcePush(rec *record) {
	if rc.maxKeyRecord == nil || rc.maxKeyRecord.Key < rec.Key {
		rc.maxKeyRecord = rec
	}
	rc.cache[rec.Offset] = rec
}

func (c *Collection) readRecord(offset int64) (*record, error) {
	if offset == 0 {
		return nil, errors.New("lm2: invalid record offset 0")
	}

	if rec := c.cache.cache[offset]; rec != nil {
		return rec, nil
	}

	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return nil, err
	}

	header := recordHeader{}
	err = binary.Read(c.f, binary.LittleEndian, &header)
	if err != nil {
		return nil, err
	}

	keyValBuf := make([]byte, int(header.KeyLen)+int(header.ValLen))
	_, err = c.f.Read(keyValBuf)
	if err != nil {
		return nil, err
	}

	key := string(keyValBuf[:int(header.KeyLen)])
	value := string(keyValBuf[int(header.KeyLen):])

	rec := &record{
		recordHeader: header,
		Offset:       offset,
		Key:          key,
		Value:        value,
	}

	c.cache.push(rec)

	return rec, nil
}

func (c *Collection) setRecordNext(offset int64, next int64) error {
	if rec := c.cache.cache[offset]; rec != nil {
		rec.recordHeader.Next = next
		c.cache.dirty[offset] = true
		return nil
	}
	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return err
	}
	return binary.Write(c.f, binary.LittleEndian, next)
}

func (c *Collection) nextRecord(rec *record) *record {
	if rec == nil {
		return nil
	}
	nextRec, err := c.readRecord(rec.Next)
	if err != nil {
		return nil
	}
	return nextRec
}

func (c *Collection) writeRecord(rec *record) (int64, error) {
	offset, err := c.f.Seek(0, 2)
	if err != nil {
		return 0, err
	}

	rec.KeyLen = uint16(len(rec.Key))
	rec.ValLen = uint32(len(rec.Value))

	err = binary.Write(c.f, binary.LittleEndian, rec.recordHeader)
	if err != nil {
		return 0, err
	}

	_, err = c.f.WriteString(rec.Key)
	if err != nil {
		return 0, err
	}
	_, err = c.f.WriteString(rec.Value)
	if err != nil {
		return 0, err
	}

	rec.Offset = offset
	return offset, nil
}

func (c *Collection) writeSentinel() (int64, error) {
	offset, err := c.f.Seek(0, 2)
	if err != nil {
		return 0, err
	}
	sentinel := sentinelRecord{
		Magic:  sentinelMagic,
		Offset: offset,
	}
	err = binary.Write(c.f, binary.LittleEndian, sentinel)
	if err != nil {
		return 0, err
	}
	return offset + 12, nil
}

func (c *Collection) updateRecordHeader(offset int64, header recordHeader) error {
	if rec := c.cache.cache[offset]; rec != nil {
		rec.recordHeader = header
		c.cache.dirty[offset] = true
		return nil
	}
	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return err
	}
	return binary.Write(c.f, binary.LittleEndian, header)
}

func (c *Collection) findLastLessThanOrEqual(key string) (int64, error) {
	offset := int64(0)

	if c.Head == 0 {
		// Empty collection.
		return 0, nil
	}

	// read the head
	rec, err := c.readRecord(c.Head)
	if err != nil {
		return 0, err
	}
	if rec.Key > key { // we have a new head
		return 0, nil
	}

	cacheResult := c.cache.findLastLessThan(key)
	if cacheResult != 0 {
		rec, err = c.readRecord(cacheResult)
		if err != nil {
			return 0, err
		}
	}

	offset = rec.Offset

	for rec != nil {
		if rec.Key > key {
			break
		}
		offset = rec.Offset
		rec = c.nextRecord(rec)
	}

	return offset, nil
}

// Update atomically and durably applies a WriteBatch (a set of updates) to the collection.
func (c *Collection) Update(wb *WriteBatch) error {
	// Clean up WriteBatch.
	wb.cleanup()

	// Find and load records that will be modified into the cache.
	recordsToLoad := []int64{}

	keys := []string{}
	for key := range wb.sets {
		keys = append(keys, key)
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		if offset > 0 {
			recordsToLoad = append(recordsToLoad, offset)
		}
	}

	// Prevent cache purges.
	c.cache.preventPurge = true
	defer func() { c.cache.preventPurge = false }()

	for _, offset := range recordsToLoad {
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		c.cache.forcePush(rec)
	}

	// Sort keys to be inserted.
	sort.Strings(keys)

	// NOTE: we shouldn't be reading any more records after this point.
	// TODO: assert it.

	walEntry := newWALEntry()
	walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))

	// Append new records with the appropriate "next" pointers.

	overwrittenRecords := []int64{}
	for _, key := range keys {
		value := wb.sets[key]

		// Find last less than.
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		if offset == 0 {
			// Head.
			rec := &record{
				recordHeader: recordHeader{
					Next: c.Head,
				},
				Key:   key,
				Value: value,
			}
			newRecordOffset, err := c.writeRecord(rec)
			if err != nil {
				return err
			}
			c.Head = newRecordOffset
			walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))
			continue
		}
		prevRec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		rec := &record{
			recordHeader: recordHeader{
				Next: prevRec.Next,
			},
			Key:   key,
			Value: value,
		}
		newRecordOffset, err := c.writeRecord(rec)
		prevRec.Next = newRecordOffset
		walEntry.Push(newWALRecord(prevRec.Offset, prevRec.recordHeader.Bytes()))
		if prevRec.Key == key {
			overwrittenRecords = append(overwrittenRecords, prevRec.Offset)
		}
		c.cache.forcePush(rec)
		c.cache.forcePush(prevRec)
	}

	// Write sentinel record.

	currentOffset, err := c.writeSentinel()
	if err != nil {
		return err
	}

	// fsync data file.
	err = c.f.Sync()
	if err != nil {
		return err
	}

	// Mark deleted and overwritten records as "deleted" at sentinel offset.
	// (This happens in memory.)

	for key := range wb.deletes {
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		if rec.Deleted == 0 {
			rec.Deleted = currentOffset
			walEntry.Push(newWALRecord(rec.Offset, rec.recordHeader.Bytes()))
		}
	}

	for _, offset := range overwrittenRecords {
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		rec.Deleted = currentOffset
		walEntry.Push(newWALRecord(rec.Offset, rec.recordHeader.Bytes()))
	}

	// ^ record changes should have been serialized + buffered. Write those entries
	// out to the WAL.
	c.LastCommit = currentOffset
	walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))
	logCommit, err := c.wal.Append(walEntry)
	if err != nil {
		return err
	}

	c.LastLogCommit = logCommit

	// Update + fsync data file header.

	for _, walRec := range walEntry.records {
		n, err := c.f.WriteAt(walRec.Data, walRec.Offset)
		if err != nil {
			return err
		}
		if int64(n) != walRec.Size {
			return errors.New("lm2: incomplete data write")
		}
	}

	return c.f.Sync()
}

func NewCollection(file string) (*Collection, error) {
	f, err := os.OpenFile(file, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		return nil, err
	}
	err = f.Truncate(0)
	if err != nil {
		f.Close()
		return nil, err
	}

	wal, err := newWAL(file + ".wal")
	if err != nil {
		f.Close()
		return nil, err
	}

	c := &Collection{
		f:     f,
		wal:   wal,
		cache: newCache(cacheSize),
	}
	c.cache.c = c

	// write file header
	c.fileHeader.Head = 0
	c.f.Seek(0, 0)
	err = binary.Write(c.f, binary.LittleEndian, c.fileHeader)
	if err != nil {
		c.f.Close()
		c.wal.Close()
		return nil, err
	}
	return c, nil
}

func OpenCollection(file string) (*Collection, error) {
	f, err := os.OpenFile(file, os.O_RDWR, 0666)
	if err != nil {
		return nil, err
	}

	wal, err := newWAL(file + ".wal")
	if err != nil {
		f.Close()
		return nil, err
	}

	c := &Collection{
		f:     f,
		wal:   wal,
		cache: newCache(cacheSize),
	}
	c.cache.c = c

	// read file header
	c.f.Seek(0, 0)
	err = binary.Read(c.f, binary.LittleEndian, &c.fileHeader)
	if err != nil {
		c.f.Close()
		c.wal.Close()
		return nil, err
	}
	return c, nil
}

func (c *Collection) Close() {
	c.f.Close()
	c.wal.Close()
}
<|endoftext|>


<commit_before>// python like log package
// can use instead of standard log package
package log

import (
	"fmt"
	"log"
	"os"
)

const (
	LL_Debug = 1 << iota
	LL_Info
	LL_Warn
	LL_Error
	LL_Fatal
	LL_All = LL_Debug | LL_Info | LL_Warn | LL_Error | LL_Fatal
)

var LogLevel2str = map[int]string{
	LL_Debug: "Debug",
	LL_Info:  "Info",
	LL_Warn:  "Warn",
	LL_Error: "Error",
	LL_Fatal: "Fatal",
}

type Log struct {
	loglevel int
	l        map[int]*log.Logger
}

func New(prefix string, loglevel int, release bool) Log {
	l := Log{
		loglevel: loglevel,
		l:        make(map[int]*log.Logger),
	}
	flags := log.LstdFlags
	if !release {
		flags = log.Ltime | log.Lshortfile
	}
	for i, v := range LogLevel2str {
		l.l[i] = log.New(os.Stderr, fmt.Sprintf("%v %v:", prefix, v), flags)
	}
	return l
}

func (l Log) String() string {
	levelstr := ""
	for i, v := range LogLevel2str {
		if l.IsLevel(i) {
			levelstr += v + " "
		}
	}
	return fmt.Sprintf("log level is %v", levelstr)
}
func (l *Log) AddLevel(level int) {
	l.loglevel |= level
}
func (l *Log) SetLevel(level int) {
	l.loglevel = level
}
func (l *Log) DelLevel(level int) {
	l.loglevel &= ^level
}
func (l *Log) IsLevel(level int) bool {
	return l.loglevel&level != 0
}

func (l Log) printf(ll int, format string, v ...interface{}) {
	if !l.IsLevel(ll) {
		return
	}
	l.l[ll].Output(3, fmt.Sprintf(format, v...))
}

func (l Log) Info(format string, v ...interface{}) {
	l.printf(LL_Info, format, v...)
}
func (l Log) Warn(format string, v ...interface{}) {
	l.printf(LL_Warn, format, v...)
}
func (l Log) Debug(format string, v ...interface{}) {
	l.printf(LL_Debug, format, v...)
}
func (l Log) Error(format string, v ...interface{}) {
	l.printf(LL_Error, format, v...)
}
func (l Log) Fatal(format string, v ...interface{}) {
	l.printf(LL_Fatal, format, v...)
	os.Exit(1)
}

// ===

var logger = New("goguelike", LL_All, false)

func LevelString() string {
	return logger.String()
}

func SetReleaseLogger() {
	logger = New("goguelike", LL_All, true)
}
func Printf(format string, v ...interface{}) {
	logger.printf(LL_Info, format, v...)
}
func Fatalf(format string, v ...interface{}) {
	logger.printf(LL_Fatal, format, v...)
	os.Exit(1)
}
func Info(format string, v ...interface{}) {
	logger.printf(LL_Info, format, v...)
}
func Warn(format string, v ...interface{}) {
	logger.printf(LL_Warn, format, v...)
}
func Debug(format string, v ...interface{}) {
	logger.printf(LL_Debug, format, v...)
}
func Error(format string, v ...interface{}) {
	logger.printf(LL_Error, format, v...)
}
func Fatal(format string, v ...interface{}) {
	logger.printf(LL_Fatal, format, v...)
	os.Exit(1)
}
func AddLevel(level int) {
	logger.loglevel |= level
}
func SetLevel(level int) {
	logger.loglevel = level
}
func DelLevel(level int) {
	logger.loglevel &= ^level
}
func IsLevel(level int) bool {
	return logger.loglevel&level != 0
}
<commit_msg>remove goguelike string from default logger<commit_after>// python like log package
// can use instead of standard log package
package log

import (
	"fmt"
	"log"
	"os"
)

const (
	LL_Debug = 1 << iota
	LL_Info
	LL_Warn
	LL_Error
	LL_Fatal
	LL_All = LL_Debug | LL_Info | LL_Warn | LL_Error | LL_Fatal
)

var LogLevel2str = map[int]string{
	LL_Debug: "Debug",
	LL_Info:  "Info",
	LL_Warn:  "Warn",
	LL_Error: "Error",
	LL_Fatal: "Fatal",
}

type Log struct {
	loglevel int
	l        map[int]*log.Logger
}

func New(prefix string, loglevel int, release bool) Log {
	l := Log{
		loglevel: loglevel,
		l:        make(map[int]*log.Logger),
	}
	flags := log.LstdFlags
	if !release {
		flags = log.Ltime | log.Lshortfile
	}
	for i, v := range LogLevel2str {
		l.l[i] = log.New(os.Stderr, fmt.Sprintf("%v %v:", prefix, v), flags)
	}
	return l
}

func (l Log) String() string {
	levelstr := ""
	for i, v := range LogLevel2str {
		if l.IsLevel(i) {
			levelstr += v + " "
		}
	}
	return fmt.Sprintf("log level is %v", levelstr)
}
func (l *Log) AddLevel(level int) {
	l.loglevel |= level
}
func (l *Log) SetLevel(level int) {
	l.loglevel = level
}
func (l *Log) DelLevel(level int) {
	l.loglevel &= ^level
}
func (l *Log) IsLevel(level int) bool {
	return l.loglevel&level != 0
}

func (l Log) printf(ll int, format string, v ...interface{}) {
	if !l.IsLevel(ll) {
		return
	}
	l.l[ll].Output(3, fmt.Sprintf(format, v...))
}

func (l Log) Info(format string, v ...interface{}) {
	l.printf(LL_Info, format, v...)
}
func (l Log) Warn(format string, v ...interface{}) {
	l.printf(LL_Warn, format, v...)
}
func (l Log) Debug(format string, v ...interface{}) {
	l.printf(LL_Debug, format, v...)
}
func (l Log) Error(format string, v ...interface{}) {
	l.printf(LL_Error, format, v...)
}
func (l Log) Fatal(format string, v ...interface{}) {
	l.printf(LL_Fatal, format, v...)
	os.Exit(1)
}

// ===

var logger = New("", LL_All, false)

func LevelString() string {
	return logger.String()
}

func SetReleaseLogger() {
	logger = New("goguelike", LL_All, true)
}
func Printf(format string, v ...interface{}) {
	logger.printf(LL_Info, format, v...)
}
func Fatalf(format string, v ...interface{}) {
	logger.printf(LL_Fatal, format, v...)
	os.Exit(1)
}
func Info(format string, v ...interface{}) {
	logger.printf(LL_Info, format, v...)
}
func Warn(format string, v ...interface{}) {
	logger.printf(LL_Warn, format, v...)
}
func Debug(format string, v ...interface{}) {
	logger.printf(LL_Debug, format, v...)
}
func Error(format string, v ...interface{}) {
	logger.printf(LL_Error, format, v...)
}
func Fatal(format string, v ...interface{}) {
	logger.printf(LL_Fatal, format, v...)
	os.Exit(1)
}
func AddLevel(level int) {
	logger.loglevel |= level
}
func SetLevel(level int) {
	logger.loglevel = level
}
func DelLevel(level int) {
	logger.loglevel &= ^level
}
func IsLevel(level int) bool {
	return logger.loglevel&level != 0
}
<|endoftext|>


<commit_before>/*

	golem - lightweight Go WebSocket-framework
    Copyright (C) 2013  Niklas Voss

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

*/

package golem

import (
	"flag"
	"log"
)

/*
	should be rewritten and use a trick like this:
	http://play.golang.org/p/8o0WywyaDT
*/

var debug *bool = flag.Bool("debug", true, "enable debug logging") // currently true for development

const (
	delimiter = " "
	FATAL     = "[FATAL]"
	ERROR     = "[ERROR]"
	WARN      = "[WARN ]"
	DEBUG     = "[DEBUG]"
	INFO      = "[INFO ]"
)

func Debugf(format string, args ...interface{}) {
	if *debug {
		log.Printf(DEBUG+delimiter+format, args)
	}
}

func Errorf(format string, args ...interface{}) {
	log.Printf(ERROR+delimiter+format, args)
}
<commit_msg>Removed simple log implementation, because it was not used.<commit_after><|endoftext|>


<commit_before>//  ---------------------------------------------------------------------------
//
//  log.go
//
//  Copyright (c) 2015, Jared Chavez.
//  All rights reserved.
//
//  Use of this source code is governed by a BSD-style
//  license that can be found in the LICENSE file.
//
//  -----------

// Package log provides interfaces, basic types and formatting helpers
// for common application logging tasks.
package log

import (
    "bytes"
    "container/list"
    "fmt"
    "path/filepath"
    "runtime"
    "strings"
    "sync"
    "time"
)

var timeFormat = "2006/01/02 15:04:05.000000"

// LogCloser represents the interface for a closable log object, such
// as those provided by file-backed logging.
type LogCloser interface {
    Close()
}

// Log represents the interface for a generalized log object.
type LogNotify interface {
    Print(msg string)
}

// LogToggler represents the interface needed to temporarily enabled/disable
// a given logger.
type LogToggler interface {
    SetEnabled(bool)
}

// LogBuffer holds a specified number of logs in memory for rendering
// by the http logs Uri.
type LogBuffer struct {
    changed bool
    enabled bool
    lock    sync.RWMutex
    logs    *list.List
    maxSize int
}

// NewLogBuffer returns a pointer to a new LogBuffer instance.
func NewLogBuffer(maxSize int) *LogBuffer {
    logBuffer := &LogBuffer {
        enabled : true,
        logs    : list.New(),
        maxSize : maxSize,
    }

    return logBuffer
}

// HasChanged reports whether or not the buffer has been modified
// since it was last read.
func (this *LogBuffer) HasChanged() bool {
    this.lock.RLock()
    defer this.lock.RUnlock()

    return this.changed
}

// Print formats and adds a new message to the log buffer. If the new
// message causes the log buffer to grow larger than its maxSize, it
// truncates the end oldest entry in the buffer. Once the message is
// stored, the changed flag is set to true.
func (this *LogBuffer) Print(msg string) {
    this.lock.Lock()
    defer this.lock.Unlock()

    if !this.enabled {
        return
    }

    this.logs.PushFront(msg)

    if this.logs.Len() > this.maxSize {
        this.logs.Remove(this.logs.Back())
    }

    this.changed = true
}

// ReadAll returns a list of all log messages currently in the buffer.
func (this *LogBuffer) ReadAll() []string {
    this.lock.RLock()
    defer this.lock.RUnlock()

    results := make([]string, 0, this.logs.Len())
    for e := this.logs.Front(); e != nil; e = e.Next() {
        results = append(results, e.Value.(string))
    }

    this.changed = false

    return results
}

// MarshalJSON implements the standard go json marshaler for the
// LogBuffer type.
func (this *LogBuffer) MarshalJSON() ([]byte, error) {
    this.lock.RLock()
    defer this.lock.RUnlock()

    var buffer bytes.Buffer

    buffer.WriteString("[")
    for e := this.logs.Front(); e != nil; {
        buffer.WriteString("\"")
        buffer.WriteString(strings.TrimSpace(e.Value.(string)))
        buffer.WriteString("\"")

        next := e.Next()
        if next == nil {
            break
        }

        buffer.WriteString(",")
        e = next
    }
    buffer.WriteString("]")

    return buffer.Bytes(), nil
}

// SetEnabled temporarily enables/disables the logger.
func (this *LogBuffer) SetEnabled(enabled bool) {
    this.lock.Lock()
    defer this.lock.Unlock()

    this.enabled = enabled
}

// SetMaxSize changes the configured maxSize for the buffer, and chops
// the oldest entries off the buffer in the case that it is currently larger
// than the newly configured maxSize.
func (this *LogBuffer) SetMaxSize(maxSize int) {
    this.lock.Lock()
    defer this.lock.Unlock()

    diff := maxSize - this.logs.Len()
    if diff < 0 {
        for i := 0; i > diff; i-- {
            this.logs.Remove(this.logs.Back())
        }
    }

    this.maxSize = maxSize
}

// FormatLogMsg formats a log given the standard logging format
// yyyy/mm/dd MM:HH:SS.ssssss [NAME] <file:line> msg
func FormatLogMsg(name, format string, callDepth int, v ...interface{}) string {
    var msg string
    if len(v) < 1 {
        msg = format
    } else {
        msg = fmt.Sprintf(format, v...)
    }

    _, file, line, ok := runtime.Caller(callDepth)
    if ok {
        file = fmt.Sprintf(" <%s:%d>", filepath.Base(file), line)
    }

    if len(msg) < 1 {
        return ""
    }

    if msg[len(msg) - 1] == '\n' {
        return fmt.Sprintf(
            "%s [%s]%s %s",
            time.Now().Format(timeFormat),
            strings.ToUpper(name),
            file,
            msg,
        )
    }

    return fmt.Sprintf(
        "%s [%s]%s %s\n",
        time.Now().Format(timeFormat),
        strings.ToUpper(name),
        file,
        msg,
    )
}
<commit_msg>less error-prone json marshaling<commit_after>//  ---------------------------------------------------------------------------
//
//  log.go
//
//  Copyright (c) 2015, Jared Chavez.
//  All rights reserved.
//
//  Use of this source code is governed by a BSD-style
//  license that can be found in the LICENSE file.
//
//  -----------

// Package log provides interfaces, basic types and formatting helpers
// for common application logging tasks.
package log

import (
    "container/list"
    "encoding/json"
    "fmt"
    "path/filepath"
    "runtime"
    "strings"
    "sync"
    "time"
)

var timeFormat = "2006/01/02 15:04:05.000000"

// LogCloser represents the interface for a closable log object, such
// as those provided by file-backed logging.
type LogCloser interface {
    Close()
}

// Log represents the interface for a generalized log object.
type LogNotify interface {
    Print(msg string)
}

// LogToggler represents the interface needed to temporarily enabled/disable
// a given logger.
type LogToggler interface {
    SetEnabled(bool)
}

// LogBuffer holds a specified number of logs in memory for rendering
// by the http logs Uri.
type LogBuffer struct {
    changed bool
    enabled bool
    lock    sync.RWMutex
    logs    *list.List
    maxSize int
}

// NewLogBuffer returns a pointer to a new LogBuffer instance.
func NewLogBuffer(maxSize int) *LogBuffer {
    logBuffer := &LogBuffer {
        enabled : true,
        logs    : list.New(),
        maxSize : maxSize,
    }

    return logBuffer
}

// HasChanged reports whether or not the buffer has been modified
// since it was last read.
func (this *LogBuffer) HasChanged() bool {
    this.lock.RLock()
    defer this.lock.RUnlock()

    return this.changed
}

// Print formats and adds a new message to the log buffer. If the new
// message causes the log buffer to grow larger than its maxSize, it
// truncates the end oldest entry in the buffer. Once the message is
// stored, the changed flag is set to true.
func (this *LogBuffer) Print(msg string) {
    this.lock.Lock()
    defer this.lock.Unlock()

    if !this.enabled {
        return
    }

    this.logs.PushFront(msg)

    if this.logs.Len() > this.maxSize {
        this.logs.Remove(this.logs.Back())
    }

    this.changed = true
}

// ReadAll returns a list of all log messages currently in the buffer.
func (this *LogBuffer) ReadAll() []string {
    this.lock.RLock()
    defer this.lock.RUnlock()

    results := make([]string, 0, this.logs.Len())
    for e := this.logs.Front(); e != nil; e = e.Next() {
        results = append(results, e.Value.(string))
    }

    this.changed = false

    return results
}

// MarshalJSON implements the standard go json marshaler for the
// LogBuffer type.
func (this *LogBuffer) MarshalJSON() ([]byte, error) {
    this.lock.RLock()
    defer this.lock.RUnlock()

    logs := make([]interface{}, 0, this.logs.Len())

    for e := this.logs.Front(); e != nil; e = e.Next() {
        logs = append(logs, e.Value)
    }

    return json.Marshal(&logs)
}

// SetEnabled temporarily enables/disables the logger.
func (this *LogBuffer) SetEnabled(enabled bool) {
    this.lock.Lock()
    defer this.lock.Unlock()

    this.enabled = enabled
}

// SetMaxSize changes the configured maxSize for the buffer, and chops
// the oldest entries off the buffer in the case that it is currently larger
// than the newly configured maxSize.
func (this *LogBuffer) SetMaxSize(maxSize int) {
    this.lock.Lock()
    defer this.lock.Unlock()

    diff := maxSize - this.logs.Len()
    if diff < 0 {
        for i := 0; i > diff; i-- {
            this.logs.Remove(this.logs.Back())
        }
    }

    this.maxSize = maxSize
}

// FormatLogMsg formats a log given the standard logging format
// yyyy/mm/dd MM:HH:SS.ssssss [NAME] <file:line> msg
func FormatLogMsg(name, format string, callDepth int, v ...interface{}) string {
    var msg string
    if len(v) < 1 {
        msg = format
    } else {
        msg = fmt.Sprintf(format, v...)
    }

    _, file, line, ok := runtime.Caller(callDepth)
    if ok {
        file = fmt.Sprintf(" <%s:%d>", filepath.Base(file), line)
    }

    if len(msg) < 1 {
        return ""
    }

    if msg[len(msg) - 1] == '\n' {
        return fmt.Sprintf(
            "%s [%s]%s %s",
            time.Now().Format(timeFormat),
            strings.ToUpper(name),
            file,
            msg,
        )
    }

    return fmt.Sprintf(
        "%s [%s]%s %s\n",
        time.Now().Format(timeFormat),
        strings.ToUpper(name),
        file,
        msg,
    )
}
<|endoftext|>


<commit_before>package arangolite

import (
	"log"
	"net/http"
	"net/http/httputil"
	"time"
)

// LogVerbosity is the logging verbosity.
type LogVerbosity int

const (
	// LogSummary prints a simple summary of the exchanges with the database.
	LogSummary LogVerbosity = iota
	// LogDebug prints all the sent and received http requests.
	LogDebug
)

// newLoggingSender returns a logging wrapper around a sender.
func newLoggingSender(sender sender, logger *log.Logger, verbosity LogVerbosity) sender {
	return &loggingSender{
		sender:    sender,
		logger:    logger,
		verbosity: verbosity,
	}
}

type loggingSender struct {
	sender    sender
	logger    *log.Logger
	verbosity LogVerbosity
}

func (s *loggingSender) Send(cli *http.Client, req *http.Request) (Result, error) {
	if s.verbosity == LogDebug {
		r, _ := httputil.DumpRequestOut(req, true)
		s.logger.Println("Request:")
		s.logger.Println(string(r))
	}

	now := time.Now()

	result, err := s.sender.Send(cli, req)
	if err != nil {
		s.logger.Printf("Error: %s\n", err.Error())
		return nil, err
	}

	s.logger.Printf("Success in %v:\n", time.Since(now))
	if s.verbosity == LogDebug {
		s.logger.Println(result.Raw())
	}

	return result, nil
}
<commit_msg>Logger fixed<commit_after>package arangolite

import (
	"log"
	"net/http"
	"net/http/httputil"
	"time"
)

// LogVerbosity is the logging verbosity.
type LogVerbosity int

const (
	// LogSummary prints a simple summary of the exchanges with the database.
	LogSummary LogVerbosity = iota
	// LogDebug prints all the sent and received http requests.
	LogDebug
)

// newLoggingSender returns a logging wrapper around a sender.
func newLoggingSender(sender sender, logger *log.Logger, verbosity LogVerbosity) sender {
	return &loggingSender{
		sender:    sender,
		logger:    logger,
		verbosity: verbosity,
	}
}

type loggingSender struct {
	sender    sender
	logger    *log.Logger
	verbosity LogVerbosity
}

func (s *loggingSender) Send(cli *http.Client, req *http.Request) (*response, error) {
	if s.verbosity == LogDebug {
		r, _ := httputil.DumpRequestOut(req, true)
		s.logger.Println("Request:")
		s.logger.Println(string(r))
	}

	now := time.Now()

	res, err := s.sender.Send(cli, req)
	if err != nil {
		s.logger.Printf("Send error: %s\n", err.Error())
		return nil, err
	}
	if res.parsed.Error {
		s.logger.Printf("Database error: %s\n", res.parsed.ErrorMessage)
		return res, nil
	}

	s.logger.Printf("Success in %v:\n", time.Since(now))
	if s.verbosity == LogDebug {
		s.logger.Println(res.raw)
	}

	return res, nil
}
<|endoftext|>


<commit_before>package log

import (
	"reflect"
	"fmt"
)

const TIME_FORMAT = "2006-01-02T15:04:05.000000"
const LEVEL_DEBUG = "DEBUG"
const LEVEL_INFO  = "INFO"

type Config struct {
	Level string
	Formatter func(level string, message string, tags map[string]string, dateFormat string) string
	Output func(formattedMessage string)
	Program string
	DateFormat string
	Tags map[string]string
}

type Log struct {
	config *Config
}
type LogFormattingFailed string
func (err LogFormattingFailed) String() string {
	return "LogFormattingFailed"
}

type InvalidContext string
func (err InvalidContext) Error() string {
	return "invalid type for context. Must be map, struct, ptr(map) or ptr(struct)"
}

func (log Log) Info(message string, tags interface{}) error {
	mergedTags, err := mergeTags(log.config.Tags, tags)
	if err != nil {
		return err
	}
	log.config.Output(log.config.Formatter(LEVEL_INFO, message, mergedTags, log.config.DateFormat))
	return nil
}

func (log Log) Debug(message string, tags interface{}) error {
	if log.config.Level == LEVEL_DEBUG {
		mergedTags, err := mergeTags(log.config.Tags, tags)
		if err != nil {
			return err
		}
		log.config.Output(log.config.Formatter(LEVEL_DEBUG, message, mergedTags, log.config.DateFormat))
	}
	return nil
}

func (log Log) ChildLogger(function string, context interface{}) (*Log, error) {
	childConfig := new(Config)
	childConfig.Level = log.config.Level
	childConfig.Formatter = log.config.Formatter
	childConfig.Output = log.config.Output
	childConfig.DateFormat = log.config.DateFormat
	childConfig.Program = log.config.Program
	childConfig.Tags = log.config.Tags
	childConfig.Tags["function"] = function
	mergedTags, err := mergeTags(log.config.Tags, context)
	if err != nil {
		return nil, err
	}
	childConfig.Tags = mergedTags
	return NewLogger(childConfig), nil

}

func NewLogger(config *Config) *Log {
	if config.Tags == nil {
		config.Tags = map[string]string{}
	}
	config.Tags["program"] = config.Program
	config.Tags["function"] = "main"
	logger := new(Log)
	logger.config = config
	return logger
}

func mergeTags(tags map[string]string, context interface{}) (map[string]string, error) {
	outputTags := map[string]string{}
	for name, value := range tags {
		outputTags[name] = value
	}

	if context == nil {
		return outputTags, nil
	}

	switch aTags := context.(type) {
	case error:
		outputTags["error"] = aTags.Error()
		return outputTags, nil
	case fmt.Stringer:
		outputTags["context"] = aTags.String()
		return outputTags, nil
	}

	reflectedValue := reflect.ValueOf(context)
	for {
		if reflectedValue.Kind() == reflect.Ptr {
			reflectedValue = reflectedValue.Elem()
		} else {
			break
		}
	}

	if reflectedValue.Kind() == reflect.Map {
		for _, name := range reflectedValue.MapKeys() {
			outputTags[name.String()] = reflectedValue.MapIndex(name).String()
		}
	} else if reflectedValue.Kind() == reflect.Struct {
		reflectedContext := reflect.TypeOf(context)
		for {
			if reflectedContext.Kind() == reflect.Ptr {
				reflectedContext = reflectedContext.Elem()
			} else {
				break
			}
		}

		for i := 0; i < reflectedContext.NumField(); i++ {
			currentField := reflectedContext.Field(i)
			if currentField.PkgPath == "" {  // merge only exported fields
				outputTags[currentField.Name] = reflectedValue.FieldByName(currentField.Name).String()
			}
		}
	} else {
		return nil, new(InvalidContext)
	}

	return outputTags, nil
}
<commit_msg>adds Logger interface<commit_after>package log

import (
	"reflect"
	"fmt"
)

const TIME_FORMAT = "2006-01-02T15:04:05.000000"
const LEVEL_DEBUG = "DEBUG"
const LEVEL_INFO  = "INFO"

type Config struct {
	Level string
	Formatter func(level string, message string, tags map[string]string, dateFormat string) string
	Output func(formattedMessage string)
	Program string
	DateFormat string
	Tags map[string]string
}

type LogFormattingFailed string
func (err LogFormattingFailed) String() string {
	return "LogFormattingFailed"
}

type InvalidContext string
func (err InvalidContext) Error() string {
	return "invalid type for context. Must be map, struct, ptr(map) or ptr(struct)"
}

type Logger interface {
	Info(string, interface{}) error
	Debug(string, interface{}) error
	ChildLogger(string, interface{}) (*Log, error)
}

type Log struct {
	config *Config
}
func (log Log) Info(message string, tags interface{}) error {
	mergedTags, err := mergeTags(log.config.Tags, tags)
	if err != nil {
		return err
	}
	log.config.Output(log.config.Formatter(LEVEL_INFO, message, mergedTags, log.config.DateFormat))
	return nil
}

func (log Log) Debug(message string, tags interface{}) error {
	if log.config.Level == LEVEL_DEBUG {
		mergedTags, err := mergeTags(log.config.Tags, tags)
		if err != nil {
			return err
		}
		log.config.Output(log.config.Formatter(LEVEL_DEBUG, message, mergedTags, log.config.DateFormat))
	}
	return nil
}

func (log Log) ChildLogger(function string, context interface{}) (*Log, error) {
	childConfig := new(Config)
	childConfig.Level = log.config.Level
	childConfig.Formatter = log.config.Formatter
	childConfig.Output = log.config.Output
	childConfig.DateFormat = log.config.DateFormat
	childConfig.Program = log.config.Program
	childConfig.Tags = log.config.Tags
	childConfig.Tags["function"] = function
	mergedTags, err := mergeTags(log.config.Tags, context)
	if err != nil {
		return nil, err
	}
	childConfig.Tags = mergedTags
	return NewLogger(childConfig), nil

}

func NewLogger(config *Config) *Log {
	if config.Tags == nil {
		config.Tags = map[string]string{}
	}
	config.Tags["program"] = config.Program
	config.Tags["function"] = "main"
	logger := new(Log)
	logger.config = config
	return logger
}

func mergeTags(tags map[string]string, context interface{}) (map[string]string, error) {
	outputTags := map[string]string{}
	for name, value := range tags {
		outputTags[name] = value
	}

	if context == nil {
		return outputTags, nil
	}

	switch aTags := context.(type) {
	case error:
		outputTags["error"] = aTags.Error()
		return outputTags, nil
	case fmt.Stringer:
		outputTags["context"] = aTags.String()
		return outputTags, nil
	}

	reflectedValue := reflect.ValueOf(context)
	for {
		if reflectedValue.Kind() == reflect.Ptr {
			reflectedValue = reflectedValue.Elem()
		} else {
			break
		}
	}

	if reflectedValue.Kind() == reflect.Map {
		for _, name := range reflectedValue.MapKeys() {
			outputTags[name.String()] = reflectedValue.MapIndex(name).String()
		}
	} else if reflectedValue.Kind() == reflect.Struct {
		reflectedContext := reflect.TypeOf(context)
		for {
			if reflectedContext.Kind() == reflect.Ptr {
				reflectedContext = reflectedContext.Elem()
			} else {
				break
			}
		}

		for i := 0; i < reflectedContext.NumField(); i++ {
			currentField := reflectedContext.Field(i)
			if currentField.PkgPath == "" {  // merge only exported fields
				outputTags[currentField.Name] = reflectedValue.FieldByName(currentField.Name).String()
			}
		}
	} else {
		return nil, new(InvalidContext)
	}

	return outputTags, nil
}
<|endoftext|>


<commit_before>// Package ltsvlog is a minimalist logging library for writing logs in
// LTSV (Labeled Tab-separated Value) format.
// See http://ltsv.org/ for LTSV.
//
// This logging library has three log levels: Debug, Info and Error.
// The Info and Error levels are always enabled.
// You can disable the Debug level but only when you create a logger.
//
// Each log record is printed as one line. A line has multiple fields
// separated by a tab character. Each field has a label and a value
// separated by a colon ':' character.
// So you must not contain a new line or a tab character in your labels
// and values. You must not contain a colon character in your labels.
package ltsvlog

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

// LV represents a label L and a value V.
type LV struct {
	L string
	V interface{}
}

// LogWriter is a LTSV logger interface
type LogWriter interface {
	DebugEnabled() bool
	Debug(lv ...LV)
	Info(lv ...LV)
	Error(lv ...LV)
	ErrorWithStack(lv ...LV)
}

// LTSVLogger is a LTSV logger.
type LTSVLogger struct {
	writer           io.Writer
	debugEnabled     bool
	appendPrefixFunc AppendPrefixFunc
	appendValueFunc  AppendValueFunc
	buf              []byte
	stackBuf         []byte
	mu               sync.Mutex
}

// Option is the function type to set an option of LTSVLogger
type Option func(l *LTSVLogger)

// StackBufSize returns the option function to set the stack buffer size.
func StackBufSize(size int) Option {
	return func(l *LTSVLogger) {
		l.stackBuf = make([]byte, size)
	}
}

// SetAppendPrefix returns the option function to set the function
// to append the log prefix. In the default setting, the prefix consists
// of the time and the level.
func SetAppendPrefix(f AppendPrefixFunc) Option {
	return func(l *LTSVLogger) {
		l.appendPrefixFunc = f
	}
}

// SetAppendValue returns the option function to set the function
// to append a value.
func SetAppendValue(f AppendValueFunc) Option {
	return func(l *LTSVLogger) {
		l.appendValueFunc = f
	}
}

// AppendPrefixFunc is a function type for appending a prefix
// for a log record to a byte buffer and returns the result buffer.
type AppendPrefixFunc func(buf []byte, level string) []byte

// AppendValueFunc is a function type for appending a value to
// a byte buffer and returns the result buffer.
type AppendValueFunc func(buf []byte, v interface{}) []byte

// NewLTSVLogger creates a LTSV logger with the default time and value format.
// Shorthand for NewLTSVLoggerCustomFormat(w, debugEnabled, 8192, nil, nil).
//
// The folloing two values are prepended to each log line.
//
// The first value is the current time with the label "time".
// The time format is RFC3339Nano UTC like 2006-01-02T15:04:05.00000000Z.
// The width of the nanoseconds are always 9. For example, the nanoseconds
// 123 is printed as 123000000.
// The second value is the log level with the label "level".
func NewLTSVLogger(w io.Writer, debugEnabled bool, options ...Option) *LTSVLogger {
	l := &LTSVLogger{
		writer:           w,
		debugEnabled:     debugEnabled,
		appendPrefixFunc: appendPrefix,
		appendValueFunc:  appendValue,
		stackBuf:         make([]byte, 8192),
	}
	for _, o := range options {
		o(l)
	}
	return l
}

// Deprecated. Use NewLTSVLogger with options instead.
//
// NewLTSVLoggerCustomFormat creates a LTSV logger with the buffer size for
// filling stack traces and user-supplied functions for appending a log
// record prefix and appending a log value.
func NewLTSVLoggerCustomFormat(w io.Writer, debugEnabled bool, stackBufSize int, appendPrefixFunc AppendPrefixFunc, appendValueFunc AppendValueFunc) *LTSVLogger {
	if appendPrefixFunc == nil {
		appendPrefixFunc = appendPrefix
	}
	if appendValueFunc == nil {
		appendValueFunc = appendValue
	}
	return &LTSVLogger{
		writer:           w,
		debugEnabled:     debugEnabled,
		appendPrefixFunc: appendPrefixFunc,
		appendValueFunc:  appendValueFunc,
		stackBuf:         make([]byte, stackBufSize),
	}
}

// DebugEnabled returns whether or not the debug level is enabled.
// You can avoid the cost of evaluation of arguments passed to Debug like:
// if logger.DebugEnabled() { logger.Debug(ltsvlog.LTV{"label1": "value1"}) }
func (l *LTSVLogger) DebugEnabled() bool {
	return l.debugEnabled
}

// Debug writes a log with the debug level if the debug level is enabled.
func (l *LTSVLogger) Debug(lv ...LV) {
	if l.debugEnabled {
		l.mu.Lock()
		l.log("Debug", lv...)
		l.mu.Unlock()
	}
}

// Info writes a log with the info level.
func (l *LTSVLogger) Info(lv ...LV) {
	l.mu.Lock()
	l.log("Info", lv...)
	l.mu.Unlock()
}

// Error writes a log with the error level.
func (l *LTSVLogger) Error(lv ...LV) {
	l.mu.Lock()
	l.log("Error", lv...)
	l.mu.Unlock()
}

// ErrorWithStack writes a log and a stack with the error level.
func (l *LTSVLogger) ErrorWithStack(lv ...LV) {
	l.mu.Lock()
	args := lv
	args = append(args, LV{"stack", stack(2, l.stackBuf)})
	l.log("Error", args...)
	l.mu.Unlock()
}

func (l *LTSVLogger) log(level string, lv ...LV) {
	// Note: To reuse the buffer, create an empty slice pointing to
	// the previously allocated buffer.
	buf := l.appendPrefixFunc(l.buf[:0], level)
	for i, labelAndVal := range lv {
		if i > 0 {
			buf = append(buf, '\t')
		}
		buf = append(buf, []byte(labelAndVal.L)...)
		buf = append(buf, ':')
		buf = l.appendValueFunc(buf, labelAndVal.V)
	}
	buf = append(buf, '\n')
	_, _ = l.writer.Write(buf)
	l.buf = buf
}

func appendPrefix(buf []byte, level string) []byte {
	buf = append(buf, "time:"...)
	now := time.Now().UTC()
	buf = appendTime(buf, now)
	buf = append(buf, "\tlevel:"...)
	buf = append(buf, []byte(level)...)
	buf = append(buf, '\t')
	return buf
}

func appendTime(buf []byte, t time.Time) []byte {
	buf = appendZeroPaddedInt(buf, t.Year(), 1000)
	buf = append(buf, byte('-'))
	buf = appendZeroPaddedInt(buf, int(t.Month()), 10)
	buf = append(buf, byte('-'))
	buf = appendZeroPaddedInt(buf, t.Day(), 10)
	buf = append(buf, byte('T'))
	buf = appendZeroPaddedInt(buf, t.Hour(), 10)
	buf = append(buf, byte(':'))
	buf = appendZeroPaddedInt(buf, t.Minute(), 10)
	buf = append(buf, byte(':'))
	buf = appendZeroPaddedInt(buf, t.Second(), 10)
	buf = append(buf, byte('.'))
	buf = appendZeroPaddedInt(buf, t.Nanosecond(), 100000000)
	return append(buf, byte('Z'))
}

var escaper = strings.NewReplacer("\t", "\\t", "\n", "\\n")

func escape(s string) string {
	return escaper.Replace(s)
}

func appendValue(buf []byte, v interface{}) []byte {
	// NOTE: In type switch case, case byte and case uint8 cannot coexist,
	// case rune and case uint cannot coexist.
	switch v.(type) {
	case nil:
		buf = append(buf, "<nil>"...)
	case string:
		buf = append(buf, []byte(escape(v.(string)))...)
	case int:
		buf = strconv.AppendInt(buf, int64(v.(int)), 10)
	case uint:
		buf = strconv.AppendUint(buf, uint64(v.(uint)), 10)
	case int8:
		buf = strconv.AppendInt(buf, int64(v.(int8)), 10)
	case int16:
		buf = strconv.AppendInt(buf, int64(v.(int16)), 10)
	case int32:
		buf = strconv.AppendInt(buf, int64(v.(int32)), 10)
	case int64:
		buf = strconv.AppendInt(buf, v.(int64), 10)
	case uint8:
		buf = strconv.AppendUint(buf, uint64(v.(uint8)), 10)
	case uint16:
		buf = strconv.AppendUint(buf, uint64(v.(uint16)), 10)
	case uint32:
		buf = strconv.AppendUint(buf, uint64(v.(uint32)), 10)
	case uint64:
		buf = strconv.AppendUint(buf, uint64(v.(uint64)), 10)
	case float32:
		buf = append(buf, []byte(strconv.FormatFloat(float64(v.(float32)), 'g', -1, 32))...)
	case float64:
		buf = append(buf, []byte(strconv.FormatFloat(v.(float64), 'g', -1, 64))...)
	case bool:
		buf = strconv.AppendBool(buf, v.(bool))
	case uintptr:
		buf = strconv.AppendUint(buf, uint64(v.(uintptr)), 10)
	case []byte:
		buf = appendHexBytes(buf, v.([]byte))
	case fmt.Stringer:
		buf = append(buf, []byte(escape(v.(fmt.Stringer).String()))...)
	default:
		buf = append(buf, []byte(escape(fmt.Sprintf("%+v", v)))...)
	}
	return buf
}

var digits = []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}

func appendHexBytes(buf []byte, v []byte) []byte {
	buf = append(buf, "0x"...)
	for _, b := range v {
		buf = append(buf, digits[b/16])
		buf = append(buf, digits[b%16])
	}
	return buf
}

func appendZeroPaddedInt(buf []byte, v, p int) []byte {
	for ; p > 1; p /= 10 {
		q := v / p
		buf = append(buf, digits[q])
		v -= q * p
	}
	return append(buf, digits[v])
}

// Logger is the global logger.
// You can change this logger like
// ltsvlog.Logger = ltsvlog.NewLTSVLogger(os.Stdout, false)
// You can change the global logger safely only before writing
// to the logger. Chaging the logger while writing may cause
// the unexpected behavior.
var Logger = NewLTSVLogger(os.Stdout, true)

// Discard discards any logging outputs.
type Discard struct{}

// DebugEnabled always return false
func (*Discard) DebugEnabled() bool { return false }

// Debug prints nothing
func (*Discard) Debug(lv ...LV) {}

// Info prints nothing
func (*Discard) Info(lv ...LV) {}

// Error prints nothing
func (*Discard) Error(lv ...LV) {}

// ErrorWithStack prints nothing
func (*Discard) ErrorWithStack(lv ...LV) {}
<commit_msg>Add SetTimeLabel and SetLevelLabel options<commit_after>// Package ltsvlog is a minimalist logging library for writing logs in
// LTSV (Labeled Tab-separated Value) format.
// See http://ltsv.org/ for LTSV.
//
// This logging library has three log levels: Debug, Info and Error.
// The Info and Error levels are always enabled.
// You can disable the Debug level but only when you create a logger.
//
// Each log record is printed as one line. A line has multiple fields
// separated by a tab character. Each field has a label and a value
// separated by a colon ':' character.
// So you must not contain a new line or a tab character in your labels
// and values. You must not contain a colon character in your labels.
package ltsvlog

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

// LV represents a label L and a value V.
type LV struct {
	L string
	V interface{}
}

// LogWriter is a LTSV logger interface
type LogWriter interface {
	DebugEnabled() bool
	Debug(lv ...LV)
	Info(lv ...LV)
	Error(lv ...LV)
	ErrorWithStack(lv ...LV)
}

// LTSVLogger is a LTSV logger.
type LTSVLogger struct {
	writer                  io.Writer
	debugEnabled            bool
	timeLabel               string
	levelLabel              string
	appendPrefixFunc        AppendPrefixFunc
	appendPrefixFuncChanged bool
	appendValueFunc         AppendValueFunc
	buf                     []byte
	stackBuf                []byte
	mu                      sync.Mutex
}

// Option is the function type to set an option of LTSVLogger
type Option func(l *LTSVLogger)

// StackBufSize returns the option function to set the stack buffer size.
func StackBufSize(size int) Option {
	return func(l *LTSVLogger) {
		l.stackBuf = make([]byte, size)
	}
}

// SetTimeLabel returns the option function to set the time label.
// If the label is empty, logger does not print time values.
func SetTimeLabel(label string) Option {
	return func(l *LTSVLogger) {
		l.timeLabel = label
	}
}

// SetLevelLabel returns the option function to set the level label.
// If the label is empty, logger does not print level values.
func SetLevelLabel(label string) Option {
	return func(l *LTSVLogger) {
		l.levelLabel = label
	}
}

// SetAppendPrefix returns the option function to set the function
// to append the log prefix. In the default setting, the prefix consists
// of the time and the level.
func SetAppendPrefix(f AppendPrefixFunc) Option {
	return func(l *LTSVLogger) {
		l.appendPrefixFunc = f
		l.appendPrefixFuncChanged = true
	}
}

// SetAppendValue returns the option function to set the function
// to append a value.
func SetAppendValue(f AppendValueFunc) Option {
	return func(l *LTSVLogger) {
		l.appendValueFunc = f
	}
}

// AppendPrefixFunc is a function type for appending a prefix
// for a log record to a byte buffer and returns the result buffer.
type AppendPrefixFunc func(buf []byte, level string) []byte

// AppendValueFunc is a function type for appending a value to
// a byte buffer and returns the result buffer.
type AppendValueFunc func(buf []byte, v interface{}) []byte

const (
	defaultTimeLabel  = "time"
	defaultLevelLabel = "level"
)

var defaultAppendPrefixFunc = appendPrefixFunc(defaultTimeLabel, defaultLevelLabel)

// NewLTSVLogger creates a LTSV logger with the default time and value format.
// Shorthand for NewLTSVLoggerCustomFormat(w, debugEnabled, 8192, nil, nil).
//
// The folloing two values are prepended to each log line.
//
// The first value is the current time with the label "time".
// The time format is RFC3339Nano UTC like 2006-01-02T15:04:05.00000000Z.
// The width of the nanoseconds are always 9. For example, the nanoseconds
// 123 is printed as 123000000.
// The second value is the log level with the label "level".
func NewLTSVLogger(w io.Writer, debugEnabled bool, options ...Option) *LTSVLogger {
	l := &LTSVLogger{
		writer:           w,
		debugEnabled:     debugEnabled,
		timeLabel:        defaultTimeLabel,
		levelLabel:       defaultLevelLabel,
		appendPrefixFunc: defaultAppendPrefixFunc,
		appendValueFunc:  appendValue,
		stackBuf:         make([]byte, 8192),
	}
	for _, o := range options {
		o(l)
	}
	if !l.appendPrefixFuncChanged &&
		(l.timeLabel != defaultTimeLabel || l.levelLabel != defaultLevelLabel) {
		l.appendPrefixFunc = appendPrefixFunc(l.timeLabel, l.levelLabel)
	}
	return l
}

// Deprecated. Use NewLTSVLogger with options instead.
//
// NewLTSVLoggerCustomFormat creates a LTSV logger with the buffer size for
// filling stack traces and user-supplied functions for appending a log
// record prefix and appending a log value.
func NewLTSVLoggerCustomFormat(w io.Writer, debugEnabled bool, stackBufSize int, appendPrefixFunc AppendPrefixFunc, appendValueFunc AppendValueFunc) *LTSVLogger {
	if appendPrefixFunc == nil {
		appendPrefixFunc = appendPrefix
	}
	if appendValueFunc == nil {
		appendValueFunc = appendValue
	}
	return &LTSVLogger{
		writer:           w,
		debugEnabled:     debugEnabled,
		appendPrefixFunc: appendPrefixFunc,
		appendValueFunc:  appendValueFunc,
		stackBuf:         make([]byte, stackBufSize),
	}
}

// DebugEnabled returns whether or not the debug level is enabled.
// You can avoid the cost of evaluation of arguments passed to Debug like:
// if logger.DebugEnabled() { logger.Debug(ltsvlog.LTV{"label1": "value1"}) }
func (l *LTSVLogger) DebugEnabled() bool {
	return l.debugEnabled
}

// Debug writes a log with the debug level if the debug level is enabled.
func (l *LTSVLogger) Debug(lv ...LV) {
	if l.debugEnabled {
		l.mu.Lock()
		l.log("Debug", lv...)
		l.mu.Unlock()
	}
}

// Info writes a log with the info level.
func (l *LTSVLogger) Info(lv ...LV) {
	l.mu.Lock()
	l.log("Info", lv...)
	l.mu.Unlock()
}

// Error writes a log with the error level.
func (l *LTSVLogger) Error(lv ...LV) {
	l.mu.Lock()
	l.log("Error", lv...)
	l.mu.Unlock()
}

// ErrorWithStack writes a log and a stack with the error level.
func (l *LTSVLogger) ErrorWithStack(lv ...LV) {
	l.mu.Lock()
	args := lv
	args = append(args, LV{"stack", stack(2, l.stackBuf)})
	l.log("Error", args...)
	l.mu.Unlock()
}

func (l *LTSVLogger) log(level string, lv ...LV) {
	// Note: To reuse the buffer, create an empty slice pointing to
	// the previously allocated buffer.
	buf := l.appendPrefixFunc(l.buf[:0], level)
	for i, labelAndVal := range lv {
		if i > 0 {
			buf = append(buf, '\t')
		}
		buf = append(buf, []byte(labelAndVal.L)...)
		buf = append(buf, ':')
		buf = l.appendValueFunc(buf, labelAndVal.V)
	}
	buf = append(buf, '\n')
	_, _ = l.writer.Write(buf)
	l.buf = buf
}

func appendPrefixFunc(timeLabel, levelLabel string) AppendPrefixFunc {
	if timeLabel != "" && levelLabel != "" {
		timeLabelBytes := []byte(timeLabel + ":")
		levelLabelBytes := []byte("\t" + levelLabel + ":")
		return func(buf []byte, level string) []byte {
			buf = append(buf, timeLabelBytes...)
			now := time.Now().UTC()
			buf = appendTime(buf, now)
			buf = append(buf, levelLabelBytes...)
			buf = append(buf, []byte(level)...)
			buf = append(buf, '\t')
			return buf
		}
	} else if timeLabel != "" && levelLabel == "" {
		timeLabelBytes := []byte(timeLabel + ":")
		return func(buf []byte, level string) []byte {
			buf = append(buf, timeLabelBytes...)
			now := time.Now().UTC()
			buf = appendTime(buf, now)
			buf = append(buf, '\t')
			return buf
		}
	} else if timeLabel == "" && levelLabel != "" {
		levelLabelBytes := []byte(levelLabel + ":")
		return func(buf []byte, level string) []byte {
			buf = append(buf, levelLabelBytes...)
			buf = append(buf, []byte(level)...)
			buf = append(buf, '\t')
			return buf
		}
	} else {
		return func(buf []byte, level string) []byte {
			return buf
		}
	}
}

func appendPrefix(buf []byte, level string) []byte {
	buf = append(buf, "time:"...)
	now := time.Now().UTC()
	buf = appendTime(buf, now)
	buf = append(buf, "\tlevel:"...)
	buf = append(buf, []byte(level)...)
	buf = append(buf, '\t')
	return buf
}

func appendTime(buf []byte, t time.Time) []byte {
	buf = appendZeroPaddedInt(buf, t.Year(), 1000)
	buf = append(buf, byte('-'))
	buf = appendZeroPaddedInt(buf, int(t.Month()), 10)
	buf = append(buf, byte('-'))
	buf = appendZeroPaddedInt(buf, t.Day(), 10)
	buf = append(buf, byte('T'))
	buf = appendZeroPaddedInt(buf, t.Hour(), 10)
	buf = append(buf, byte(':'))
	buf = appendZeroPaddedInt(buf, t.Minute(), 10)
	buf = append(buf, byte(':'))
	buf = appendZeroPaddedInt(buf, t.Second(), 10)
	buf = append(buf, byte('.'))
	buf = appendZeroPaddedInt(buf, t.Nanosecond(), 100000000)
	return append(buf, byte('Z'))
}

var escaper = strings.NewReplacer("\t", "\\t", "\n", "\\n")

func escape(s string) string {
	return escaper.Replace(s)
}

func appendValue(buf []byte, v interface{}) []byte {
	// NOTE: In type switch case, case byte and case uint8 cannot coexist,
	// case rune and case uint cannot coexist.
	switch v.(type) {
	case nil:
		buf = append(buf, "<nil>"...)
	case string:
		buf = append(buf, []byte(escape(v.(string)))...)
	case int:
		buf = strconv.AppendInt(buf, int64(v.(int)), 10)
	case uint:
		buf = strconv.AppendUint(buf, uint64(v.(uint)), 10)
	case int8:
		buf = strconv.AppendInt(buf, int64(v.(int8)), 10)
	case int16:
		buf = strconv.AppendInt(buf, int64(v.(int16)), 10)
	case int32:
		buf = strconv.AppendInt(buf, int64(v.(int32)), 10)
	case int64:
		buf = strconv.AppendInt(buf, v.(int64), 10)
	case uint8:
		buf = strconv.AppendUint(buf, uint64(v.(uint8)), 10)
	case uint16:
		buf = strconv.AppendUint(buf, uint64(v.(uint16)), 10)
	case uint32:
		buf = strconv.AppendUint(buf, uint64(v.(uint32)), 10)
	case uint64:
		buf = strconv.AppendUint(buf, uint64(v.(uint64)), 10)
	case float32:
		buf = append(buf, []byte(strconv.FormatFloat(float64(v.(float32)), 'g', -1, 32))...)
	case float64:
		buf = append(buf, []byte(strconv.FormatFloat(v.(float64), 'g', -1, 64))...)
	case bool:
		buf = strconv.AppendBool(buf, v.(bool))
	case uintptr:
		buf = strconv.AppendUint(buf, uint64(v.(uintptr)), 10)
	case []byte:
		buf = appendHexBytes(buf, v.([]byte))
	case fmt.Stringer:
		buf = append(buf, []byte(escape(v.(fmt.Stringer).String()))...)
	default:
		buf = append(buf, []byte(escape(fmt.Sprintf("%+v", v)))...)
	}
	return buf
}

var digits = []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}

func appendHexBytes(buf []byte, v []byte) []byte {
	buf = append(buf, "0x"...)
	for _, b := range v {
		buf = append(buf, digits[b/16])
		buf = append(buf, digits[b%16])
	}
	return buf
}

func appendZeroPaddedInt(buf []byte, v, p int) []byte {
	for ; p > 1; p /= 10 {
		q := v / p
		buf = append(buf, digits[q])
		v -= q * p
	}
	return append(buf, digits[v])
}

// Logger is the global logger.
// You can change this logger like
// ltsvlog.Logger = ltsvlog.NewLTSVLogger(os.Stdout, false)
// You can change the global logger safely only before writing
// to the logger. Chaging the logger while writing may cause
// the unexpected behavior.
var Logger = NewLTSVLogger(os.Stdout, true)

// Discard discards any logging outputs.
type Discard struct{}

// DebugEnabled always return false
func (*Discard) DebugEnabled() bool { return false }

// Debug prints nothing
func (*Discard) Debug(lv ...LV) {}

// Info prints nothing
func (*Discard) Info(lv ...LV) {}

// Error prints nothing
func (*Discard) Error(lv ...LV) {}

// ErrorWithStack prints nothing
func (*Discard) ErrorWithStack(lv ...LV) {}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strings"
)

// These are the environmental variables that determine if we log, and if
// we log whether or not the log should go to a file.
const EnvLog = "PACKER_LOG"          //Set to True
const EnvLogFile = "PACKER_LOG_PATH" //Set to a file

// logOutput determines where we should send logs (if anywhere).
func logOutput() (logOutput io.Writer, err error) {
	logOutput = nil
	if os.Getenv(EnvLog) != "" && os.Getenv(EnvLog) != "0" {
		logOutput = os.Stderr

		if logPath := os.Getenv(EnvLogFile); logPath != "" {
			var err error
			logOutput, err = os.Create(logPath)
			if err != nil {
				return nil, err
			}
		} else {
			// no path; do a little light filtering to avoid double-dipping UI
			// calls.
			r, w := io.Pipe()
			scanner := bufio.NewScanner(r)
			go func(scanner *bufio.Scanner) {
				for scanner.Scan() {
					if strings.Contains(scanner.Text(), "ui:") {
						continue
					}
					os.Stderr.WriteString(fmt.Sprint(scanner.Text() + "\n"))
				}
			}(scanner)
			logOutput = w
		}
	}

	return
}
<commit_msg>deduplicate loglines that stream both to ui ERROR call and to streaming logs when PACKER_LOG=1<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strings"
)

// These are the environmental variables that determine if we log, and if
// we log whether or not the log should go to a file.
const EnvLog = "PACKER_LOG"          //Set to True
const EnvLogFile = "PACKER_LOG_PATH" //Set to a file

// logOutput determines where we should send logs (if anywhere).
func logOutput() (logOutput io.Writer, err error) {
	logOutput = nil
	if os.Getenv(EnvLog) != "" && os.Getenv(EnvLog) != "0" {
		logOutput = os.Stderr

		if logPath := os.Getenv(EnvLogFile); logPath != "" {
			var err error
			logOutput, err = os.Create(logPath)
			if err != nil {
				return nil, err
			}
		} else {
			// no path; do a little light filtering to avoid double-dipping UI
			// calls.
			r, w := io.Pipe()
			scanner := bufio.NewScanner(r)
			go func(scanner *bufio.Scanner) {
				for scanner.Scan() {
					if strings.Contains(scanner.Text(), "ui:") {
						continue
					}
					if strings.Contains(scanner.Text(), "ui error:") {
						continue
					}
					os.Stderr.WriteString(fmt.Sprint(scanner.Text() + "\n"))
				}
			}(scanner)
			logOutput = w
		}
	}

	return
}
<|endoftext|>


<commit_before>package builds

import (
	"context"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	buildv1 "github.com/openshift/api/build/v1"
	exutil "github.com/openshift/origin/test/extended/util"
)

func Before(oc *exutil.CLI) {
	exutil.PreTestDump()
}

func After(oc *exutil.CLI) {
	if g.CurrentGinkgoTestDescription().Failed {
		exutil.DumpPodStates(oc)
		exutil.DumpConfigMapStates(oc)
		exutil.DumpPodLogsStartingWith("", oc)
	}
}

var _ = g.Describe("[sig-builds][Feature:Builds] s2i build with a root user image", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLI("s2i-build-root")

	g.It("should create a root build and fail without a privileged SCC", func() {
		g.Skip("TODO: figure out why we aren't properly denying this, also consider whether we still need to deny it")
		Before(oc)
		defer After(oc)

		err := oc.Run("new-app").Args("docker.io/openshift/test-build-roots2i~https://github.com/sclorg/nodejs-ex", "--name", "nodejsfail").Execute()
		o.Expect(err).NotTo(o.HaveOccurred())

		err = exutil.WaitForABuild(oc.BuildClient().BuildV1().Builds(oc.Namespace()), "nodejsfail-1", nil, nil, nil)
		o.Expect(err).To(o.HaveOccurred())

		build, err := oc.BuildClient().BuildV1().Builds(oc.Namespace()).Get(context.Background(), "nodejsfail-1", metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		o.Expect(build.Status.Phase).To(o.Equal(buildv1.BuildPhaseFailed))
		o.Expect(build.Status.Reason).To(o.BeEquivalentTo("PullBuilderImageFailed" /*s2istatus.ReasonPullBuilderImageFailed*/))
		o.Expect(build.Status.Message).To(o.BeEquivalentTo("Failed to pull builder image." /*s2istatus.ReasonMessagePullBuilderImageFailed*/))

		podname := build.Annotations[buildv1.BuildPodNameAnnotation]
		pod, err := oc.KubeClient().CoreV1().Pods(oc.Namespace()).Get(context.Background(), podname, metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())

		containers := make([]corev1.Container, len(pod.Spec.Containers)+len(pod.Spec.InitContainers))
		copy(containers, pod.Spec.Containers)
		copy(containers[len(pod.Spec.Containers):], pod.Spec.InitContainers)

		for _, c := range containers {
			env := map[string]string{}
			for _, e := range c.Env {
				env[e.Name] = e.Value
			}
			o.Expect(env["DROP_CAPS"]).To(o.Equal("KILL,MKNOD,SETGID,SETUID"))
			o.Expect(env["ALLOWED_UIDS"]).To(o.Equal("1-"))
		}
	})

	g.It("should create a root build and pass with a privileged SCC", func() {
		Before(oc)
		defer After(oc)
		g.By("adding builder account to privileged SCC")
		// create a namespace local role and role binding to the privileged SCC;
		// role and role binding will be deleted as part of test namespace clean up
		role := &rbacv1.Role{}
		role.Namespace = oc.Namespace()
		role.Name = "privileged-builder-role"
		role.Rules = []rbacv1.PolicyRule{
			{
				APIGroups: []string{
					"security.openshift.io",
				},
				ResourceNames: []string{
					"privileged",
				},
				Resources: []string{
					"securitycontextconstraints",
				},
				Verbs: []string{
					"use",
				},
			},
		}
		role, err := oc.AdminKubeClient().RbacV1().Roles(oc.Namespace()).Create(context.Background(), role, metav1.CreateOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		roleBinding := &rbacv1.RoleBinding{}
		roleBinding.Namespace = oc.Namespace()
		roleBinding.Name = "privileged-builder-rolebinding"
		roleBinding.RoleRef = rbacv1.RoleRef{
			APIGroup: "rbac.authorization.k8s.io",
			Kind:     "Role",
			Name:     "privileged-builder-role",
		}
		roleBinding.Subjects = []rbacv1.Subject{
			{
				Kind:      "ServiceAccount",
				APIGroup:  "",
				Name:      "builder",
				Namespace: oc.Namespace(),
			},
		}
		roleBinding, err = oc.AdminKubeClient().RbacV1().RoleBindings(oc.Namespace()).Create(context.Background(), roleBinding, metav1.CreateOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())

		err = oc.Run("new-build").Args("docker.io/openshift/test-build-roots2i~https://github.com/sclorg/nodejs-ex", "--name", "nodejspass").Execute()
		o.Expect(err).NotTo(o.HaveOccurred())

		err = exutil.WaitForABuild(oc.BuildClient().BuildV1().Builds(oc.Namespace()), "nodejspass-1", nil, nil, nil)
		o.Expect(err).NotTo(o.HaveOccurred())

		build, err := oc.BuildClient().BuildV1().Builds(oc.Namespace()).Get(context.Background(), "nodejspass-1", metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())

		podname := build.Annotations[buildv1.BuildPodNameAnnotation]
		pod, err := oc.KubeClient().CoreV1().Pods(oc.Namespace()).Get(context.Background(), podname, metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())

		containers := make([]corev1.Container, len(pod.Spec.Containers)+len(pod.Spec.InitContainers))
		copy(containers, pod.Spec.Containers)
		copy(containers[len(pod.Spec.Containers):], pod.Spec.InitContainers)

		for _, c := range containers {
			env := map[string]string{}
			for _, e := range c.Env {
				env[e.Name] = e.Value
			}
			o.Expect(env).NotTo(o.HaveKey("DROP_CAPS"))
			o.Expect(env).NotTo(o.HaveKey("ALLOWED_UIDS"))
		}
	})
})
<commit_msg>update s2i_root.go to use CI image that will be mirrored to quay.io/community-images<commit_after>package builds

import (
	"context"
	"fmt"
	"github.com/openshift/origin/test/extended/util/image"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	buildv1 "github.com/openshift/api/build/v1"
	exutil "github.com/openshift/origin/test/extended/util"
)

func Before(oc *exutil.CLI) {
	exutil.PreTestDump()
}

func After(oc *exutil.CLI) {
	if g.CurrentGinkgoTestDescription().Failed {
		exutil.DumpPodStates(oc)
		exutil.DumpConfigMapStates(oc)
		exutil.DumpPodLogsStartingWith("", oc)
	}
}

var _ = g.Describe("[sig-builds][Feature:Builds] s2i build with a root user image", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLI("s2i-build-root")

	g.It("should create a root build and fail without a privileged SCC", func() {
		g.Skip("TODO: figure out why we aren't properly denying this, also consider whether we still need to deny it")
		Before(oc)
		defer After(oc)

		firstArgString := fmt.Sprintf("%s~https://github.com/sclorg/nodejs-ex", image.LocationFor("registry.svc.ci.openshift.org/ocp/4.7:test-build-roots2i"))
		err := oc.Run("new-app").Args(firstArgString, "--name", "nodejsfail").Execute()
		o.Expect(err).NotTo(o.HaveOccurred())

		err = exutil.WaitForABuild(oc.BuildClient().BuildV1().Builds(oc.Namespace()), "nodejsfail-1", nil, nil, nil)
		o.Expect(err).To(o.HaveOccurred())

		build, err := oc.BuildClient().BuildV1().Builds(oc.Namespace()).Get(context.Background(), "nodejsfail-1", metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		o.Expect(build.Status.Phase).To(o.Equal(buildv1.BuildPhaseFailed))
		o.Expect(build.Status.Reason).To(o.BeEquivalentTo("PullBuilderImageFailed" /*s2istatus.ReasonPullBuilderImageFailed*/))
		o.Expect(build.Status.Message).To(o.BeEquivalentTo("Failed to pull builder image." /*s2istatus.ReasonMessagePullBuilderImageFailed*/))

		podname := build.Annotations[buildv1.BuildPodNameAnnotation]
		pod, err := oc.KubeClient().CoreV1().Pods(oc.Namespace()).Get(context.Background(), podname, metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())

		containers := make([]corev1.Container, len(pod.Spec.Containers)+len(pod.Spec.InitContainers))
		copy(containers, pod.Spec.Containers)
		copy(containers[len(pod.Spec.Containers):], pod.Spec.InitContainers)

		for _, c := range containers {
			env := map[string]string{}
			for _, e := range c.Env {
				env[e.Name] = e.Value
			}
			o.Expect(env["DROP_CAPS"]).To(o.Equal("KILL,MKNOD,SETGID,SETUID"))
			o.Expect(env["ALLOWED_UIDS"]).To(o.Equal("1-"))
		}
	})

	g.It("should create a root build and pass with a privileged SCC", func() {
		Before(oc)
		defer After(oc)
		g.By("adding builder account to privileged SCC")
		// create a namespace local role and role binding to the privileged SCC;
		// role and role binding will be deleted as part of test namespace clean up
		role := &rbacv1.Role{}
		role.Namespace = oc.Namespace()
		role.Name = "privileged-builder-role"
		role.Rules = []rbacv1.PolicyRule{
			{
				APIGroups: []string{
					"security.openshift.io",
				},
				ResourceNames: []string{
					"privileged",
				},
				Resources: []string{
					"securitycontextconstraints",
				},
				Verbs: []string{
					"use",
				},
			},
		}
		role, err := oc.AdminKubeClient().RbacV1().Roles(oc.Namespace()).Create(context.Background(), role, metav1.CreateOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		roleBinding := &rbacv1.RoleBinding{}
		roleBinding.Namespace = oc.Namespace()
		roleBinding.Name = "privileged-builder-rolebinding"
		roleBinding.RoleRef = rbacv1.RoleRef{
			APIGroup: "rbac.authorization.k8s.io",
			Kind:     "Role",
			Name:     "privileged-builder-role",
		}
		roleBinding.Subjects = []rbacv1.Subject{
			{
				Kind:      "ServiceAccount",
				APIGroup:  "",
				Name:      "builder",
				Namespace: oc.Namespace(),
			},
		}
		roleBinding, err = oc.AdminKubeClient().RbacV1().RoleBindings(oc.Namespace()).Create(context.Background(), roleBinding, metav1.CreateOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())

		firstArgString := fmt.Sprintf("%s~https://github.com/sclorg/nodejs-ex", image.LocationFor("registry.svc.ci.openshift.org/ocp/4.7:test-build-roots2i"))
		err = oc.Run("new-build").Args(firstArgString, "--name", "nodejspass").Execute()
		o.Expect(err).NotTo(o.HaveOccurred())

		err = exutil.WaitForABuild(oc.BuildClient().BuildV1().Builds(oc.Namespace()), "nodejspass-1", nil, nil, nil)
		o.Expect(err).NotTo(o.HaveOccurred())

		build, err := oc.BuildClient().BuildV1().Builds(oc.Namespace()).Get(context.Background(), "nodejspass-1", metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())

		podname := build.Annotations[buildv1.BuildPodNameAnnotation]
		pod, err := oc.KubeClient().CoreV1().Pods(oc.Namespace()).Get(context.Background(), podname, metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())

		containers := make([]corev1.Container, len(pod.Spec.Containers)+len(pod.Spec.InitContainers))
		copy(containers, pod.Spec.Containers)
		copy(containers[len(pod.Spec.Containers):], pod.Spec.InitContainers)

		for _, c := range containers {
			env := map[string]string{}
			for _, e := range c.Env {
				env[e.Name] = e.Value
			}
			o.Expect(env).NotTo(o.HaveKey("DROP_CAPS"))
			o.Expect(env).NotTo(o.HaveKey("ALLOWED_UIDS"))
		}
	})
})
<|endoftext|>