<commit_before>// Copyright (c) 2012-2016 The Revel Framework Authors, All rights reserved.
// Revel Framework source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package revel

import (
	"io"
	"net/http"
	"strconv"
	"strings"

	"github.com/klauspost/compress/gzip"
	"github.com/klauspost/compress/zlib"
)

var compressionTypes = [...]string{
	"gzip",
	"deflate",
}

var compressableMimes = [...]string{
	"text/plain",
	"text/html",
	"text/xml",
	"text/css",
	"application/json",
	"application/xml",
	"application/xhtml+xml",
	"application/rss+xml",
	"application/javascript",
	"application/x-javascript",
}

// WriteFlusher interface for compress writer
type WriteFlusher interface {
	io.Writer
	io.Closer
	Flush() error
}

type CompressResponseWriter struct {
	Header             *BufferedServerHeader
	ControllerResponse *Response
	OriginalWriter     io.Writer
	compressWriter     WriteFlusher
	compressionType    string
	headersWritten     bool
	closeNotify        chan bool
	parentNotify       <-chan bool
	closed             bool
}

// CompressFilter does compression of response body in gzip/deflate if
// `results.compressed=true` in the app.conf
func CompressFilter(c *Controller, fc []Filter) {
	if c.Response.Out.internalHeader.Server != nil && Config.BoolDefault("results.compressed", false) {
		if c.Response.Status != http.StatusNoContent && c.Response.Status != http.StatusNotModified {
			if found, compressType, compressWriter := detectCompressionType(c.Request, c.Response); found {
				writer := CompressResponseWriter{
					ControllerResponse: c.Response,
					OriginalWriter:     c.Response.GetWriter(),
					compressWriter:     compressWriter,
					compressionType:    compressType,
					headersWritten:     false,
					closeNotify:        make(chan bool, 1),
					closed:             false}
				// Swap out the header with our own
				writer.Header = NewBufferedServerHeader(c.Response.Out.internalHeader.Server)
				c.Response.Out.internalHeader.Server = writer.Header
				if w, ok := c.Response.GetWriter().(http.CloseNotifier); ok {
					writer.parentNotify = w.CloseNotify()
				}
				c.Response.SetWriter(&writer)
			}
		} else {
			TRACE.Printf("Compression disabled for response status (%d)", c.Response.Status)
		}
	}
	fc[0](c, fc[1:])
}

func (c CompressResponseWriter) CloseNotify() <-chan bool {
	if c.parentNotify != nil {
		return c.parentNotify
	}
	return c.closeNotify
}

func (c *CompressResponseWriter) prepareHeaders() {
	if c.compressionType != "" {
		responseMime := ""
		if t := c.Header.Get("Content-Type"); len(t)>0 {
			responseMime = t[0]
		}
		responseMime = strings.TrimSpace(strings.SplitN(responseMime, ";", 2)[0])
		shouldEncode := false

		if len(c.Header.Get("Content-Encoding")) == 0 {
			for _, compressableMime := range compressableMimes {
				if responseMime == compressableMime {
					shouldEncode = true
					c.Header.Set("Content-Encoding", c.compressionType)
					c.Header.Del("Content-Length")
					break
				}
			}
		}

		if !shouldEncode {
			c.compressWriter = nil
			c.compressionType = ""
		}
	}
	c.Header.Release()
}

func (c *CompressResponseWriter) WriteHeader(status int) {
	c.headersWritten = true
	c.prepareHeaders()
	c.Header.SetStatus(status)
}

func (c *CompressResponseWriter) Close() error {
	if !c.headersWritten {
		c.prepareHeaders()
	}
	if c.compressionType != "" {
		c.Header.Del("Content-Length")
		if err := c.compressWriter.Close(); err != nil {
			// TODO When writing directly to stream, an error will be generated
			ERROR.Println("Error closing compress writer", c.compressionType, err)
		}

	}
	// Non-blocking write to the closenotifier, if we for some reason should
	// get called multiple times
	select {
	case c.closeNotify <- true:
	default:
	}
	c.closed = true
	return nil
}

func (c *CompressResponseWriter) Write(b []byte) (int, error) {
	// Abort if parent has been closed
	if c.parentNotify != nil {
		select {
		case <-c.parentNotify:
			return 0, io.ErrClosedPipe
		default:
		}
	}
	// Abort if we ourselves have been closed
	if c.closed {
		return 0, io.ErrClosedPipe
	}

	if !c.headersWritten {
		c.prepareHeaders()
		c.headersWritten = true
	}
	if c.compressionType != "" {
		return c.compressWriter.Write(b)
	}
	return c.OriginalWriter.Write(b)
}

// DetectCompressionType method detects the compression type
// from header "Accept-Encoding"
func detectCompressionType(req *Request, resp *Response) (found bool, compressionType string, compressionKind WriteFlusher) {
	if Config.BoolDefault("results.compressed", false) {
		acceptedEncodings := strings.Split(req.GetHttpHeader("Accept-Encoding"), ",")

		largestQ := 0.0
		chosenEncoding := len(compressionTypes)

		// I have fixed one edge case for issue #914
		// But it's better to cover all possible edge cases or
		// Adapt to https://github.com/golang/gddo/blob/master/httputil/header/header.go#L172
		for _, encoding := range acceptedEncodings {
			encoding = strings.TrimSpace(encoding)
			encodingParts := strings.SplitN(encoding, ";", 2)

			// If we are the format "gzip;q=0.8"
			if len(encodingParts) > 1 {
				q := strings.TrimSpace(encodingParts[1])
				if len(q) == 0 || !strings.HasPrefix(q, "q=") {
					continue
				}

				// Strip off the q=
				num, err := strconv.ParseFloat(q[2:], 32)
				if err != nil {
					continue
				}

				if num >= largestQ && num > 0 {
					if encodingParts[0] == "*" {
						chosenEncoding = 0
						largestQ = num
						continue
					}
					for i, encoding := range compressionTypes {
						if encoding == encodingParts[0] {
							if i < chosenEncoding {
								largestQ = num
								chosenEncoding = i
							}
							break
						}
					}
				}
			} else {
				// If we can accept anything, chose our preferred method.
				if encodingParts[0] == "*" {
					chosenEncoding = 0
					largestQ = 1
					break
				}
				// This is for just plain "gzip"
				for i, encoding := range compressionTypes {
					if encoding == encodingParts[0] {
						if i < chosenEncoding {
							largestQ = 1.0
							chosenEncoding = i
						}
						break
					}
				}
			}
		}

		if largestQ == 0 {
			return
		}

		compressionType = compressionTypes[chosenEncoding]

		switch compressionType {
		case "gzip":
			compressionKind = gzip.NewWriter(resp.GetWriter())
			found = true
		case "deflate":
			compressionKind = zlib.NewWriter(resp.GetWriter())
			found = true
		}
	}
	return
}

// BufferedServerHeader will not send content out until the Released is called, from that point on it will act normally
// It implements all the ServerHeader
type BufferedServerHeader struct {
	cookieList []string
	headerMap  map[string][]string
	status     int
	released   bool
	original   ServerHeader
}

func NewBufferedServerHeader(o ServerHeader) *BufferedServerHeader {
	return &BufferedServerHeader{original: o, headerMap: map[string][]string{}}
}
func (bsh *BufferedServerHeader) SetCookie(cookie string) {
	if bsh.released {
		bsh.original.SetCookie(cookie)
	} else {
		bsh.cookieList = append(bsh.cookieList, cookie)
	}
}
func (bsh *BufferedServerHeader) GetCookie(key string) (value ServerCookie, err error) {
	return bsh.original.GetCookie(key)
}
func (bsh *BufferedServerHeader) Set(key string, value string) {
	if bsh.released {
		bsh.original.Set(key, value)
	} else {
		bsh.headerMap[key] = []string{value}
	}
}
func (bsh *BufferedServerHeader) Add(key string, value string) {
	if bsh.released {
		bsh.original.Set(key, value)
	} else {
		old := []string{}
		if v, found := bsh.headerMap[key]; found {
			old = v
		}
		bsh.headerMap[key] = append(old, value)
	}

}
func (bsh *BufferedServerHeader) Del(key string) {
	if bsh.released {
		bsh.original.Del(key)
	} else {
		delete(bsh.headerMap, key)
	}

}
func (bsh *BufferedServerHeader) Get(key string) (value []string) {
	if bsh.released {
		value = bsh.original.Get(key)
	} else {
		if v, found := bsh.headerMap[key]; found && len(v) > 0 {
			value = v
		} else {
			value = bsh.original.Get(key)
		}
	}
	return
}
func (bsh *BufferedServerHeader) SetStatus(statusCode int) {
	if bsh.released {
		bsh.original.SetStatus(statusCode)
	} else {
		bsh.status = statusCode
	}
}
func (bsh *BufferedServerHeader) Release() {
	bsh.released = true
	bsh.original.SetStatus(bsh.status)
	for k, v := range bsh.headerMap {
		for _, r := range v {
			bsh.original.Set(k, r)
		}
	}
	for _, c := range bsh.cookieList {
		bsh.original.SetCookie(c)
	}
}
<commit_msg>Formatted file spacing<commit_after>// Copyright (c) 2012-2016 The Revel Framework Authors, All rights reserved.
// Revel Framework source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package revel

import (
	"io"
	"net/http"
	"strconv"
	"strings"

	"github.com/klauspost/compress/gzip"
	"github.com/klauspost/compress/zlib"
)

var compressionTypes = [...]string{
	"gzip",
	"deflate",
}

var compressableMimes = [...]string{
	"text/plain",
	"text/html",
	"text/xml",
	"text/css",
	"application/json",
	"application/xml",
	"application/xhtml+xml",
	"application/rss+xml",
	"application/javascript",
	"application/x-javascript",
}

// WriteFlusher interface for compress writer
type WriteFlusher interface {
	io.Writer
	io.Closer
	Flush() error
}

type CompressResponseWriter struct {
	Header             *BufferedServerHeader
	ControllerResponse *Response
	OriginalWriter     io.Writer
	compressWriter     WriteFlusher
	compressionType    string
	headersWritten     bool
	closeNotify        chan bool
	parentNotify       <-chan bool
	closed             bool
}

// CompressFilter does compression of response body in gzip/deflate if
// `results.compressed=true` in the app.conf
func CompressFilter(c *Controller, fc []Filter) {
	if c.Response.Out.internalHeader.Server != nil && Config.BoolDefault("results.compressed", false) {
		if c.Response.Status != http.StatusNoContent && c.Response.Status != http.StatusNotModified {
			if found, compressType, compressWriter := detectCompressionType(c.Request, c.Response); found {
				writer := CompressResponseWriter{
					ControllerResponse: c.Response,
					OriginalWriter:     c.Response.GetWriter(),
					compressWriter:     compressWriter,
					compressionType:    compressType,
					headersWritten:     false,
					closeNotify:        make(chan bool, 1),
					closed:             false,
				}
				// Swap out the header with our own
				writer.Header = NewBufferedServerHeader(c.Response.Out.internalHeader.Server)
				c.Response.Out.internalHeader.Server = writer.Header
				if w, ok := c.Response.GetWriter().(http.CloseNotifier); ok {
					writer.parentNotify = w.CloseNotify()
				}
				c.Response.SetWriter(&writer)
			}
		} else {
			TRACE.Printf("Compression disabled for response status (%d)", c.Response.Status)
		}
	}
	fc[0](c, fc[1:])
}

func (c CompressResponseWriter) CloseNotify() <-chan bool {
	if c.parentNotify != nil {
		return c.parentNotify
	}
	return c.closeNotify
}

func (c *CompressResponseWriter) prepareHeaders() {
	if c.compressionType != "" {
		responseMime := ""
		if t := c.Header.Get("Content-Type"); len(t) > 0 {
			responseMime = t[0]
		}
		responseMime = strings.TrimSpace(strings.SplitN(responseMime, ";", 2)[0])
		shouldEncode := false

		if len(c.Header.Get("Content-Encoding")) == 0 {
			for _, compressableMime := range compressableMimes {
				if responseMime == compressableMime {
					shouldEncode = true
					c.Header.Set("Content-Encoding", c.compressionType)
					c.Header.Del("Content-Length")
					break
				}
			}
		}

		if !shouldEncode {
			c.compressWriter = nil
			c.compressionType = ""
		}
	}
	c.Header.Release()
}

func (c *CompressResponseWriter) WriteHeader(status int) {
	c.headersWritten = true
	c.prepareHeaders()
	c.Header.SetStatus(status)
}

func (c *CompressResponseWriter) Close() error {
	if !c.headersWritten {
		c.prepareHeaders()
	}
	if c.compressionType != "" {
		c.Header.Del("Content-Length")
		if err := c.compressWriter.Close(); err != nil {
			// TODO When writing directly to stream, an error will be generated
			ERROR.Println("Error closing compress writer", c.compressionType, err)
		}

	}
	// Non-blocking write to the closenotifier, if we for some reason should
	// get called multiple times
	select {
	case c.closeNotify <- true:
	default:
	}
	c.closed = true
	return nil
}

func (c *CompressResponseWriter) Write(b []byte) (int, error) {
	// Abort if parent has been closed
	if c.parentNotify != nil {
		select {
		case <-c.parentNotify:
			return 0, io.ErrClosedPipe
		default:
		}
	}
	// Abort if we ourselves have been closed
	if c.closed {
		return 0, io.ErrClosedPipe
	}

	if !c.headersWritten {
		c.prepareHeaders()
		c.headersWritten = true
	}
	if c.compressionType != "" {
		return c.compressWriter.Write(b)
	}
	return c.OriginalWriter.Write(b)
}

// DetectCompressionType method detects the compression type
// from header "Accept-Encoding"
func detectCompressionType(req *Request, resp *Response) (found bool, compressionType string, compressionKind WriteFlusher) {
	if Config.BoolDefault("results.compressed", false) {
		acceptedEncodings := strings.Split(req.GetHttpHeader("Accept-Encoding"), ",")

		largestQ := 0.0
		chosenEncoding := len(compressionTypes)

		// I have fixed one edge case for issue #914
		// But it's better to cover all possible edge cases or
		// Adapt to https://github.com/golang/gddo/blob/master/httputil/header/header.go#L172
		for _, encoding := range acceptedEncodings {
			encoding = strings.TrimSpace(encoding)
			encodingParts := strings.SplitN(encoding, ";", 2)

			// If we are the format "gzip;q=0.8"
			if len(encodingParts) > 1 {
				q := strings.TrimSpace(encodingParts[1])
				if len(q) == 0 || !strings.HasPrefix(q, "q=") {
					continue
				}

				// Strip off the q=
				num, err := strconv.ParseFloat(q[2:], 32)
				if err != nil {
					continue
				}

				if num >= largestQ && num > 0 {
					if encodingParts[0] == "*" {
						chosenEncoding = 0
						largestQ = num
						continue
					}
					for i, encoding := range compressionTypes {
						if encoding == encodingParts[0] {
							if i < chosenEncoding {
								largestQ = num
								chosenEncoding = i
							}
							break
						}
					}
				}
			} else {
				// If we can accept anything, chose our preferred method.
				if encodingParts[0] == "*" {
					chosenEncoding = 0
					largestQ = 1
					break
				}
				// This is for just plain "gzip"
				for i, encoding := range compressionTypes {
					if encoding == encodingParts[0] {
						if i < chosenEncoding {
							largestQ = 1.0
							chosenEncoding = i
						}
						break
					}
				}
			}
		}

		if largestQ == 0 {
			return
		}

		compressionType = compressionTypes[chosenEncoding]

		switch compressionType {
		case "gzip":
			compressionKind = gzip.NewWriter(resp.GetWriter())
			found = true
		case "deflate":
			compressionKind = zlib.NewWriter(resp.GetWriter())
			found = true
		}
	}
	return
}

// BufferedServerHeader will not send content out until the Released is called, from that point on it will act normally
// It implements all the ServerHeader
type BufferedServerHeader struct {
	cookieList []string
	headerMap  map[string][]string
	status     int
	released   bool
	original   ServerHeader
}

func NewBufferedServerHeader(o ServerHeader) *BufferedServerHeader {
	return &BufferedServerHeader{original: o, headerMap: map[string][]string{}}
}
func (bsh *BufferedServerHeader) SetCookie(cookie string) {
	if bsh.released {
		bsh.original.SetCookie(cookie)
	} else {
		bsh.cookieList = append(bsh.cookieList, cookie)
	}
}
func (bsh *BufferedServerHeader) GetCookie(key string) (value ServerCookie, err error) {
	return bsh.original.GetCookie(key)
}
func (bsh *BufferedServerHeader) Set(key string, value string) {
	if bsh.released {
		bsh.original.Set(key, value)
	} else {
		bsh.headerMap[key] = []string{value}
	}
}
func (bsh *BufferedServerHeader) Add(key string, value string) {
	if bsh.released {
		bsh.original.Set(key, value)
	} else {
		old := []string{}
		if v, found := bsh.headerMap[key]; found {
			old = v
		}
		bsh.headerMap[key] = append(old, value)
	}

}
func (bsh *BufferedServerHeader) Del(key string) {
	if bsh.released {
		bsh.original.Del(key)
	} else {
		delete(bsh.headerMap, key)
	}

}
func (bsh *BufferedServerHeader) Get(key string) (value []string) {
	if bsh.released {
		value = bsh.original.Get(key)
	} else {
		if v, found := bsh.headerMap[key]; found && len(v) > 0 {
			value = v
		} else {
			value = bsh.original.Get(key)
		}
	}
	return
}
func (bsh *BufferedServerHeader) SetStatus(statusCode int) {
	if bsh.released {
		bsh.original.SetStatus(statusCode)
	} else {
		bsh.status = statusCode
	}
}
func (bsh *BufferedServerHeader) Release() {
	bsh.released = true
	bsh.original.SetStatus(bsh.status)
	for k, v := range bsh.headerMap {
		for _, r := range v {
			bsh.original.Set(k, r)
		}
	}
	for _, c := range bsh.cookieList {
		bsh.original.SetCookie(c)
	}
}
<|endoftext|>


<commit_before>package bbolt

import (
	"math/rand"
	"reflect"
	"sort"
	"testing"
	"unsafe"
)

// Ensure that a page is added to a transaction's freelist.
func TestFreelist_free(t *testing.T) {
	f := newFreelist()
	f.free(100, &page{id: 12})
	if !reflect.DeepEqual([]pgid{12}, f.pending[100].ids) {
		t.Fatalf("exp=%v; got=%v", []pgid{12}, f.pending[100])
	}
}

// Ensure that a page and its overflow is added to a transaction's freelist.
func TestFreelist_free_overflow(t *testing.T) {
	f := newFreelist()
	f.free(100, &page{id: 12, overflow: 3})
	if exp := []pgid{12, 13, 14, 15}; !reflect.DeepEqual(exp, f.pending[100].ids) {
		t.Fatalf("exp=%v; got=%v", exp, f.pending[100])
	}
}

// Ensure that a transaction's free pages can be released.
func TestFreelist_release(t *testing.T) {
	f := newFreelist()
	f.free(100, &page{id: 12, overflow: 1})
	f.free(100, &page{id: 9})
	f.free(102, &page{id: 39})
	f.release(100)
	f.release(101)
	if exp := []pgid{9, 12, 13}; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f.getFreePageIDs())
	}

	f.release(102)
	if exp := []pgid{9, 12, 13, 39}; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f.getFreePageIDs())
	}
}

// Ensure that releaseRange handles boundary conditions correctly
func TestFreelist_releaseRange(t *testing.T) {
	type testRange struct {
		begin, end txid
	}

	type testPage struct {
		id       pgid
		n        int
		allocTxn txid
		freeTxn  txid
	}

	var releaseRangeTests = []struct {
		title         string
		pagesIn       []testPage
		releaseRanges []testRange
		wantFree      []pgid
	}{
		{
			title:         "Single pending in range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
			releaseRanges: []testRange{{1, 300}},
			wantFree:      []pgid{3},
		},
		{
			title:         "Single pending with minimum end range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
			releaseRanges: []testRange{{1, 200}},
			wantFree:      []pgid{3},
		},
		{
			title:         "Single pending outsize minimum end range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
			releaseRanges: []testRange{{1, 199}},
			wantFree:      nil,
		},
		{
			title:         "Single pending with minimum begin range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
			releaseRanges: []testRange{{100, 300}},
			wantFree:      []pgid{3},
		},
		{
			title:         "Single pending outside minimum begin range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
			releaseRanges: []testRange{{101, 300}},
			wantFree:      nil,
		},
		{
			title:         "Single pending in minimum range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 199, freeTxn: 200}},
			releaseRanges: []testRange{{199, 200}},
			wantFree:      []pgid{3},
		},
		{
			title:         "Single pending and read transaction at 199",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 199, freeTxn: 200}},
			releaseRanges: []testRange{{100, 198}, {200, 300}},
			wantFree:      nil,
		},
		{
			title: "Adjacent pending and read transactions at 199, 200",
			pagesIn: []testPage{
				{id: 3, n: 1, allocTxn: 199, freeTxn: 200},
				{id: 4, n: 1, allocTxn: 200, freeTxn: 201},
			},
			releaseRanges: []testRange{
				{100, 198},
				{200, 199}, // Simulate the ranges db.freePages might produce.
				{201, 300},
			},
			wantFree: nil,
		},
		{
			title: "Out of order ranges",
			pagesIn: []testPage{
				{id: 3, n: 1, allocTxn: 199, freeTxn: 200},
				{id: 4, n: 1, allocTxn: 200, freeTxn: 201},
			},
			releaseRanges: []testRange{
				{201, 199},
				{201, 200},
				{200, 200},
			},
			wantFree: nil,
		},
		{
			title: "Multiple pending, read transaction at 150",
			pagesIn: []testPage{
				{id: 3, n: 1, allocTxn: 100, freeTxn: 200},
				{id: 4, n: 1, allocTxn: 100, freeTxn: 125},
				{id: 5, n: 1, allocTxn: 125, freeTxn: 150},
				{id: 6, n: 1, allocTxn: 125, freeTxn: 175},
				{id: 7, n: 2, allocTxn: 150, freeTxn: 175},
				{id: 9, n: 2, allocTxn: 175, freeTxn: 200},
			},
			releaseRanges: []testRange{{50, 149}, {151, 300}},
			wantFree:      []pgid{4, 9},
		},
	}

	for _, c := range releaseRangeTests {
		f := newFreelist()
		var ids []pgid
		for _, p := range c.pagesIn {
			for i := uint64(0); i < uint64(p.n); i++ {
				ids = append(ids, pgid(uint64(p.id)+i))
			}
		}
		f.readIDs(ids)
		for _, p := range c.pagesIn {
			f.allocate(p.allocTxn, p.n)
		}

		for _, p := range c.pagesIn {
			f.free(p.freeTxn, &page{id: p.id})
		}

		for _, r := range c.releaseRanges {
			f.releaseRange(r.begin, r.end)
		}

		if exp := c.wantFree; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
			t.Errorf("exp=%v; got=%v for %s", exp, f.getFreePageIDs(), c.title)
		}
	}
}

// Ensure that a freelist can find contiguous blocks of pages.
func TestFreelist_allocate(t *testing.T) {
	f := newFreelist()
	ids := []pgid{3, 4, 5, 6, 7, 9, 12, 13, 18}
	f.readIDs(ids)
	if id := int(f.allocate(1, 3)); id != 3 {
		t.Fatalf("exp=3; got=%v", id)
	}
	if id := int(f.allocate(1, 1)); id != 6 {
		t.Fatalf("exp=6; got=%v", id)
	}
	if id := int(f.allocate(1, 3)); id != 0 {
		t.Fatalf("exp=0; got=%v", id)
	}
	if id := int(f.allocate(1, 2)); id != 12 {
		t.Fatalf("exp=12; got=%v", id)
	}
	if id := int(f.allocate(1, 1)); id != 7 {
		t.Fatalf("exp=7; got=%v", id)
	}
	if id := int(f.allocate(1, 0)); id != 0 {
		t.Fatalf("exp=0; got=%v", id)
	}
	if id := int(f.allocate(1, 0)); id != 0 {
		t.Fatalf("exp=0; got=%v", id)
	}
	if exp := []pgid{9, 18}; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f.getFreePageIDs())
	}

	if id := int(f.allocate(1, 1)); id != 9 {
		t.Fatalf("exp=9; got=%v", id)
	}
	if id := int(f.allocate(1, 1)); id != 18 {
		t.Fatalf("exp=18; got=%v", id)
	}
	if id := int(f.allocate(1, 1)); id != 0 {
		t.Fatalf("exp=0; got=%v", id)
	}
	if exp := []pgid{}; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f.getFreePageIDs())
	}
}

// Ensure that a freelist can deserialize from a freelist page.
func TestFreelist_read(t *testing.T) {
	// Create a page.
	var buf [4096]byte
	page := (*page)(unsafe.Pointer(&buf[0]))
	page.flags = freelistPageFlag
	page.count = 2

	// Insert 2 page ids.
	ids := (*[3]pgid)(unsafe.Pointer(&page.ptr))
	ids[0] = 23
	ids[1] = 50

	// Deserialize page into a freelist.
	f := newFreelist()
	f.read(page)

	// Ensure that there are two page ids in the freelist.
	if exp := []pgid{23, 50}; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f.getFreePageIDs())
	}
}

// Ensure that a freelist can serialize into a freelist page.
func TestFreelist_write(t *testing.T) {
	// Create a freelist and write it to a page.
	var buf [4096]byte
	f := &freelist{ids: []pgid{12, 39}, pending: make(map[txid]*txPending)}
	f.pending[100] = &txPending{ids: []pgid{28, 11}}
	f.pending[101] = &txPending{ids: []pgid{3}}
	p := (*page)(unsafe.Pointer(&buf[0]))
	if err := f.write(p); err != nil {
		t.Fatal(err)
	}

	// Read the page back out.
	f2 := newFreelist()
	f2.read(p)

	// Ensure that the freelist is correct.
	// All pages should be present and in reverse order.
	if exp := []pgid{3, 11, 12, 28, 39}; !reflect.DeepEqual(exp, f2.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f2.getFreePageIDs())
	}
}

func Benchmark_FreelistRelease10K(b *testing.B)    { benchmark_FreelistRelease(b, 10000) }
func Benchmark_FreelistRelease100K(b *testing.B)   { benchmark_FreelistRelease(b, 100000) }
func Benchmark_FreelistRelease1000K(b *testing.B)  { benchmark_FreelistRelease(b, 1000000) }
func Benchmark_FreelistRelease10000K(b *testing.B) { benchmark_FreelistRelease(b, 10000000) }

func benchmark_FreelistRelease(b *testing.B, size int) {
	ids := randomPgids(size)
	pending := randomPgids(len(ids) / 400)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		txp := &txPending{ids: pending}
		f := &freelist{ids: ids, pending: map[txid]*txPending{1: txp}}
		f.release(1)
	}
}

func randomPgids(n int) []pgid {
	rand.Seed(42)
	pgids := make(pgids, n)
	for i := range pgids {
		pgids[i] = pgid(rand.Int63())
	}
	sort.Sort(pgids)
	return pgids
}
<commit_msg>fix test in releaserange (#142)<commit_after>package bbolt

import (
	"math/rand"
	"reflect"
	"sort"
	"testing"
	"unsafe"
)

// Ensure that a page is added to a transaction's freelist.
func TestFreelist_free(t *testing.T) {
	f := newFreelist()
	f.free(100, &page{id: 12})
	if !reflect.DeepEqual([]pgid{12}, f.pending[100].ids) {
		t.Fatalf("exp=%v; got=%v", []pgid{12}, f.pending[100])
	}
}

// Ensure that a page and its overflow is added to a transaction's freelist.
func TestFreelist_free_overflow(t *testing.T) {
	f := newFreelist()
	f.free(100, &page{id: 12, overflow: 3})
	if exp := []pgid{12, 13, 14, 15}; !reflect.DeepEqual(exp, f.pending[100].ids) {
		t.Fatalf("exp=%v; got=%v", exp, f.pending[100])
	}
}

// Ensure that a transaction's free pages can be released.
func TestFreelist_release(t *testing.T) {
	f := newFreelist()
	f.free(100, &page{id: 12, overflow: 1})
	f.free(100, &page{id: 9})
	f.free(102, &page{id: 39})
	f.release(100)
	f.release(101)
	if exp := []pgid{9, 12, 13}; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f.getFreePageIDs())
	}

	f.release(102)
	if exp := []pgid{9, 12, 13, 39}; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f.getFreePageIDs())
	}
}

// Ensure that releaseRange handles boundary conditions correctly
func TestFreelist_releaseRange(t *testing.T) {
	type testRange struct {
		begin, end txid
	}

	type testPage struct {
		id       pgid
		n        int
		allocTxn txid
		freeTxn  txid
	}

	var releaseRangeTests = []struct {
		title         string
		pagesIn       []testPage
		releaseRanges []testRange
		wantFree      []pgid
	}{
		{
			title:         "Single pending in range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
			releaseRanges: []testRange{{1, 300}},
			wantFree:      []pgid{3},
		},
		{
			title:         "Single pending with minimum end range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
			releaseRanges: []testRange{{1, 200}},
			wantFree:      []pgid{3},
		},
		{
			title:         "Single pending outsize minimum end range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
			releaseRanges: []testRange{{1, 199}},
			wantFree:      nil,
		},
		{
			title:         "Single pending with minimum begin range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
			releaseRanges: []testRange{{100, 300}},
			wantFree:      []pgid{3},
		},
		{
			title:         "Single pending outside minimum begin range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
			releaseRanges: []testRange{{101, 300}},
			wantFree:      nil,
		},
		{
			title:         "Single pending in minimum range",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 199, freeTxn: 200}},
			releaseRanges: []testRange{{199, 200}},
			wantFree:      []pgid{3},
		},
		{
			title:         "Single pending and read transaction at 199",
			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 199, freeTxn: 200}},
			releaseRanges: []testRange{{100, 198}, {200, 300}},
			wantFree:      nil,
		},
		{
			title: "Adjacent pending and read transactions at 199, 200",
			pagesIn: []testPage{
				{id: 3, n: 1, allocTxn: 199, freeTxn: 200},
				{id: 4, n: 1, allocTxn: 200, freeTxn: 201},
			},
			releaseRanges: []testRange{
				{100, 198},
				{200, 199}, // Simulate the ranges db.freePages might produce.
				{201, 300},
			},
			wantFree: nil,
		},
		{
			title: "Out of order ranges",
			pagesIn: []testPage{
				{id: 3, n: 1, allocTxn: 199, freeTxn: 200},
				{id: 4, n: 1, allocTxn: 200, freeTxn: 201},
			},
			releaseRanges: []testRange{
				{201, 199},
				{201, 200},
				{200, 200},
			},
			wantFree: nil,
		},
		{
			title: "Multiple pending, read transaction at 150",
			pagesIn: []testPage{
				{id: 3, n: 1, allocTxn: 100, freeTxn: 200},
				{id: 4, n: 1, allocTxn: 100, freeTxn: 125},
				{id: 5, n: 1, allocTxn: 125, freeTxn: 150},
				{id: 6, n: 1, allocTxn: 125, freeTxn: 175},
				{id: 7, n: 2, allocTxn: 150, freeTxn: 175},
				{id: 9, n: 2, allocTxn: 175, freeTxn: 200},
			},
			releaseRanges: []testRange{{50, 149}, {151, 300}},
			wantFree:      []pgid{4, 9, 10},
		},
	}

	for _, c := range releaseRangeTests {
		f := newFreelist()
		var ids []pgid
		for _, p := range c.pagesIn {
			for i := uint64(0); i < uint64(p.n); i++ {
				ids = append(ids, pgid(uint64(p.id)+i))
			}
		}
		f.readIDs(ids)
		for _, p := range c.pagesIn {
			f.allocate(p.allocTxn, p.n)
		}

		for _, p := range c.pagesIn {
			f.free(p.freeTxn, &page{id: p.id, overflow: uint32(p.n - 1)})
		}

		for _, r := range c.releaseRanges {
			f.releaseRange(r.begin, r.end)
		}

		if exp := c.wantFree; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
			t.Errorf("exp=%v; got=%v for %s", exp, f.getFreePageIDs(), c.title)
		}
	}
}

// Ensure that a freelist can find contiguous blocks of pages.
func TestFreelist_allocate(t *testing.T) {
	f := newFreelist()
	ids := []pgid{3, 4, 5, 6, 7, 9, 12, 13, 18}
	f.readIDs(ids)
	if id := int(f.allocate(1, 3)); id != 3 {
		t.Fatalf("exp=3; got=%v", id)
	}
	if id := int(f.allocate(1, 1)); id != 6 {
		t.Fatalf("exp=6; got=%v", id)
	}
	if id := int(f.allocate(1, 3)); id != 0 {
		t.Fatalf("exp=0; got=%v", id)
	}
	if id := int(f.allocate(1, 2)); id != 12 {
		t.Fatalf("exp=12; got=%v", id)
	}
	if id := int(f.allocate(1, 1)); id != 7 {
		t.Fatalf("exp=7; got=%v", id)
	}
	if id := int(f.allocate(1, 0)); id != 0 {
		t.Fatalf("exp=0; got=%v", id)
	}
	if id := int(f.allocate(1, 0)); id != 0 {
		t.Fatalf("exp=0; got=%v", id)
	}
	if exp := []pgid{9, 18}; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f.getFreePageIDs())
	}

	if id := int(f.allocate(1, 1)); id != 9 {
		t.Fatalf("exp=9; got=%v", id)
	}
	if id := int(f.allocate(1, 1)); id != 18 {
		t.Fatalf("exp=18; got=%v", id)
	}
	if id := int(f.allocate(1, 1)); id != 0 {
		t.Fatalf("exp=0; got=%v", id)
	}
	if exp := []pgid{}; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f.getFreePageIDs())
	}
}

// Ensure that a freelist can deserialize from a freelist page.
func TestFreelist_read(t *testing.T) {
	// Create a page.
	var buf [4096]byte
	page := (*page)(unsafe.Pointer(&buf[0]))
	page.flags = freelistPageFlag
	page.count = 2

	// Insert 2 page ids.
	ids := (*[3]pgid)(unsafe.Pointer(&page.ptr))
	ids[0] = 23
	ids[1] = 50

	// Deserialize page into a freelist.
	f := newFreelist()
	f.read(page)

	// Ensure that there are two page ids in the freelist.
	if exp := []pgid{23, 50}; !reflect.DeepEqual(exp, f.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f.getFreePageIDs())
	}
}

// Ensure that a freelist can serialize into a freelist page.
func TestFreelist_write(t *testing.T) {
	// Create a freelist and write it to a page.
	var buf [4096]byte
	f := &freelist{ids: []pgid{12, 39}, pending: make(map[txid]*txPending)}
	f.pending[100] = &txPending{ids: []pgid{28, 11}}
	f.pending[101] = &txPending{ids: []pgid{3}}
	p := (*page)(unsafe.Pointer(&buf[0]))
	if err := f.write(p); err != nil {
		t.Fatal(err)
	}

	// Read the page back out.
	f2 := newFreelist()
	f2.read(p)

	// Ensure that the freelist is correct.
	// All pages should be present and in reverse order.
	if exp := []pgid{3, 11, 12, 28, 39}; !reflect.DeepEqual(exp, f2.getFreePageIDs()) {
		t.Fatalf("exp=%v; got=%v", exp, f2.getFreePageIDs())
	}
}

func Benchmark_FreelistRelease10K(b *testing.B)    { benchmark_FreelistRelease(b, 10000) }
func Benchmark_FreelistRelease100K(b *testing.B)   { benchmark_FreelistRelease(b, 100000) }
func Benchmark_FreelistRelease1000K(b *testing.B)  { benchmark_FreelistRelease(b, 1000000) }
func Benchmark_FreelistRelease10000K(b *testing.B) { benchmark_FreelistRelease(b, 10000000) }

func benchmark_FreelistRelease(b *testing.B, size int) {
	ids := randomPgids(size)
	pending := randomPgids(len(ids) / 400)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		txp := &txPending{ids: pending}
		f := &freelist{ids: ids, pending: map[txid]*txPending{1: txp}}
		f.release(1)
	}
}

func randomPgids(n int) []pgid {
	rand.Seed(42)
	pgids := make(pgids, n)
	for i := range pgids {
		pgids[i] = pgid(rand.Int63())
	}
	sort.Sort(pgids)
	return pgids
}
<|endoftext|>


<commit_before>package orcraft

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"strings"
	"time"

	"github.com/openark/golib/log"

	"github.com/hashicorp/raft"
)

const (
	retainSnapshotCount = 288
	snapshotInterval    = 10 * time.Minute
	raftTimeout         = 10 * time.Second
)

type Store struct {
	raftDir       string
	raftBind      string
	raftAdvertise string

	raft      *raft.Raft // The consensus mechanism
	peerStore raft.PeerStore

	applier                CommandApplier
	snapshotCreatorApplier SnapshotCreatorApplier
}

type storeCommand struct {
	Op    string `json:"op,omitempty"`
	Value []byte `json:"value,omitempty"`
}

// NewStore inits and returns a new store
func NewStore(raftDir string, raftBind string, raftAdvertise string, applier CommandApplier, snapshotCreatorApplier SnapshotCreatorApplier) *Store {
	return &Store{
		raftDir:                raftDir,
		raftBind:               raftBind,
		raftAdvertise:          raftAdvertise,
		applier:                applier,
		snapshotCreatorApplier: snapshotCreatorApplier,
	}
}

// Open opens the store. If enableSingle is set, and there are no existing peers,
// then this node becomes the first node, and therefore leader, of the cluster.
func (store *Store) Open(peerNodes []string) error {
	// Setup Raft configuration.
	config := raft.DefaultConfig()
	config.SnapshotThreshold = 1
	config.SnapshotInterval = snapshotInterval

	// Setup Raft communication.
	advertise, err := net.ResolveTCPAddr("tcp", store.raftAdvertise)
	if err != nil {
		return err
	}
	log.Debugf("raft: advertise=%+v", advertise)

	transport, err := raft.NewTCPTransport(store.raftBind, advertise, 3, 10*time.Second, os.Stderr)
	if err != nil {
		return err
	}
	log.Debugf("raft: transport=%+v", transport)

	peers := make([]string, 0, 10)
	for _, peerNode := range peerNodes {
		peerNode = strings.TrimSpace(peerNode)
		peers = raft.AddUniquePeer(peers, peerNode)
	}
	log.Debugf("raft: peers=%+v", peers)

	// Create peer storage.
	peerStore := &raft.StaticPeers{}
	if err := peerStore.SetPeers(peers); err != nil {
		return err
	}

	// Allow the node to enter single-mode, potentially electing itself, if
	// explicitly enabled and there is only 1 node in the cluster already.
	if len(peerNodes) == 0 && len(peers) <= 1 {
		log.Infof("enabling single-node mode")
		config.EnableSingleNode = true
		config.DisableBootstrapAfterElect = false
	}

	// Create the snapshot store. This allows the Raft to truncate the log.
	snapshots, err := raft.NewFileSnapshotStore(store.raftDir, retainSnapshotCount, os.Stderr)
	if err != nil {
		return log.Errorf("file snapshot store: %s", err)
	}

	// Create the log store and stable store.
	logStore := NewRelationalStore()
	log.Debugf("raft: logStore=%+v", logStore)
	if snapshotsList, err := snapshots.List(); err == nil && len(snapshotsList) == 0 {
		log.Debugf("raft: cannot find any snapshots.")
		if firstIndex, _ := logStore.FirstIndex(); firstIndex > 1 {
			log.Debugf("raft: log store first index is %+v. Since no spanshots exist I will purge the log store", firstIndex)
			logStore.DeleteAll()
		}
	}

	// Instantiate the Raft systems.
	if store.raft, err = raft.NewRaft(config, (*fsm)(store), logStore, logStore, snapshots, peerStore, transport); err != nil {
		return fmt.Errorf("error creating new raft: %s", err)
	}
	store.peerStore = peerStore
	log.Infof("new raft created")

	return nil
}

// Join joins a node, located at addr, to this store. The node must be ready to
// respond to Raft communications at that address.
func (store *Store) Join(addr string) error {
	log.Infof("received join request for remote node as %s", addr)

	f := store.raft.AddPeer(addr)
	if f.Error() != nil {
		return f.Error()
	}
	log.Infof("node at %s joined successfully", addr)
	return nil
}

// genericCommand requests consensus for applying a single command.
// This is an internal orchestrator implementation
func (store *Store) genericCommand(op string, bytes []byte) (response interface{}, err error) {
	if store.raft.State() != raft.Leader {
		return nil, fmt.Errorf("not leader")
	}

	b, err := json.Marshal(&storeCommand{Op: op, Value: bytes})
	if err != nil {
		return nil, err
	}

	f := store.raft.Apply(b, raftTimeout)
	if err = f.Error(); err != nil {
		return nil, err
	}
	r := f.Response()
	return r, nil
}
<commit_msg>since remote snapshot restore is available, we don't need much snapshot history<commit_after>package orcraft

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"strings"
	"time"

	"github.com/openark/golib/log"

	"github.com/hashicorp/raft"
)

const (
	retainSnapshotCount = 10
	snapshotInterval    = 30 * time.Minute
	raftTimeout         = 10 * time.Second
)

type Store struct {
	raftDir       string
	raftBind      string
	raftAdvertise string

	raft      *raft.Raft // The consensus mechanism
	peerStore raft.PeerStore

	applier                CommandApplier
	snapshotCreatorApplier SnapshotCreatorApplier
}

type storeCommand struct {
	Op    string `json:"op,omitempty"`
	Value []byte `json:"value,omitempty"`
}

// NewStore inits and returns a new store
func NewStore(raftDir string, raftBind string, raftAdvertise string, applier CommandApplier, snapshotCreatorApplier SnapshotCreatorApplier) *Store {
	return &Store{
		raftDir:                raftDir,
		raftBind:               raftBind,
		raftAdvertise:          raftAdvertise,
		applier:                applier,
		snapshotCreatorApplier: snapshotCreatorApplier,
	}
}

// Open opens the store. If enableSingle is set, and there are no existing peers,
// then this node becomes the first node, and therefore leader, of the cluster.
func (store *Store) Open(peerNodes []string) error {
	// Setup Raft configuration.
	config := raft.DefaultConfig()
	config.SnapshotThreshold = 1
	config.SnapshotInterval = snapshotInterval

	// Setup Raft communication.
	advertise, err := net.ResolveTCPAddr("tcp", store.raftAdvertise)
	if err != nil {
		return err
	}
	log.Debugf("raft: advertise=%+v", advertise)

	transport, err := raft.NewTCPTransport(store.raftBind, advertise, 3, 10*time.Second, os.Stderr)
	if err != nil {
		return err
	}
	log.Debugf("raft: transport=%+v", transport)

	peers := make([]string, 0, 10)
	for _, peerNode := range peerNodes {
		peerNode = strings.TrimSpace(peerNode)
		peers = raft.AddUniquePeer(peers, peerNode)
	}
	log.Debugf("raft: peers=%+v", peers)

	// Create peer storage.
	peerStore := &raft.StaticPeers{}
	if err := peerStore.SetPeers(peers); err != nil {
		return err
	}

	// Allow the node to enter single-mode, potentially electing itself, if
	// explicitly enabled and there is only 1 node in the cluster already.
	if len(peerNodes) == 0 && len(peers) <= 1 {
		log.Infof("enabling single-node mode")
		config.EnableSingleNode = true
		config.DisableBootstrapAfterElect = false
	}

	// Create the snapshot store. This allows the Raft to truncate the log.
	snapshots, err := raft.NewFileSnapshotStore(store.raftDir, retainSnapshotCount, os.Stderr)
	if err != nil {
		return log.Errorf("file snapshot store: %s", err)
	}

	// Create the log store and stable store.
	logStore := NewRelationalStore()
	log.Debugf("raft: logStore=%+v", logStore)
	if snapshotsList, err := snapshots.List(); err == nil && len(snapshotsList) == 0 {
		log.Debugf("raft: cannot find any snapshots.")
		if firstIndex, _ := logStore.FirstIndex(); firstIndex > 1 {
			log.Debugf("raft: log store first index is %+v. Since no spanshots exist I will purge the log store", firstIndex)
			logStore.DeleteAll()
		}
	}

	// Instantiate the Raft systems.
	if store.raft, err = raft.NewRaft(config, (*fsm)(store), logStore, logStore, snapshots, peerStore, transport); err != nil {
		return fmt.Errorf("error creating new raft: %s", err)
	}
	store.peerStore = peerStore
	log.Infof("new raft created")

	return nil
}

// Join joins a node, located at addr, to this store. The node must be ready to
// respond to Raft communications at that address.
func (store *Store) Join(addr string) error {
	log.Infof("received join request for remote node as %s", addr)

	f := store.raft.AddPeer(addr)
	if f.Error() != nil {
		return f.Error()
	}
	log.Infof("node at %s joined successfully", addr)
	return nil
}

// genericCommand requests consensus for applying a single command.
// This is an internal orchestrator implementation
func (store *Store) genericCommand(op string, bytes []byte) (response interface{}, err error) {
	if store.raft.State() != raft.Leader {
		return nil, fmt.Errorf("not leader")
	}

	b, err := json.Marshal(&storeCommand{Op: op, Value: bytes})
	if err != nil {
		return nil, err
	}

	f := store.raft.Apply(b, raftTimeout)
	if err = f.Error(); err != nil {
		return nil, err
	}
	r := f.Response()
	return r, nil
}
<|endoftext|>


<commit_before>package gode

import (
	"bytes"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"path/filepath"
)

// Package represents an npm package.
type Package struct {
	Name    string `json:"name"`
	Version string `json:"version"`
}

// Packages returns a list of npm packages installed.
func Packages() ([]Package, error) {
	stdout, stderr, err := execNpm("list", "--json", "--depth=0")
	if err != nil {
		return nil, errors.New(stderr)
	}
	var response map[string]map[string]Package
	if err := json.Unmarshal([]byte(stdout), &response); err != nil {
		return nil, errors.New(stderr)
	}
	packages := make([]Package, 0, len(response["dependencies"]))
	for name, p := range response["dependencies"] {
		p.Name = name
		packages = append(packages, p)
	}
	return packages, nil
}

// InstallPackages installs a npm packages.
func InstallPackages(packages ...string) error {
	args := append([]string{"install"}, packages...)
	_, stderr, err := execNpm(args...)
	if err != nil {
		return errors.New("Error installing package. \n" + stderr + "\nTry running again with GODE_DEBUG=info to see more output.")
	}
	return nil
}

// RemovePackages removes a npm packages.
func RemovePackages(packages ...string) error {
	args := append([]string{"remove"}, packages...)
	_, stderr, err := execNpm(args...)
	if err != nil {
		return errors.New(stderr)
	}
	return nil
}

// OutdatedPackages returns a map of packages and their latest version
func OutdatedPackages(names ...string) (map[string]string, error) {
	args := append([]string{"outdated", "--json"}, names...)
	stdout, stderr, err := execNpm(args...)
	if err != nil {
		return nil, errors.New(stderr)
	}
	var outdated map[string]struct{ Latest string }
	json.Unmarshal([]byte(stdout), &outdated)
	packages := make(map[string]string, len(outdated))
	for name, versions := range outdated {
		packages[name] = versions.Latest
	}
	return packages, nil
}

// ClearCache clears the npm cache
func ClearCache() error {
	cmd, err := npmCmd("cache", "clean")
	if err != nil {
		return err
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func npmCmd(args ...string) (*exec.Cmd, error) {
	if err := os.MkdirAll(filepath.Join(rootPath, "node_modules"), 0755); err != nil {
		return nil, err
	}
	nodePath, err := filepath.Rel(rootPath, nodePath)
	if err != nil {
		return nil, err
	}
	npmPath, err := filepath.Rel(rootPath, npmPath)
	if err != nil {
		return nil, err
	}
	args = append([]string{npmPath}, args...)
	if debugging() {
		args = append(args, "--loglevel="+os.Getenv("GODE_DEBUG"))
	}
	cmd := exec.Command(nodePath, args...)
	cmd.Dir = rootPath
	cmd.Env = environ()
	return cmd, nil
}

func execNpm(args ...string) (string, string, error) {
	cmd, err := npmCmd(args...)
	if err != nil {
		return "", "", err
	}
	var stdout, stderr bytes.Buffer
	if debugging() {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	} else {
		cmd.Stdout = &stdout
		cmd.Stderr = &stderr
	}
	err = cmd.Run()
	return stdout.String(), stderr.String(), err
}

func environ() []string {
	env := append(os.Environ(), "NPM_CONFIG_ALWAYS_AUTH=false")
	env = append(env, "NPM_CONFIG_CACHE="+filepath.Join(rootPath, ".npm-cache"))
	env = append(env, "NPM_CONFIG_REGISTRY="+registry)
	env = append(env, "NPM_CONFIG_GLOBAL=false")
	return env
}

func debugging() bool {
	e := os.Getenv("GODE_DEBUG")
	return e != "" && e != "0" && e != "false"
}
<commit_msg>Set npm config onload-script=false<commit_after>package gode

import (
	"bytes"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"path/filepath"
)

// Package represents an npm package.
type Package struct {
	Name    string `json:"name"`
	Version string `json:"version"`
}

// Packages returns a list of npm packages installed.
func Packages() ([]Package, error) {
	stdout, stderr, err := execNpm("list", "--json", "--depth=0")
	if err != nil {
		return nil, errors.New(stderr)
	}
	var response map[string]map[string]Package
	if err := json.Unmarshal([]byte(stdout), &response); err != nil {
		return nil, errors.New(stderr)
	}
	packages := make([]Package, 0, len(response["dependencies"]))
	for name, p := range response["dependencies"] {
		p.Name = name
		packages = append(packages, p)
	}
	return packages, nil
}

// InstallPackages installs a npm packages.
func InstallPackages(packages ...string) error {
	args := append([]string{"install"}, packages...)
	_, stderr, err := execNpm(args...)
	if err != nil {
		return errors.New("Error installing package. \n" + stderr + "\nTry running again with GODE_DEBUG=info to see more output.")
	}
	return nil
}

// RemovePackages removes a npm packages.
func RemovePackages(packages ...string) error {
	args := append([]string{"remove"}, packages...)
	_, stderr, err := execNpm(args...)
	if err != nil {
		return errors.New(stderr)
	}
	return nil
}

// OutdatedPackages returns a map of packages and their latest version
func OutdatedPackages(names ...string) (map[string]string, error) {
	args := append([]string{"outdated", "--json"}, names...)
	stdout, stderr, err := execNpm(args...)
	if err != nil {
		return nil, errors.New(stderr)
	}
	var outdated map[string]struct{ Latest string }
	json.Unmarshal([]byte(stdout), &outdated)
	packages := make(map[string]string, len(outdated))
	for name, versions := range outdated {
		packages[name] = versions.Latest
	}
	return packages, nil
}

// ClearCache clears the npm cache
func ClearCache() error {
	cmd, err := npmCmd("cache", "clean")
	if err != nil {
		return err
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func npmCmd(args ...string) (*exec.Cmd, error) {
	if err := os.MkdirAll(filepath.Join(rootPath, "node_modules"), 0755); err != nil {
		return nil, err
	}
	nodePath, err := filepath.Rel(rootPath, nodePath)
	if err != nil {
		return nil, err
	}
	npmPath, err := filepath.Rel(rootPath, npmPath)
	if err != nil {
		return nil, err
	}
	args = append([]string{npmPath}, args...)
	if debugging() {
		args = append(args, "--loglevel="+os.Getenv("GODE_DEBUG"))
	}
	cmd := exec.Command(nodePath, args...)
	cmd.Dir = rootPath
	cmd.Env = environ()
	return cmd, nil
}

func execNpm(args ...string) (string, string, error) {
	cmd, err := npmCmd(args...)
	if err != nil {
		return "", "", err
	}
	var stdout, stderr bytes.Buffer
	if debugging() {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	} else {
		cmd.Stdout = &stdout
		cmd.Stderr = &stderr
	}
	err = cmd.Run()
	return stdout.String(), stderr.String(), err
}

func environ() []string {
	env := append(os.Environ(), "NPM_CONFIG_ALWAYS_AUTH=false")
	env = append(env, "NPM_CONFIG_CACHE="+filepath.Join(rootPath, ".npm-cache"))
	env = append(env, "NPM_CONFIG_REGISTRY="+registry)
	env = append(env, "NPM_CONFIG_GLOBAL=false")
	env = append(env, "NPM_CONFIG_ONLOAD_SCRIPT=false")
	return env
}

func debugging() bool {
	e := os.Getenv("GODE_DEBUG")
	return e != "" && e != "0" && e != "false"
}
<|endoftext|>


<commit_before>package gofakes3_test

import (
	"bytes"
	"net/http/httptest"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/johannesboyne/gofakes3"
)

func TestCreateBucket(t *testing.T) {
	//@TODO(jb): implement them for sanity reasons

	faker := gofakes3.New("tests3.db")
	ts := httptest.NewServer(faker.Server())
	defer ts.Close()

	config := aws.NewConfig()
	config.WithEndpoint(ts.URL)
	config.WithRegion("mine")

	svc := s3.New(session.New(), config)

	_, err := svc.CreateBucket(&s3.CreateBucketInput{
		Bucket: aws.String("BucketName"),
	})
	_, err = svc.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String("BucketName"),
	})
	_, err = svc.PutObject(&s3.PutObjectInput{
		Bucket: aws.String("BucketName"),
		Key:    aws.String("ObjectKey"),
		Body:   bytes.NewReader([]byte("{\"test\": \"foo\"}")),
		Metadata: map[string]*string{
			"Key": aws.String("MetadataValue"),
		},
	})
	obj, _ := svc.GetObject(&s3.GetObjectInput{
		Bucket: aws.String("BucketName"),
		Key:    aws.String("ObjectKey"),
	})
	head, _ := svc.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String("BucketName"),
		Key:    aws.String("ObjectKey"),
	})
	if *obj.ContentLength != 15 {
		t.Error("wrong content length")
	}
	if *head.ETag != "\"b2cf3fa731f5d49f742992d020537c4d\"" {
		t.Error("wrong head")
	}

	if err != nil {
		t.Errorf("ERROR:\n%+v\n", err)
		return
	}
}
<commit_msg>Change for CI<commit_after>package gofakes3_test

import (
	"bytes"
	"log"
	"net/http/httptest"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/johannesboyne/gofakes3"
)

func TestCreateBucket(t *testing.T) {
	//@TODO(jb): implement them for sanity reasons

	faker := gofakes3.New("tests3.db")
	ts := httptest.NewServer(faker.Server())
	defer ts.Close()

	config := aws.NewConfig()
	config.WithEndpoint(ts.URL)
	config.WithRegion("mine")

	svc := s3.New(session.New(), config)

	_, err := svc.CreateBucket(&s3.CreateBucketInput{
		Bucket: aws.String("BucketName"),
	})
	_, err = svc.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String("BucketName"),
	})
	_, err = svc.PutObject(&s3.PutObjectInput{
		Bucket: aws.String("BucketName"),
		Key:    aws.String("ObjectKey"),
		Body:   bytes.NewReader([]byte("{\"test\": \"foo\"}")),
		Metadata: map[string]*string{
			"Key": aws.String("MetadataValue"),
		},
	})
	obj, _ := svc.GetObject(&s3.GetObjectInput{
		Bucket: aws.String("BucketName"),
		Key:    aws.String("ObjectKey"),
	})
	head, _ := svc.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String("BucketName"),
		Key:    aws.String("ObjectKey"),
	})
	log.Println("-> CONTENT LENGTH", *obj.ContentLength)
	log.Println("-> HEAD ETAG     ", *head.ETag)

	if err != nil {
		t.Errorf("ERROR:\n%+v\n", err)
		return
	}
}
<|endoftext|>


<commit_before>package todolist

import (
	"fmt"
	"regexp"
	"strings"
	"time"
)

type App struct {
	TodoStore *FileStore
	TodoList  *TodoList
}

func NewApp() *App {
	app := &App{TodoList: &TodoList{}, TodoStore: NewFileStore()}
	return app
}

func (a *App) InitializeRepo() {
	a.TodoStore.Initialize()
}

func (a *App) AddTodo(input string) {
	a.Load()
	parser := &Parser{}
	todo := parser.ParseNewTodo(input)
	if todo == nil {
		fmt.Println("I need more information. Try something like 'todo a chat with @bob due tom'")
		return
	}

	id := a.TodoList.NextId()
	a.TodoList.Add(todo)
	a.Save()
	fmt.Printf("Todo %d added.\n", id)
}

func (a *App) DeleteTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Delete(id)
	a.Save()
	fmt.Println("Todo deleted.")
}

func (a *App) CompleteTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Complete(id)
	a.Save()
	fmt.Println("Todo completed.")
}

func (a *App) UncompleteTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Uncomplete(id)
	a.Save()
	fmt.Println("Todo uncompleted.")
}

func (a *App) ArchiveTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Archive(id)
	a.Save()
	fmt.Println("Todo archived.")
}

func (a *App) UnarchiveTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Unarchive(id)
	a.Save()
	fmt.Println("Todo unarchived.")
}

func (a *App) EditTodoSubject(input string) {
	a.Load()

	_, id, subject := Parser{input}.Parse()
	if id == -1 {
		return
	}

	_, todo := a.getId(input)
	if todo == nil {
		fmt.Println("Todo not found.")
		return
	}
	todo.Subject = subject

	a.Save()
	fmt.Println("Todo subject updated.")
}

func (a *App) EditTodoDue(input string) {
	a.Load()
	id, todo := a.getId(input)
	if id == -1 {
		return
	}
	parser := &Parser{}
	todo.Due = parser.Due(input, time.Now())
	a.Save()
	fmt.Println("Todo due date updated.")
}

func (a *App) ExpandTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	parser := &Parser{}
	if id == -1 {
		return
	}

	commonProject := parser.ExpandProject(input)
	todos := strings.LastIndex(input, ":")
	if commonProject == "" || len(input) <= todos+1 || todos == -1 {
		fmt.Println("I'm expecting a format like \"todolist ex <project>: <todo1>, <todo2>, ...\"")
		return
	}

	newTodos := strings.Split(input[todos+1:], ",")

	for _, todo := range newTodos {
		args := []string{"add ", commonProject, " ", todo}
		a.AddTodo(strings.Join(args, ""))
	}

	a.TodoList.Delete(id)
	a.Save()
	fmt.Println("Todo expanded.")
}

func (a *App) ArchiveCompleted() {
	a.Load()
	for _, todo := range a.TodoList.Todos() {
		if todo.Completed {
			todo.Archived = true
		}
	}
	a.Save()
	fmt.Println("All completed todos have been archived.")
}

func (a *App) ListTodos(input string) {
	a.Load()
	filtered := NewFilter(a.TodoList.Todos()).Filter(input)
	grouped := a.getGroups(input, filtered)

	formatter := NewFormatter(grouped)
	formatter.Print()
}

func (a *App) PrioritizeTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Prioritize(id)
	a.Save()
	fmt.Println("Todo prioritized.")
}

func (a *App) UnprioritizeTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Unprioritize(id)
	a.Save()
	fmt.Println("Todo un-prioritized.")
}

func (a *App) getId(input string) (int, *Todo) {
	_, id, _ := Parser{input}.Parse()
	todo := a.TodoList.FindById(id)
	if todo == nil {
		fmt.Println("No such id.")
		return -1, nil
	}
	return id, todo
}

func (a *App) getGroups(input string, todos []*Todo) *GroupedTodos {
	grouper := &Grouper{}
	contextRegex, _ := regexp.Compile(`by c.*$`)
	projectRegex, _ := regexp.Compile(`by p.*$`)

	var grouped *GroupedTodos

	if contextRegex.MatchString(input) {
		grouped = grouper.GroupByContext(todos)
	} else if projectRegex.MatchString(input) {
		grouped = grouper.GroupByProject(todos)
	} else {
		grouped = grouper.GroupByNothing(todos)
	}
	return grouped
}

func (a *App) GarbageCollect() {
	a.Load()
	a.TodoList.GarbageCollect()
	a.Save()
	fmt.Println("Garbage collection complete.")
}

func (a *App) Load() error {
	todos, err := a.TodoStore.Load()
	if err != nil {
		return err
	}
	a.TodoList.Load(todos)
	return nil
}

func (a *App) Save() {
	a.TodoStore.Save(a.TodoList.Data)
}
<commit_msg>do not duplicate error messages again<commit_after>package todolist

import (
	"fmt"
	"regexp"
	"strings"
	"time"
)

type App struct {
	TodoStore *FileStore
	TodoList  *TodoList
}

func NewApp() *App {
	app := &App{TodoList: &TodoList{}, TodoStore: NewFileStore()}
	return app
}

func (a *App) InitializeRepo() {
	a.TodoStore.Initialize()
}

func (a *App) AddTodo(input string) {
	a.Load()
	parser := &Parser{}
	todo := parser.ParseNewTodo(input)
	if todo == nil {
		fmt.Println("I need more information. Try something like 'todo a chat with @bob due tom'")
		return
	}

	id := a.TodoList.NextId()
	a.TodoList.Add(todo)
	a.Save()
	fmt.Printf("Todo %d added.\n", id)
}

func (a *App) DeleteTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Delete(id)
	a.Save()
	fmt.Println("Todo deleted.")
}

func (a *App) CompleteTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Complete(id)
	a.Save()
	fmt.Println("Todo completed.")
}

func (a *App) UncompleteTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Uncomplete(id)
	a.Save()
	fmt.Println("Todo uncompleted.")
}

func (a *App) ArchiveTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Archive(id)
	a.Save()
	fmt.Println("Todo archived.")
}

func (a *App) UnarchiveTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Unarchive(id)
	a.Save()
	fmt.Println("Todo unarchived.")
}

func (a *App) EditTodoSubject(input string) {
	a.Load()

	_, id, subject := Parser{input}.Parse()
	if id == -1 {
		return
	}

	_, todo := a.getId(input)
	if todo == nil {
		return
	}
	todo.Subject = subject

	a.Save()
	fmt.Println("Todo subject updated.")
}

func (a *App) EditTodoDue(input string) {
	a.Load()
	id, todo := a.getId(input)
	if id == -1 {
		return
	}
	parser := &Parser{}
	todo.Due = parser.Due(input, time.Now())
	a.Save()
	fmt.Println("Todo due date updated.")
}

func (a *App) ExpandTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	parser := &Parser{}
	if id == -1 {
		return
	}

	commonProject := parser.ExpandProject(input)
	todos := strings.LastIndex(input, ":")
	if commonProject == "" || len(input) <= todos+1 || todos == -1 {
		fmt.Println("I'm expecting a format like \"todolist ex <project>: <todo1>, <todo2>, ...\"")
		return
	}

	newTodos := strings.Split(input[todos+1:], ",")

	for _, todo := range newTodos {
		args := []string{"add ", commonProject, " ", todo}
		a.AddTodo(strings.Join(args, ""))
	}

	a.TodoList.Delete(id)
	a.Save()
	fmt.Println("Todo expanded.")
}

func (a *App) ArchiveCompleted() {
	a.Load()
	for _, todo := range a.TodoList.Todos() {
		if todo.Completed {
			todo.Archived = true
		}
	}
	a.Save()
	fmt.Println("All completed todos have been archived.")
}

func (a *App) ListTodos(input string) {
	a.Load()
	filtered := NewFilter(a.TodoList.Todos()).Filter(input)
	grouped := a.getGroups(input, filtered)

	formatter := NewFormatter(grouped)
	formatter.Print()
}

func (a *App) PrioritizeTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Prioritize(id)
	a.Save()
	fmt.Println("Todo prioritized.")
}

func (a *App) UnprioritizeTodo(input string) {
	a.Load()
	id, _ := a.getId(input)
	if id == -1 {
		return
	}
	a.TodoList.Unprioritize(id)
	a.Save()
	fmt.Println("Todo un-prioritized.")
}

func (a *App) getId(input string) (int, *Todo) {
	_, id, _ := Parser{input}.Parse()
	todo := a.TodoList.FindById(id)
	if todo == nil {
		fmt.Println("No such id.")
		return -1, nil
	}
	return id, todo
}

func (a *App) getGroups(input string, todos []*Todo) *GroupedTodos {
	grouper := &Grouper{}
	contextRegex, _ := regexp.Compile(`by c.*$`)
	projectRegex, _ := regexp.Compile(`by p.*$`)

	var grouped *GroupedTodos

	if contextRegex.MatchString(input) {
		grouped = grouper.GroupByContext(todos)
	} else if projectRegex.MatchString(input) {
		grouped = grouper.GroupByProject(todos)
	} else {
		grouped = grouper.GroupByNothing(todos)
	}
	return grouped
}

func (a *App) GarbageCollect() {
	a.Load()
	a.TodoList.GarbageCollect()
	a.Save()
	fmt.Println("Garbage collection complete.")
}

func (a *App) Load() error {
	todos, err := a.TodoStore.Load()
	if err != nil {
		return err
	}
	a.TodoList.Load(todos)
	return nil
}

func (a *App) Save() {
	a.TodoStore.Save(a.TodoList.Data)
}
<|endoftext|>


<commit_before>// This package provides a framework for creating powerful lexical
// preprocessors for go code.
package goprep

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"go/printer"
	"go/parser"
	"go/scanner"
	"go/token"
)

// Represents a token as returned by scanner.Scanner.Scan(), with the position,
// token type, and string representation.
type TokenInfo struct {
	Pos   token.Position
	Token token.Token
	Str   string
}

// The pipeline from TokenInfo inputs to string outputs. This would probably be
// a monad in haskell.
//
// We're using channels to serve as an abstracted for-loop, not because we want
// parallelism. All processing is synchronized with Sync. A message to Sync
// signals that the token has been 'used', and the sender is ready for the
// next. Since adding a read to sync (which is 'always' being read from by the
// frontend goroutine) would result in undeterministic behaviour, routines
// originating artificial tokens must create a new synchronization channel.
type Pipeline struct {
	Input  chan TokenInfo
	Output chan string
	Sync   chan interface{}
}

// PipeInit initializes a pipeline; input will be read from iReader.
func PipeInit(iReader io.Reader) *Pipeline {
	input := make(chan TokenInfo)
	output := make(chan string)
	sync := make(chan interface{})
	p := &Pipeline{ input, output, sync }

	src, err := ioutil.ReadAll(iReader)
	if err != nil { panic(err) }

	fset := token.NewFileSet()
	file := fset.AddFile("<stdin>", fset.Base(), len(src))

	s := scanner.Scanner{}
	s.Init(file, src, nil, scanner.InsertSemis | scanner.ScanComments)

	go func() {
		pos, tok, str := s.Scan()
		for tok != token.EOF {
			if tok == token.COMMENT {
				str = str + "\n"
			}
			input <- TokenInfo{fset.Position(pos), tok, str}
			<-sync // wait for sent token to land
			pos, tok, str = s.Scan()
		}
		close(input)
	}()

	return p
}

// PipeEnd implements the closing (writing) portion of a pipeline.
func PipeEnd(p *Pipeline, oWriter io.Writer) {
	outbuf := new(bytes.Buffer)
	_, output, sync := p.Input, p.Output, p.Sync

	// spit the tokens to the write end of the pipe
	for tok := range output {
		fmt.Fprintf(outbuf, " %s", tok)
		sync <- nil
	}

	// parse the tokens into an AST and write to output
	fset := token.NewFileSet()
	file, err := parser.ParseFile(
		fset, "<stdin>", outbuf, parser.ParseComments)
	if err != nil {
		panic(err)
	}
	printer.Fprint(oWriter, fset, file)
	close(sync)
}

// True takes a TokenInfo and always returns true.
func True(TokenInfo) bool { return true }

// False takes a TokenInfo and always returns false.
func False(TokenInfo) bool { return false}

// Lines passes line pragma information as needed to the output channel, thus
// ensuring that line numbers in the output match line numbers in the input.
func Lines(pipeline *Pipeline) {
	tIn := pipeline.Input
	tOut := make(chan TokenInfo)
	out := pipeline.Output
	isync := pipeline.Sync
	osync := make(chan interface{})
	fname := ""
	lno := 0
	pipeline.Input = tOut
	pipeline.Sync = osync
	go func() {
		for tok := range tIn {
			fname = tok.Pos.Filename
			lno = tok.Pos.Line
			tOut <- tok
			<-osync // wait for the bounce

			// now send the 'line' directive if necessary
			if tok.Str == "\n" {
				ld := fmt.Sprintf("//line %s:%d\n", fname, lno)
				out <- ld
				<-osync
			}

			isync <- nil
		}
		close(tOut)
	}()
}

// Ignore produces a modified input stream that does not include any tokens for
// which f evaluates to true, thus discarding a certain class of tokens.
func Ignore(f func(TokenInfo) bool) func(*Pipeline) {
	return func(p *Pipeline) {
		tOut := make(chan TokenInfo)
		tIn := p.Input
		sync := p.Sync
		go func() {
			for tok := range tIn {
				if !f(tok) {
					tOut <- tok
				} else {
					sync <- nil
				}
			}
			close(tOut)
		}()
		p.Input = tOut
	}
}

// IgnoreToken is like Ignore, discarding all tokens whose string content is
// equal to the given string.
func IgnoreToken(str string) func(*Pipeline) {
	return Ignore(func(ti TokenInfo) bool {
		return ti.Str == str
	})
}

// IgnoreType is like Ignore, discarding all tokens of a certain type.
func IgnoreType(tok token.Token) func(*Pipeline) {
	return Ignore(func(ti TokenInfo) bool {
		return ti.Token == tok
	})
}

// Pass redirects all tokens for which f evaluates to true to the output
// channel, returning the altered input channel.
func Pass(f func(TokenInfo) bool) func(*Pipeline) {
	return func(p *Pipeline) {
		tOut := make(chan TokenInfo)
		tIn := p.Input
		out := p.Output
		go func() {
			for tok := range tIn {
				if f(tok) {
					out <- tok.Str
				} else {
					tOut <- tok
				}
			}
			close(tOut)
		}()
		p.Input = tOut
	}
}

// PassToken is like Pass, passing all tokens whose string content is equal to
// the given string.
func PassToken(str string) func(*Pipeline) {
	return Pass(func(ti TokenInfo) bool {
		return ti.Str == str
	})
}

// PassType is like Pass, passing all tokens of a certain type.
func PassType(tok token.Token) func(*Pipeline) {
	return Pass(func(ti TokenInfo) bool {
		return ti.Token == tok
	})
}

// Discard discards whatever tokens remain and waits for the channel to close,
// then closing the output channel as well.  It is an appropriate way to end a
// long list of processing declarations.
func Discard(p *Pipeline) {
	input, output, sync := p.Input, p.Output, p.Sync
	go func() {
		for _ = range input {
			sync <- nil
		}
		close(output)
	}()
}
<commit_msg>Slightly changing the semantics of the sync channel<commit_after>// This package provides a framework for creating powerful lexical
// preprocessors for go code.
package goprep

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"go/printer"
	"go/parser"
	"go/scanner"
	"go/token"
)

// Represents a token as returned by scanner.Scanner.Scan(), with the position,
// token type, and string representation.
type TokenInfo struct {
	Pos   token.Position
	Token token.Token
	Str   string
}

// The pipeline from TokenInfo inputs to string outputs. This would probably be
// a monad in haskell.
//
// We're using channels to serve as an abstracted for-loop, not because we want
// parallelism. All processing is synchronized with Sync. A message to Sync
// signals that the token has been 'used', and the sender is ready for the
// next. Since adding a read to sync (which is 'always' being read from by the
// frontend goroutine) would result in undeterministic behaviour, routines
// originating artificial tokens must create a new synchronization channel.
type Pipeline struct {
	Input  chan TokenInfo
	Output chan string
	Sync   chan interface{}
}

// PipeInit initializes a pipeline; input will be read from iReader.
func PipeInit(iReader io.Reader) *Pipeline {
	input := make(chan TokenInfo)
	output := make(chan string)
	sync := make(chan interface{})
	p := &Pipeline{ input, output, sync }

	src, err := ioutil.ReadAll(iReader)
	if err != nil { panic(err) }

	fset := token.NewFileSet()
	file := fset.AddFile("<stdin>", fset.Base(), len(src))

	s := scanner.Scanner{}
	s.Init(file, src, nil, scanner.InsertSemis | scanner.ScanComments)

	go func() {
		pos, tok, str := s.Scan()
		for tok != token.EOF {
			if tok == token.COMMENT {
				str = str + "\n"
			}
			input <- TokenInfo{fset.Position(pos), tok, str}
			<-sync // wait for sent token to land
			pos, tok, str = s.Scan()
		}
		close(input)
	}()

	return p
}

// PipeEnd implements the closing (writing) portion of a pipeline.
func PipeEnd(p *Pipeline, oWriter io.Writer) {
	outbuf := new(bytes.Buffer)
	output := p.Output

	// spit the tokens to the write end of the pipe
	for tok := range output {
		fmt.Fprintf(outbuf, " %s", tok)
	}

	// parse the tokens into an AST and write to output
	fset := token.NewFileSet()
	file, err := parser.ParseFile(
		fset, "<stdin>", outbuf, parser.ParseComments)
	if err != nil { panic(err) }
	printer.Fprint(oWriter, fset, file)
}

// True takes a TokenInfo and always returns true.
func True(TokenInfo) bool { return true }

// False takes a TokenInfo and always returns false.
func False(TokenInfo) bool { return false}

// Lines passes line pragma information as needed to the output channel, thus
// ensuring that line numbers in the output match line numbers in the input.
func Lines(pipeline *Pipeline) {
	tIn := pipeline.Input
	tOut := make(chan TokenInfo)
	out := pipeline.Output
	isync := pipeline.Sync
	osync := make(chan interface{})
	fname := ""
	lno := 0
	pipeline.Input = tOut
	pipeline.Sync = osync
	go func() {
		for tok := range tIn {
			fname = tok.Pos.Filename
			lno = tok.Pos.Line
			tOut <- tok
			<-osync // wait for the bounce

			// now send the 'line' directive if necessary
			if tok.Str == "\n" {
				ld := fmt.Sprintf("//line %s:%d\n", fname, lno)
				out <- ld
			}

			isync <- nil
		}
		close(tOut)
	}()
}

// Ignore produces a modified input stream that does not include any tokens for
// which f evaluates to true, thus discarding a certain class of tokens.
func Ignore(f func(TokenInfo) bool) func(*Pipeline) {
	return func(p *Pipeline) {
		tOut := make(chan TokenInfo)
		tIn := p.Input
		sync := p.Sync
		go func() {
			for tok := range tIn {
				if !f(tok) {
					tOut <- tok
				} else {
					sync <- nil
				}
			}
			close(tOut)
		}()
		p.Input = tOut
	}
}

// IgnoreToken is like Ignore, discarding all tokens whose string content is
// equal to the given string.
func IgnoreToken(str string) func(*Pipeline) {
	return Ignore(func(ti TokenInfo) bool {
		return ti.Str == str
	})
}

// IgnoreType is like Ignore, discarding all tokens of a certain type.
func IgnoreType(tok token.Token) func(*Pipeline) {
	return Ignore(func(ti TokenInfo) bool {
		return ti.Token == tok
	})
}

// Pass redirects all tokens for which f evaluates to true to the output
// channel, returning the altered input channel.
func Pass(f func(TokenInfo) bool) func(*Pipeline) {
	return func(p *Pipeline) {
		tOut := make(chan TokenInfo)
		tIn := p.Input
		out := p.Output
		sync := p.Sync
		go func() {
			for tok := range tIn {
				if f(tok) {
					out <- tok.Str
					sync <- nil
				} else {
					tOut <- tok
				}
			}
			close(tOut)
		}()
		p.Input = tOut
	}
}

// PassToken is like Pass, passing all tokens whose string content is equal to
// the given string.
func PassToken(str string) func(*Pipeline) {
	return Pass(func(ti TokenInfo) bool {
		return ti.Str == str
	})
}

// PassType is like Pass, passing all tokens of a certain type.
func PassType(tok token.Token) func(*Pipeline) {
	return Pass(func(ti TokenInfo) bool {
		return ti.Token == tok
	})
}

// Discard discards whatever tokens remain and waits for the channel to close,
// then closing the output channel as well.  It is an appropriate way to end a
// long list of processing declarations.
func Discard(p *Pipeline) {
	input, output, sync := p.Input, p.Output, p.Sync
	go func() {
		for _ = range input {
			sync <- nil
		}
		close(output)
		close(sync)
	}()
}
<|endoftext|>


<commit_before>package applier

import (
	"errors"
	"fmt"
	"strings"

	"github.com/skeema/skeema/internal/fs"
	"github.com/skeema/skeema/internal/tengo"
	"github.com/skeema/skeema/internal/workspace"
)

// VerifyDiff verifies the result of all AlterTable values found in
// diff.TableDiffs, confirming that applying the corresponding ALTER would
// bring a table from the version currently in the instance to the version
// specified in the filesystem.
func VerifyDiff(diff *tengo.SchemaDiff, t *Target) error {
	if !t.Dir.Config.GetBool("verify") || len(diff.TableDiffs) == 0 || t.briefOutput() {
		return nil
	}

	// If diff contains no ALTER TABLEs, nothing to verify
	altersInDiff := diff.FilteredTableDiffs(tengo.DiffTypeAlter)
	if len(altersInDiff) == 0 {
		return nil
	}

	// The goal of VerifyDiff is to confirm that the diff contains the correct and
	// complete set of differences between all modified tables. We use a strict set
	// of statement modifiers that will transform the initial state into an exact
	// match of the desired state. This is all run in a workspace, so we can be
	// more aggressive about which statement modifiers are used in generating the
	// ALTER here. When the diff is actually used against the real live table
	// later, a different looser set of modifiers is used which filters out some
	// of the undesired cosmetic clauses by default.
	mods := tengo.StatementModifiers{
		NextAutoInc:            tengo.NextAutoIncAlways,      // use whichever auto_increment is in the fs
		Partitioning:           tengo.PartitioningPermissive, // ditto with partitioning status
		AllowUnsafe:            true,                         // needed since we're just running against the temp schema
		AlgorithmClause:        "copy",                       // needed so the DB doesn't ignore attempts to re-order indexes
		StrictIndexOrder:       true,                         // needed since we want the SHOW CREATE TABLEs to match
		StrictCheckOrder:       true,                         // ditto (only affects MariaDB)
		StrictForeignKeyNaming: true,                         // ditto
		StrictColumnDefinition: true,                         // ditto (only affects MySQL 8 edge cases)
		SkipPreDropAlters:      true,                         // ignore DROP PARTITIONs that were only generated to speed up a DROP TABLE
		Flavor:                 t.Instance.Flavor(),
	}
	if mods.Flavor.Matches(tengo.FlavorMySQL55) {
		mods.AlgorithmClause = "" // MySQL 5.5 doesn't support ALGORITHM clause
	}

	// Gather CREATE and ALTER for modified tables, and put into a LogicalSchema,
	// which we then materialize into a real schema using a workspace
	logicalSchema := &fs.LogicalSchema{
		CharSet:   t.Dir.Config.Get("default-character-set"),
		Collation: t.Dir.Config.Get("default-collation"),
		Creates:   make(map[tengo.ObjectKey]*fs.Statement),
		Alters:    make([]*fs.Statement, 0),
	}
	desiredTables := make(map[string]*tengo.Table)
	for _, td := range altersInDiff {
		stmt, err := td.Statement(mods)
		if stmt != "" && err == nil {
			// Note: sometimes a table's diff gets split into multiple ALTERs, but this
			// logic can ignore that fact. If there are redundant AddStatement calls for
			// one CREATE, the first AddStatement for that CREATE succeeds and the
			// subsequent duplicate CREATEs error, but that is harmless in this code path!
			logicalSchema.AddStatement(&fs.Statement{
				Type:       fs.StatementTypeCreate,
				Text:       td.From.CreateStatement,
				ObjectType: tengo.ObjectTypeTable,
				ObjectName: td.From.Name,
			})
			logicalSchema.AddStatement(&fs.Statement{
				Type:       fs.StatementTypeAlter,
				Text:       stmt,
				ObjectType: tengo.ObjectTypeTable,
				ObjectName: td.From.Name,
			})
			desiredTables[td.From.Name] = td.To
		}
	}

	opts, err := workspace.OptionsForDir(t.Dir, t.Instance)
	if err != nil {
		return err
	}
	wsSchema, err := workspace.ExecLogicalSchema(logicalSchema, opts)
	if err == nil && len(wsSchema.Failures) > 0 {
		err = wsSchema.Failures[0]
	}
	if err != nil {
		return fmt.Errorf("Diff verification failure: %s", err.Error())
	}

	// Compare the "expected" version of each table ("to" side of original diff,
	// from the filesystem) with the "actual" version (from the workspace after the
	// generated ALTERs were run there) by running a second diff. Verification
	// is successful if this second diff has no clauses (tables completely and
	// exactly match) or only a blank statement (suppressed by StatementModifiers).
	// We use very strict StatementModifiers here, except StrictColumnDefinition
	// must be omitted because MySQL 8 behaves inconsistently with respect to
	// superfluous column-level charset/collation clauses in some specific edge-
	// cases. (These MySQL 8 discrepancies are purely cosmetic, safe to ignore.)
	mods.StrictColumnDefinition = false
	mods.AlgorithmClause = ""
	actualTables := wsSchema.TablesByName()
	for name, desiredTable := range desiredTables {
		if err := verifyTable(actualTables[name], desiredTable, mods); err != nil {
			return err
		}
	}
	return nil
}

// verifyTable confirms that a table has the expected structure by doing an
// additional diff. Typically this diff will return quickly based on SHOW CREATE
// TABLE matching, but if they don't match (as happens with some MySQL 8 edge-
// cases) it will do a full structural comparison of the tables' fields. If this
// second diff returns a non-empty ALTER, an error, or an unsupported diff, it
// means the first diff did not properly do its job, so verification fails.
func verifyTable(actual, desired *tengo.Table, mods tengo.StatementModifiers) error {
	var unsupportedErr *tengo.UnsupportedDiffError
	td := tengo.NewAlterTable(actual, desired)
	stmt, err := td.Statement(mods)
	if errors.As(err, &unsupportedErr) {
		unsupportedErr.Reason = strings.Replace(unsupportedErr.Reason, "original state", "post-verification state", 1)
		unsupportedErr.ExpectedDesc = strings.Replace(unsupportedErr.ExpectedDesc, "original state", "post-verification state", 1)
		unsupportedErr.ActualDesc = strings.Replace(unsupportedErr.ActualDesc, "original state", "post-verification state", 1)
		return fmt.Errorf("Diff verification failure on table %s. This may indicate a Skeema bug.\nRun command again with --skip-verify if this discrepancy is safe to ignore.\nDebug details: %s", desired.Name, unsupportedErr.ExtendedError())
	} else if err != nil {
		return fmt.Errorf("Diff verification failure on table %s due to unexpected error: %w.\nRun command again with --skip-verify if this is safe to ignore.", desired.Name, err)
	} else if stmt != "" {
		return fmt.Errorf("Diff verification failure on table %s: the generated ALTER TABLE does not fully bring the table to the desired state.\nRun command again with --skip-verify if this discrepancy is safe to ignore.\nDebug details: secondary verification diff is non-empty, yielding this DDL: %s", desired.Name, stmt)
	}
	return nil
}
<commit_msg>diff/push: adjust --verify message generation<commit_after>package applier

import (
	"errors"
	"fmt"
	"strings"

	"github.com/skeema/skeema/internal/fs"
	"github.com/skeema/skeema/internal/tengo"
	"github.com/skeema/skeema/internal/workspace"
)

// VerifyDiff verifies the result of all AlterTable values found in
// diff.TableDiffs, confirming that applying the corresponding ALTER would
// bring a table from the version currently in the instance to the version
// specified in the filesystem.
func VerifyDiff(diff *tengo.SchemaDiff, t *Target) error {
	if !t.Dir.Config.GetBool("verify") || len(diff.TableDiffs) == 0 || t.briefOutput() {
		return nil
	}

	// If diff contains no ALTER TABLEs, nothing to verify
	altersInDiff := diff.FilteredTableDiffs(tengo.DiffTypeAlter)
	if len(altersInDiff) == 0 {
		return nil
	}

	// The goal of VerifyDiff is to confirm that the diff contains the correct and
	// complete set of differences between all modified tables. We use a strict set
	// of statement modifiers that will transform the initial state into an exact
	// match of the desired state. This is all run in a workspace, so we can be
	// more aggressive about which statement modifiers are used in generating the
	// ALTER here. When the diff is actually used against the real live table
	// later, a different looser set of modifiers is used which filters out some
	// of the undesired cosmetic clauses by default.
	mods := tengo.StatementModifiers{
		NextAutoInc:            tengo.NextAutoIncAlways,      // use whichever auto_increment is in the fs
		Partitioning:           tengo.PartitioningPermissive, // ditto with partitioning status
		AllowUnsafe:            true,                         // needed since we're just running against the temp schema
		AlgorithmClause:        "copy",                       // needed so the DB doesn't ignore attempts to re-order indexes
		StrictIndexOrder:       true,                         // needed since we want the SHOW CREATE TABLEs to match
		StrictCheckOrder:       true,                         // ditto (only affects MariaDB)
		StrictForeignKeyNaming: true,                         // ditto
		StrictColumnDefinition: true,                         // ditto (only affects MySQL 8 edge cases)
		SkipPreDropAlters:      true,                         // ignore DROP PARTITIONs that were only generated to speed up a DROP TABLE
		Flavor:                 t.Instance.Flavor(),
	}
	if mods.Flavor.Matches(tengo.FlavorMySQL55) {
		mods.AlgorithmClause = "" // MySQL 5.5 doesn't support ALGORITHM clause
	}

	// Gather CREATE and ALTER for modified tables, and put into a LogicalSchema,
	// which we then materialize into a real schema using a workspace
	logicalSchema := &fs.LogicalSchema{
		CharSet:   t.Dir.Config.Get("default-character-set"),
		Collation: t.Dir.Config.Get("default-collation"),
		Creates:   make(map[tengo.ObjectKey]*fs.Statement),
		Alters:    make([]*fs.Statement, 0),
	}
	desiredTables := make(map[string]*tengo.Table)
	for _, td := range altersInDiff {
		stmt, err := td.Statement(mods)
		if stmt != "" && err == nil {
			// Note: sometimes a table's diff gets split into multiple ALTERs, but this
			// logic can ignore that fact. If there are redundant AddStatement calls for
			// one CREATE, the first AddStatement for that CREATE succeeds and the
			// subsequent duplicate CREATEs error, but that is harmless in this code path!
			logicalSchema.AddStatement(&fs.Statement{
				Type:       fs.StatementTypeCreate,
				Text:       td.From.CreateStatement,
				ObjectType: tengo.ObjectTypeTable,
				ObjectName: td.From.Name,
			})
			logicalSchema.AddStatement(&fs.Statement{
				Type:       fs.StatementTypeAlter,
				Text:       stmt,
				ObjectType: tengo.ObjectTypeTable,
				ObjectName: td.From.Name,
			})
			desiredTables[td.From.Name] = td.To
		}
	}

	opts, err := workspace.OptionsForDir(t.Dir, t.Instance)
	if err != nil {
		return err
	}
	wsSchema, err := workspace.ExecLogicalSchema(logicalSchema, opts)
	if err == nil && len(wsSchema.Failures) > 0 {
		err = wsSchema.Failures[0]
	}
	if err != nil {
		return fmt.Errorf("Diff verification failure: %s", err.Error())
	}

	// Compare the "expected" version of each table ("to" side of original diff,
	// from the filesystem) with the "actual" version (from the workspace after the
	// generated ALTERs were run there) by running a second diff. Verification
	// is successful if this second diff has no clauses (tables completely and
	// exactly match) or only a blank statement (suppressed by StatementModifiers).
	// We use very strict StatementModifiers here, except StrictColumnDefinition
	// must be omitted because MySQL 8 behaves inconsistently with respect to
	// superfluous column-level charset/collation clauses in some specific edge-
	// cases. (These MySQL 8 discrepancies are purely cosmetic, safe to ignore.)
	mods.StrictColumnDefinition = false
	mods.AlgorithmClause = ""
	actualTables := wsSchema.TablesByName()
	for name, desiredTable := range desiredTables {
		if err := verifyTable(actualTables[name], desiredTable, mods); err != nil {
			return err
		}
	}
	return nil
}

// verifyTable confirms that a table has the expected structure by doing an
// additional diff. Typically this diff will return quickly based on SHOW CREATE
// TABLE matching, but if they don't match (as happens with some MySQL 8 edge-
// cases) it will do a full structural comparison of the tables' fields. If this
// second diff returns a non-empty ALTER, an error, or an unsupported diff, it
// means the first diff did not properly do its job, so verification fails.
func verifyTable(actual, desired *tengo.Table, mods tengo.StatementModifiers) error {
	var unsupportedErr *tengo.UnsupportedDiffError
	td := tengo.NewAlterTable(actual, desired)
	stmt, err := td.Statement(mods)
	header := "Diff verification failure on table " + desired.Name
	if errors.As(err, &unsupportedErr) {
		unsupportedErr.Reason = strings.Replace(unsupportedErr.Reason, "original state", "post-verification state", 1)
		unsupportedErr.ExpectedDesc = strings.Replace(unsupportedErr.ExpectedDesc, "original state", "post-verification state", 1)
		unsupportedErr.ActualDesc = strings.Replace(unsupportedErr.ActualDesc, "original state", "post-verification state", 1)
		return fmt.Errorf(header+". This may indicate a Skeema bug.\nRun command again with --skip-verify if this discrepancy is safe to ignore.\nDebug details: %s", unsupportedErr.ExtendedError())
	} else if err != nil {
		return fmt.Errorf(header+" due to unexpected error: %w.\nRun command again with --skip-verify if this is safe to ignore.", err)
	} else if stmt != "" {
		return fmt.Errorf(header+": the generated ALTER TABLE does not fully bring the table to the desired state.\nRun command again with --skip-verify if this discrepancy is safe to ignore.\nDebug details: secondary verification diff is non-empty, yielding this DDL: %s", stmt)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kms

import (
	"context"
	"encoding/base64"
	"errors"
	"fmt"

	"github.com/ceph/ceph-csi/internal/util/k8s"

	"github.com/aws/aws-sdk-go/aws"
	awsCreds "github.com/aws/aws-sdk-go/aws/credentials"
	awsSession "github.com/aws/aws-sdk-go/aws/session"
	awsKMS "github.com/aws/aws-sdk-go/service/kms"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	kmsTypeAWSMetadata = "aws-metadata"

	// awsMetadataDefaultSecretsName is the default name of the Kubernetes Secret
	// that contains the credentials to access the Amazon KMS. The name of
	// the Secret can be configured by setting the `KMS_SECRET_NAME`
	// option.
	//
	// #nosec:G101, value not credential, just references token.
	awsMetadataDefaultSecretsName = "ceph-csi-aws-credentials"

	// awsSecretNameKey contains the name of the Kubernetes Secret that has
	// the credentials to access the Amazon KMS.
	//
	// #nosec:G101, no hardcoded secret, this is a configuration key.
	awsSecretNameKey = "KMS_SECRET_NAME"
	awsRegionKey     = "AWS_REGION"

	// The following options are part of the Kubernetes Secrets.
	//
	// #nosec:G101, no hardcoded secrets, only configuration keys.
	awsAccessKey = "AWS_ACCESS_KEY_ID"
	// #nosec:G101.
	awsSecretAccessKey = "AWS_SECRET_ACCESS_KEY"
	// #nosec:G101.
	awsSessionToken = "AWS_SESSION_TOKEN"
	awsCMK          = "AWS_CMK_ARN"
)

var _ = RegisterProvider(Provider{
	UniqueID:    kmsTypeAWSMetadata,
	Initializer: initAWSMetadataKMS,
})

type AWSMetadataKMS struct {
	// basic options to get the secret
	namespace  string
	secretName string

	// standard AWS configuration options
	region          string
	secretAccessKey string
	accessKey       string
	sessionToken    string
	cmk             string
}

func initAWSMetadataKMS(args ProviderInitArgs) (EncryptionKMS, error) {
	kms := &AWSMetadataKMS{
		namespace: args.Namespace,
	}

	// required options for further configuration (getting secrets)
	err := setConfigString(&kms.secretName, args.Config, awsSecretNameKey)
	if errors.Is(err, errConfigOptionInvalid) {
		return nil, err
	} else if errors.Is(err, errConfigOptionMissing) {
		kms.secretName = awsMetadataDefaultSecretsName
	}
	err = setConfigString(&kms.region, args.Config, awsRegionKey)
	if err != nil {
		return nil, err
	}

	// read the Kubernetes Secret with credentials
	secrets, err := kms.getSecrets()
	if err != nil {
		return nil, fmt.Errorf("failed to get secrets for %T: %w", kms,
			err)
	}

	err = setConfigString(&kms.accessKey, secrets, awsAccessKey)
	if err != nil {
		return nil, err
	}
	err = setConfigString(&kms.secretAccessKey, secrets,
		awsSecretAccessKey)
	if err != nil {
		return nil, err
	}
	// awsSessionToken is optional
	err = setConfigString(&kms.sessionToken, secrets, awsSessionToken)
	if errors.Is(err, errConfigOptionInvalid) {
		return nil, err
	}
	err = setConfigString(&kms.cmk, secrets, awsCMK)
	if err != nil {
		return nil, err
	}

	return kms, nil
}

func (kms *AWSMetadataKMS) getSecrets() (map[string]interface{}, error) {
	c, err := k8s.NewK8sClient()
	if err != nil {
		return nil, fmt.Errorf("failed to connect to Kubernetes to "+
			"get Secret %s/%s: %w", kms.namespace, kms.secretName, err)
	}

	secret, err := c.CoreV1().Secrets(kms.namespace).Get(context.TODO(),
		kms.secretName, metav1.GetOptions{})
	if err != nil {
		return nil, fmt.Errorf("failed to get Secret %s/%s: %w",
			kms.namespace, kms.secretName, err)
	}

	config := make(map[string]interface{})

	for k, v := range secret.Data {
		switch k {
		case awsSecretAccessKey, awsAccessKey, awsSessionToken, awsCMK:
			config[k] = string(v)
		default:
			return nil, fmt.Errorf("unsupported option for KMS "+
				"provider %q: %s", kmsTypeAWSMetadata, k)
		}
	}

	return config, nil
}

func (kms *AWSMetadataKMS) Destroy() {
	// Nothing to do.
}

// RequiresDEKStore indicates that the DEKs should get stored in the metadata
// of the volumes. This Amazon KMS provider does not support storing DEKs in
// AWS as that adds additional costs.
func (kms *AWSMetadataKMS) RequiresDEKStore() DEKStoreType {
	return DEKStoreMetadata
}

func (kms *AWSMetadataKMS) getService() (*awsKMS.KMS, error) {
	creds := awsCreds.NewStaticCredentials(kms.accessKey,
		kms.secretAccessKey, kms.sessionToken)

	sess, err := awsSession.NewSessionWithOptions(awsSession.Options{
		SharedConfigState: awsSession.SharedConfigDisable,
		Config: aws.Config{
			Credentials: creds,
			Region:      aws.String(kms.region),
		},
	})
	if err != nil {
		return nil, fmt.Errorf("failed to create AWS session: %w", err)
	}

	return awsKMS.New(sess), nil
}

// EncryptDEK uses the Amazon KMS and the configured CMK to encrypt the DEK.
func (kms *AWSMetadataKMS) EncryptDEK(volumeID, plainDEK string) (string, error) {
	svc, err := kms.getService()
	if err != nil {
		return "", fmt.Errorf("could not get KMS service: %w", err)
	}

	result, err := svc.Encrypt(&awsKMS.EncryptInput{
		KeyId:     aws.String(kms.cmk),
		Plaintext: []byte(plainDEK),
	})
	if err != nil {
		return "", fmt.Errorf("failed to encrypt DEK: %w", err)
	}

	// base64 encode the encrypted DEK, so that storing it should not have
	// issues
	encryptedDEK :=
		base64.StdEncoding.EncodeToString(result.CiphertextBlob)

	return encryptedDEK, nil
}

// DecryptDEK uses the Amazon KMS and the configured CMK to decrypt the DEK.
func (kms *AWSMetadataKMS) DecryptDEK(volumeID, encryptedDEK string) (string, error) {
	svc, err := kms.getService()
	if err != nil {
		return "", fmt.Errorf("could not get KMS service: %w", err)
	}

	ciphertextBlob, err := base64.StdEncoding.DecodeString(encryptedDEK)
	if err != nil {
		return "", fmt.Errorf("failed to decode base64 cipher: %w",
			err)
	}

	result, err := svc.Decrypt(&awsKMS.DecryptInput{
		CiphertextBlob: ciphertextBlob,
	})
	if err != nil {
		return "", fmt.Errorf("failed to decrypt DEK: %w", err)
	}

	return string(result.Plaintext), nil
}
<commit_msg>rbd: unexport aws kms structs<commit_after>/*
Copyright 2021 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kms

import (
	"context"
	"encoding/base64"
	"errors"
	"fmt"

	"github.com/ceph/ceph-csi/internal/util/k8s"

	"github.com/aws/aws-sdk-go/aws"
	awsCreds "github.com/aws/aws-sdk-go/aws/credentials"
	awsSession "github.com/aws/aws-sdk-go/aws/session"
	awsKMS "github.com/aws/aws-sdk-go/service/kms"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	kmsTypeAWSMetadata = "aws-metadata"

	// awsMetadataDefaultSecretsName is the default name of the Kubernetes Secret
	// that contains the credentials to access the Amazon KMS. The name of
	// the Secret can be configured by setting the `KMS_SECRET_NAME`
	// option.
	//
	// #nosec:G101, value not credential, just references token.
	awsMetadataDefaultSecretsName = "ceph-csi-aws-credentials"

	// awsSecretNameKey contains the name of the Kubernetes Secret that has
	// the credentials to access the Amazon KMS.
	//
	// #nosec:G101, no hardcoded secret, this is a configuration key.
	awsSecretNameKey = "KMS_SECRET_NAME"
	awsRegionKey     = "AWS_REGION"

	// The following options are part of the Kubernetes Secrets.
	//
	// #nosec:G101, no hardcoded secrets, only configuration keys.
	awsAccessKey = "AWS_ACCESS_KEY_ID"
	// #nosec:G101.
	awsSecretAccessKey = "AWS_SECRET_ACCESS_KEY"
	// #nosec:G101.
	awsSessionToken = "AWS_SESSION_TOKEN"
	awsCMK          = "AWS_CMK_ARN"
)

var _ = RegisterProvider(Provider{
	UniqueID:    kmsTypeAWSMetadata,
	Initializer: initAWSMetadataKMS,
})

type awsMetadataKMS struct {
	// basic options to get the secret
	namespace  string
	secretName string

	// standard AWS configuration options
	region          string
	secretAccessKey string
	accessKey       string
	sessionToken    string
	cmk             string
}

func initAWSMetadataKMS(args ProviderInitArgs) (EncryptionKMS, error) {
	kms := &awsMetadataKMS{
		namespace: args.Namespace,
	}

	// required options for further configuration (getting secrets)
	err := setConfigString(&kms.secretName, args.Config, awsSecretNameKey)
	if errors.Is(err, errConfigOptionInvalid) {
		return nil, err
	} else if errors.Is(err, errConfigOptionMissing) {
		kms.secretName = awsMetadataDefaultSecretsName
	}
	err = setConfigString(&kms.region, args.Config, awsRegionKey)
	if err != nil {
		return nil, err
	}

	// read the Kubernetes Secret with credentials
	secrets, err := kms.getSecrets()
	if err != nil {
		return nil, fmt.Errorf("failed to get secrets for %T: %w", kms,
			err)
	}

	err = setConfigString(&kms.accessKey, secrets, awsAccessKey)
	if err != nil {
		return nil, err
	}
	err = setConfigString(&kms.secretAccessKey, secrets,
		awsSecretAccessKey)
	if err != nil {
		return nil, err
	}
	// awsSessionToken is optional
	err = setConfigString(&kms.sessionToken, secrets, awsSessionToken)
	if errors.Is(err, errConfigOptionInvalid) {
		return nil, err
	}
	err = setConfigString(&kms.cmk, secrets, awsCMK)
	if err != nil {
		return nil, err
	}

	return kms, nil
}

func (kms *awsMetadataKMS) getSecrets() (map[string]interface{}, error) {
	c, err := k8s.NewK8sClient()
	if err != nil {
		return nil, fmt.Errorf("failed to connect to Kubernetes to "+
			"get Secret %s/%s: %w", kms.namespace, kms.secretName, err)
	}

	secret, err := c.CoreV1().Secrets(kms.namespace).Get(context.TODO(),
		kms.secretName, metav1.GetOptions{})
	if err != nil {
		return nil, fmt.Errorf("failed to get Secret %s/%s: %w",
			kms.namespace, kms.secretName, err)
	}

	config := make(map[string]interface{})

	for k, v := range secret.Data {
		switch k {
		case awsSecretAccessKey, awsAccessKey, awsSessionToken, awsCMK:
			config[k] = string(v)
		default:
			return nil, fmt.Errorf("unsupported option for KMS "+
				"provider %q: %s", kmsTypeAWSMetadata, k)
		}
	}

	return config, nil
}

func (kms *awsMetadataKMS) Destroy() {
	// Nothing to do.
}

// RequiresDEKStore indicates that the DEKs should get stored in the metadata
// of the volumes. This Amazon KMS provider does not support storing DEKs in
// AWS as that adds additional costs.
func (kms *awsMetadataKMS) RequiresDEKStore() DEKStoreType {
	return DEKStoreMetadata
}

func (kms *awsMetadataKMS) getService() (*awsKMS.KMS, error) {
	creds := awsCreds.NewStaticCredentials(kms.accessKey,
		kms.secretAccessKey, kms.sessionToken)

	sess, err := awsSession.NewSessionWithOptions(awsSession.Options{
		SharedConfigState: awsSession.SharedConfigDisable,
		Config: aws.Config{
			Credentials: creds,
			Region:      aws.String(kms.region),
		},
	})
	if err != nil {
		return nil, fmt.Errorf("failed to create AWS session: %w", err)
	}

	return awsKMS.New(sess), nil
}

// EncryptDEK uses the Amazon KMS and the configured CMK to encrypt the DEK.
func (kms *awsMetadataKMS) EncryptDEK(volumeID, plainDEK string) (string, error) {
	svc, err := kms.getService()
	if err != nil {
		return "", fmt.Errorf("could not get KMS service: %w", err)
	}

	result, err := svc.Encrypt(&awsKMS.EncryptInput{
		KeyId:     aws.String(kms.cmk),
		Plaintext: []byte(plainDEK),
	})
	if err != nil {
		return "", fmt.Errorf("failed to encrypt DEK: %w", err)
	}

	// base64 encode the encrypted DEK, so that storing it should not have
	// issues
	encryptedDEK :=
		base64.StdEncoding.EncodeToString(result.CiphertextBlob)

	return encryptedDEK, nil
}

// DecryptDEK uses the Amazon KMS and the configured CMK to decrypt the DEK.
func (kms *awsMetadataKMS) DecryptDEK(volumeID, encryptedDEK string) (string, error) {
	svc, err := kms.getService()
	if err != nil {
		return "", fmt.Errorf("could not get KMS service: %w", err)
	}

	ciphertextBlob, err := base64.StdEncoding.DecodeString(encryptedDEK)
	if err != nil {
		return "", fmt.Errorf("failed to decode base64 cipher: %w",
			err)
	}

	result, err := svc.Decrypt(&awsKMS.DecryptInput{
		CiphertextBlob: ciphertextBlob,
	})
	if err != nil {
		return "", fmt.Errorf("failed to decrypt DEK: %w", err)
	}

	return string(result.Plaintext), nil
}
<|endoftext|>