<commit_before>/*
Package pipe provides filters that can be chained together in a manner
similar to Unix pipelines.

Each filter is a function that takes as input a sequence of
strings (read from a channel) and produces as output a sequence of
strings (written to a channel).

Filters can be chained together (e.g., via the Run function), the
output of one filter is fed as input to the next filter.  The empty
input is passed to the first filter. The following sequence will
print two lines to standard output:

	err := pipe.Run(
		pipe.Echo("hello", "world"),
		pipe.Reverse(),
		pipe.WriteLines(os.Stdout),
	)

An application can implement its own filters easily. For example,
repeat(n) returns a filter that repeats every input n times.

	func repeat(n int) Filter {
		return func(arg pipe.Arg) {
			for s := range arg.In {
				for i := 0; i < n; i++ {
					arg.Out <- s
				}
			}
		}
	}

	pipe.Run(
		pipe.Echo("hello"),
		repeat(10),
	)

Note that repeat is not a Filter since it needs to accept the
parameter n. Instead, it returns a Filter.  This convention is
followed throughout this library: all filtering functionality is
provided by functions that return a Filter.

*/
package pipe

import (
	"fmt"
	"regexp"
	"sync"
)

// filterErrors records errors accumulated during the execution of a filter.
type filterErrors struct {
	mu     sync.Mutex
	errors []error
}

// Arg contains the data passed to a Filter. Arg.In is a channel that
// produces the input to the filter, and Arg.Out is a channel that
// receives the output from the filter.
type Arg struct {
	In     <-chan string
	Out    chan<- string
	errors *filterErrors
}

// ReportError records an error encountered during an execution of a filter.
// This error will be reported by whatever facility (e.g., ForEach or Run)
// was being used to execute the filters.
//
// A filter should report any errors by calling ReportError.  Even if
// the filter has reported an error, it should read all data from
// arg.In, if only to disard it immediately.
func (a *Arg) ReportError(err error) {
	a.errors.mu.Lock()
	defer a.errors.mu.Unlock()
	a.errors.errors = append(a.errors.errors, err)
}

// Filter is the type of a function that reads a sequence of strings
// from a channel and produces a sequence on another channel.
type Filter func(Arg)

// Sequence returns a filter that is the concatenation of all filter arguments.
// The output of a filter is fed as input to the next filter.
func Sequence(filters ...Filter) Filter {
	return func(arg Arg) {
		in := arg.In
		for _, f := range filters {
			c := make(chan string, 10000)
			go runAndClose(f, Arg{in, c, arg.errors})
			in = c
		}
		passThrough(Arg{in, arg.Out, arg.errors})
	}
}

// Run executes the sequence of filters and discards all output.
// It returns either nil, an error if any filter reported an error.
func Run(filters ...Filter) error {
	return ForEach(Sequence(filters...), func(s string) {})
}

// ForEach calls fn(s) for every item s in the output of filter and
// returns either nil, or any error reported by the execution of the filter.
func ForEach(filter Filter, fn func(s string)) error {
	in := make(chan string, 0)
	close(in)
	out := make(chan string, 10000)
	e := &filterErrors{}
	go runAndClose(filter, Arg{in, out, e})
	for s := range out {
		fn(s)
	}

	e.mu.Lock()
	defer e.mu.Unlock()
	switch len(e.errors) {
	case 0:
		return nil
	case 1:
		return e.errors[0]
	default:
		return fmt.Errorf("Filter errors: %s", e.errors)
	}
}

func runAndClose(f Filter, arg Arg) {
	f(arg)
	close(arg.Out)
}

// passThrough copies all items read from in to out.
func passThrough(arg Arg) {
	for s := range arg.In {
		arg.Out <- s
	}
}

// Echo emits items.
// Any input items are copied verbatim to the output before items are emitted.
func Echo(items ...string) Filter {
	return func(arg Arg) {
		passThrough(arg)
		for _, s := range items {
			arg.Out <- s
		}
	}
}

// Numbers copies its input and then emits the integers x..y
func Numbers(x, y int) Filter {
	return func(arg Arg) {
		passThrough(arg)
		for i := x; i <= y; i++ {
			arg.Out <- fmt.Sprintf("%d", i)
		}
	}
}

// Map calls fn(x) for every item x and yields the outputs of the fn calls.
func Map(fn func(string) string) Filter {
	return func(arg Arg) {
		for s := range arg.In {
			arg.Out <- fn(s)
		}
	}
}

// If emits every input x for which fn(x) is true.
func If(fn func(string) bool) Filter {
	return func(arg Arg) {
		for s := range arg.In {
			if fn(s) {
				arg.Out <- s
			}
		}
	}
}

func errorFilter(err error) Filter {
	return func(arg Arg) {
		arg.ReportError(err)
		for _ = range arg.In {
			// Drop the input
		}
	}
}

// Grep emits every input x that matches the regular expression r.
func Grep(r string) Filter {
	re, err := regexp.Compile(r)
	if err != nil {
		return errorFilter(err)
	}
	return If(re.MatchString)
}

// GrepNot emits every input x that does not match the regular expression r.
func GrepNot(r string) Filter {
	re, err := regexp.Compile(r)
	if err != nil {
		return errorFilter(err)
	}
	return If(func(s string) bool { return !re.MatchString(s) })
}

// Uniq squashes adjacent identical items in arg.In into a single output.
func Uniq() Filter {
	return func(arg Arg) {
		first := true
		last := ""
		for s := range arg.In {
			if first || last != s {
				arg.Out <- s
			}
			last = s
			first = false
		}
	}
}

// UniqWithCount squashes adjacent identical items in arg.In into a single
// output prefixed with the count of identical items.
func UniqWithCount() Filter {
	return func(arg Arg) {
		current := ""
		count := 0
		for s := range arg.In {
			if s != current {
				if count > 0 {
					arg.Out <- fmt.Sprintf("%d %s", count, current)
				}
				count = 0
				current = s
			}
			count++
		}
		if count > 0 {
			arg.Out <- fmt.Sprintf("%d %s", count, current)
		}
	}
}

// Substitute replaces all occurrences of the regular expression r in
// an input item with replacement.  The replacement string can contain
// $1, $2, etc. which represent submatches of r.
func Substitute(r, replacement string) Filter {
	re, err := regexp.Compile(r)
	if err != nil {
		return errorFilter(err)
	}
	return func(arg Arg) {
		for s := range arg.In {
			arg.Out <- re.ReplaceAllString(s, replacement)
		}
	}
}

// Reverse yields items in the reverse of the order it received them.
func Reverse() Filter {
	return func(arg Arg) {
		var data []string
		for s := range arg.In {
			data = append(data, s)
		}
		for i := len(data) - 1; i >= 0; i-- {
			arg.Out <- data[i]
		}
	}
}

// First yields the first n items that it receives.
func First(n int) Filter {
	return func(arg Arg) {
		emitted := 0
		for s := range arg.In {
			if emitted < n {
				arg.Out <- s
				emitted++
			}
		}
	}
}

// DropFirst yields all items except for the first n items that it receives.
func DropFirst(n int) Filter {
	return func(arg Arg) {
		emitted := 0
		for s := range arg.In {
			if emitted >= n {
				arg.Out <- s
			}
			emitted++
		}
	}
}

// Last yields the last n items that it receives.
func Last(n int) Filter {
	return func(arg Arg) {
		var buf []string
		for s := range arg.In {
			buf = append(buf, s)
			if len(buf) > n {
				buf = buf[1:]
			}
		}
		for _, s := range buf {
			arg.Out <- s
		}
	}
}

// DropLast yields all items except for the last n items that it receives.
func DropLast(n int) Filter {
	return func(arg Arg) {
		var buf []string
		for s := range arg.In {
			buf = append(buf, s)
			if len(buf) > n {
				arg.Out <- buf[0]
				buf = buf[1:]
			}
		}
	}
}

// NumberLines prefixes its item with its index in the input sequence
// (starting at 1).
func NumberLines() Filter {
	return func(arg Arg) {
		line := 1
		for s := range arg.In {
			arg.Out <- fmt.Sprintf("%5d %s", line, s)
			line++
		}
	}
}

// Slice emits s[startOffset:endOffset] for each input item s.  Note
// that Slice follows Go conventions, and unlike the "cut" utility,
// offsets are numbered starting at zero, and the end offset is not
// included in the output.
func Slice(startOffset, endOffset int) Filter {
	return func(arg Arg) {
		for s := range arg.In {
			if len(s) > endOffset {
				s = s[:endOffset]
			}
			if len(s) < startOffset {
				s = ""
			} else {
				s = s[startOffset:]
			}
			arg.Out <- s
		}
	}
}

// Select splits each item into columns and yields the concatenation
// of the columns numbers passed as arguments to Select.  Columns are
// numbered starting at 1. A column number of 0 is interpreted as the
// full string.
func Select(columns ...int) Filter {
	return func(arg Arg) {
		for s := range arg.In {
			result := ""
			for _, col := range columns {
				if _, c := column(s, col); c != "" {
					if result != "" {
						result = result + " "
					}
					result = result + c
				}
			}
			arg.Out <- result
		}
	}
}
<commit_msg>small error string fix<commit_after>/*
Package pipe provides filters that can be chained together in a manner
similar to Unix pipelines.

Each filter is a function that takes as input a sequence of
strings (read from a channel) and produces as output a sequence of
strings (written to a channel).

Filters can be chained together (e.g., via the Run function), the
output of one filter is fed as input to the next filter.  The empty
input is passed to the first filter. The following sequence will
print two lines to standard output:

	err := pipe.Run(
		pipe.Echo("hello", "world"),
		pipe.Reverse(),
		pipe.WriteLines(os.Stdout),
	)

An application can implement its own filters easily. For example,
repeat(n) returns a filter that repeats every input n times.

	func repeat(n int) Filter {
		return func(arg pipe.Arg) {
			for s := range arg.In {
				for i := 0; i < n; i++ {
					arg.Out <- s
				}
			}
		}
	}

	pipe.Run(
		pipe.Echo("hello"),
		repeat(10),
	)

Note that repeat is not a Filter since it needs to accept the
parameter n. Instead, it returns a Filter.  This convention is
followed throughout this library: all filtering functionality is
provided by functions that return a Filter.

*/
package pipe

import (
	"fmt"
	"regexp"
	"sync"
)

// filterErrors records errors accumulated during the execution of a filter.
type filterErrors struct {
	mu     sync.Mutex
	errors []error
}

// Arg contains the data passed to a Filter. Arg.In is a channel that
// produces the input to the filter, and Arg.Out is a channel that
// receives the output from the filter.
type Arg struct {
	In     <-chan string
	Out    chan<- string
	errors *filterErrors
}

// ReportError records an error encountered during an execution of a filter.
// This error will be reported by whatever facility (e.g., ForEach or Run)
// was being used to execute the filters.
//
// A filter should report any errors by calling ReportError.  Even if
// the filter has reported an error, it should read all data from
// arg.In, if only to disard it immediately.
func (a *Arg) ReportError(err error) {
	a.errors.mu.Lock()
	defer a.errors.mu.Unlock()
	a.errors.errors = append(a.errors.errors, err)
}

// Filter is the type of a function that reads a sequence of strings
// from a channel and produces a sequence on another channel.
type Filter func(Arg)

// Sequence returns a filter that is the concatenation of all filter arguments.
// The output of a filter is fed as input to the next filter.
func Sequence(filters ...Filter) Filter {
	return func(arg Arg) {
		in := arg.In
		for _, f := range filters {
			c := make(chan string, 10000)
			go runAndClose(f, Arg{in, c, arg.errors})
			in = c
		}
		passThrough(Arg{in, arg.Out, arg.errors})
	}
}

// Run executes the sequence of filters and discards all output.
// It returns either nil, an error if any filter reported an error.
func Run(filters ...Filter) error {
	return ForEach(Sequence(filters...), func(s string) {})
}

// ForEach calls fn(s) for every item s in the output of filter and
// returns either nil, or any error reported by the execution of the filter.
func ForEach(filter Filter, fn func(s string)) error {
	in := make(chan string, 0)
	close(in)
	out := make(chan string, 10000)
	e := &filterErrors{}
	go runAndClose(filter, Arg{in, out, e})
	for s := range out {
		fn(s)
	}

	e.mu.Lock()
	defer e.mu.Unlock()
	switch len(e.errors) {
	case 0:
		return nil
	case 1:
		return e.errors[0]
	default:
		return fmt.Errorf("pipe.Filter errors: %s", e.errors)
	}
}

func runAndClose(f Filter, arg Arg) {
	f(arg)
	close(arg.Out)
}

// passThrough copies all items read from in to out.
func passThrough(arg Arg) {
	for s := range arg.In {
		arg.Out <- s
	}
}

// Echo emits items.
// Any input items are copied verbatim to the output before items are emitted.
func Echo(items ...string) Filter {
	return func(arg Arg) {
		passThrough(arg)
		for _, s := range items {
			arg.Out <- s
		}
	}
}

// Numbers copies its input and then emits the integers x..y
func Numbers(x, y int) Filter {
	return func(arg Arg) {
		passThrough(arg)
		for i := x; i <= y; i++ {
			arg.Out <- fmt.Sprintf("%d", i)
		}
	}
}

// Map calls fn(x) for every item x and yields the outputs of the fn calls.
func Map(fn func(string) string) Filter {
	return func(arg Arg) {
		for s := range arg.In {
			arg.Out <- fn(s)
		}
	}
}

// If emits every input x for which fn(x) is true.
func If(fn func(string) bool) Filter {
	return func(arg Arg) {
		for s := range arg.In {
			if fn(s) {
				arg.Out <- s
			}
		}
	}
}

func errorFilter(err error) Filter {
	return func(arg Arg) {
		arg.ReportError(err)
		for _ = range arg.In {
			// Drop the input
		}
	}
}

// Grep emits every input x that matches the regular expression r.
func Grep(r string) Filter {
	re, err := regexp.Compile(r)
	if err != nil {
		return errorFilter(err)
	}
	return If(re.MatchString)
}

// GrepNot emits every input x that does not match the regular expression r.
func GrepNot(r string) Filter {
	re, err := regexp.Compile(r)
	if err != nil {
		return errorFilter(err)
	}
	return If(func(s string) bool { return !re.MatchString(s) })
}

// Uniq squashes adjacent identical items in arg.In into a single output.
func Uniq() Filter {
	return func(arg Arg) {
		first := true
		last := ""
		for s := range arg.In {
			if first || last != s {
				arg.Out <- s
			}
			last = s
			first = false
		}
	}
}

// UniqWithCount squashes adjacent identical items in arg.In into a single
// output prefixed with the count of identical items.
func UniqWithCount() Filter {
	return func(arg Arg) {
		current := ""
		count := 0
		for s := range arg.In {
			if s != current {
				if count > 0 {
					arg.Out <- fmt.Sprintf("%d %s", count, current)
				}
				count = 0
				current = s
			}
			count++
		}
		if count > 0 {
			arg.Out <- fmt.Sprintf("%d %s", count, current)
		}
	}
}

// Substitute replaces all occurrences of the regular expression r in
// an input item with replacement.  The replacement string can contain
// $1, $2, etc. which represent submatches of r.
func Substitute(r, replacement string) Filter {
	re, err := regexp.Compile(r)
	if err != nil {
		return errorFilter(err)
	}
	return func(arg Arg) {
		for s := range arg.In {
			arg.Out <- re.ReplaceAllString(s, replacement)
		}
	}
}

// Reverse yields items in the reverse of the order it received them.
func Reverse() Filter {
	return func(arg Arg) {
		var data []string
		for s := range arg.In {
			data = append(data, s)
		}
		for i := len(data) - 1; i >= 0; i-- {
			arg.Out <- data[i]
		}
	}
}

// First yields the first n items that it receives.
func First(n int) Filter {
	return func(arg Arg) {
		emitted := 0
		for s := range arg.In {
			if emitted < n {
				arg.Out <- s
				emitted++
			}
		}
	}
}

// DropFirst yields all items except for the first n items that it receives.
func DropFirst(n int) Filter {
	return func(arg Arg) {
		emitted := 0
		for s := range arg.In {
			if emitted >= n {
				arg.Out <- s
			}
			emitted++
		}
	}
}

// Last yields the last n items that it receives.
func Last(n int) Filter {
	return func(arg Arg) {
		var buf []string
		for s := range arg.In {
			buf = append(buf, s)
			if len(buf) > n {
				buf = buf[1:]
			}
		}
		for _, s := range buf {
			arg.Out <- s
		}
	}
}

// DropLast yields all items except for the last n items that it receives.
func DropLast(n int) Filter {
	return func(arg Arg) {
		var buf []string
		for s := range arg.In {
			buf = append(buf, s)
			if len(buf) > n {
				arg.Out <- buf[0]
				buf = buf[1:]
			}
		}
	}
}

// NumberLines prefixes its item with its index in the input sequence
// (starting at 1).
func NumberLines() Filter {
	return func(arg Arg) {
		line := 1
		for s := range arg.In {
			arg.Out <- fmt.Sprintf("%5d %s", line, s)
			line++
		}
	}
}

// Slice emits s[startOffset:endOffset] for each input item s.  Note
// that Slice follows Go conventions, and unlike the "cut" utility,
// offsets are numbered starting at zero, and the end offset is not
// included in the output.
func Slice(startOffset, endOffset int) Filter {
	return func(arg Arg) {
		for s := range arg.In {
			if len(s) > endOffset {
				s = s[:endOffset]
			}
			if len(s) < startOffset {
				s = ""
			} else {
				s = s[startOffset:]
			}
			arg.Out <- s
		}
	}
}

// Select splits each item into columns and yields the concatenation
// of the columns numbers passed as arguments to Select.  Columns are
// numbered starting at 1. A column number of 0 is interpreted as the
// full string.
func Select(columns ...int) Filter {
	return func(arg Arg) {
		for s := range arg.In {
			result := ""
			for _, col := range columns {
				if _, c := column(s, col); c != "" {
					if result != "" {
						result = result + " "
					}
					result = result + c
				}
			}
			arg.Out <- result
		}
	}
}
<|endoftext|>


<commit_before>package mem_constrained_buffer

import (
	"bytes"
	"io"
	"io/ioutil"
	"mime/multipart"
	"os"
)

var (
	DefaultMemorySize int64 = 1 << 17 // 128K
	FilenamePrefix          = "mem-buf-"
)

type MemoryConstrainedBuffer struct {
	b             bytes.Buffer
	tmpfile       string
	max           int64
	size          int64
	removeOnClose bool
	file          multipart.File
}

func New() *MemoryConstrainedBuffer {
	return NewWithSize(DefaultMemorySize, true)
}

func NewWithSize(maxMemory int64, removeOnClose bool) *MemoryConstrainedBuffer {
	return &MemoryConstrainedBuffer{
		max:           maxMemory,
		removeOnClose: removeOnClose,
	}
}

func (m *MemoryConstrainedBuffer) Write(p []byte) (int, error) {
	n, err := m.ReadFrom(bytes.NewReader(p))
	return int(n), err
}

func (m *MemoryConstrainedBuffer) open() error {
	if m.file != nil {
		return nil
	}
	if m.tmpfile == "" {
		m.file = &sectionReadCloser{
			io.NewSectionReader(bytes.NewReader(m.b.Bytes()), 0, int64(m.b.Len()))}
		return nil
	}
	f, err := os.Open(m.tmpfile)
	m.file = f
	return err
}

func (m *MemoryConstrainedBuffer) close() error {
	if m.file == nil {
		return nil
	}
	err := m.file.Close()
	m.file = nil
	return err
}

func (m *MemoryConstrainedBuffer) Read(p []byte) (int, error) {
	if err := m.open(); err != nil {
		return 0, err
	}
	defer m.close()
	n, err := m.file.Read(p)
	return n, err
}

func (m *MemoryConstrainedBuffer) ReadAt(p []byte, off int64) (int, error) {
	if m.file == nil {
		if err := m.open(); err != nil {
			return 0, err
		}
	}
	defer m.close()
	return m.file.ReadAt(p, off)
}

func (m *MemoryConstrainedBuffer) ReadFrom(r io.Reader) (int64, error) {
	var (
		n   int64
		err error
	)

	for {
		n, err = io.CopyN(&m.b, r, m.max+1)
		if err != nil && err != io.EOF {
			return 0, err
		}

		m.size += n

		if err == io.EOF {
			err = nil
			break
		}

		if n > m.max {
			// too big, write to disk and flush buffer
			file, err := ioutil.TempFile("", FilenamePrefix)
			if err != nil {
				return 0, err
			}

			n, err = io.Copy(file, io.MultiReader(&m.b, r))
			if err != nil {
				file.Close()
				os.Remove(file.Name())
				return 0, err
			}
			m.b.Reset()
			m.tmpfile = file.Name()
			m.file = file
			m.file.Seek(0, 0)
			m.size = n
			break

		} else {
			m.max -= n
		}
	}

	return m.size, err
}

func (m *MemoryConstrainedBuffer) Len() int64 {
	return m.size
}

func (m *MemoryConstrainedBuffer) Seek(offset int64, whence int) (int64, error) {
	if err := m.open(); err != nil {
		return 0, err
	}
	return m.file.Seek(offset, whence)
}

func (m *MemoryConstrainedBuffer) Remove() (err error) {
	if m.file == nil && m.tmpfile == "" {
		return nil
	}
	err = m.close()
	if m.tmpfile != "" {
		err = os.Remove(m.tmpfile)
	}
	return err
}

func (m *MemoryConstrainedBuffer) Close() error {
	m.b.Reset()
	err := m.close()
	if m.removeOnClose {
		err = m.Remove()
	}
	return err
}

type sectionReadCloser struct {
	*io.SectionReader
}

func (rc sectionReadCloser) Close() error {
	return nil
}
<commit_msg>reverted to previous<commit_after>package mem_constrained_buffer

import (
	"bytes"
	"io"
	"io/ioutil"
	"mime/multipart"
	"os"
)

var (
	DefaultMemorySize int64 = 1 << 17 // 128K
	FilenamePrefix          = "mem-buf-"
)

type MemoryConstrainedBuffer struct {
	b             bytes.Buffer
	tmpfile       string
	max           int64
	size          int64
	removeOnClose bool
	file          multipart.File
}

func New() *MemoryConstrainedBuffer {
	return NewWithSize(DefaultMemorySize, true)
}

func NewWithSize(maxMemory int64, removeOnClose bool) *MemoryConstrainedBuffer {
	return &MemoryConstrainedBuffer{
		max:           maxMemory,
		removeOnClose: removeOnClose,
	}
}

func (m *MemoryConstrainedBuffer) Write(p []byte) (int, error) {
	n, err := m.ReadFrom(bytes.NewReader(p))
	return int(n), err
}

func (m *MemoryConstrainedBuffer) open() error {
	if m.file != nil {
		return nil
	}
	if m.tmpfile == "" {
		m.file = &sectionReadCloser{
			io.NewSectionReader(bytes.NewReader(m.b.Bytes()), 0, int64(m.b.Len()))}
		return nil
	}
	f, err := os.Open(m.tmpfile)
	m.file = f
	return err
}

func (m *MemoryConstrainedBuffer) Read(p []byte) (int, error) {
	if err := m.open(); err != nil {
		return 0, err
	}
	n, err := m.file.Read(p)
	return n, err
}

func (m *MemoryConstrainedBuffer) ReadAt(p []byte, off int64) (int, error) {
	if m.file == nil {
		if err := m.open(); err != nil {
			return 0, err
		}
	}
	return m.file.ReadAt(p, off)
}

func (m *MemoryConstrainedBuffer) ReadFrom(r io.Reader) (int64, error) {
	var (
		n   int64
		err error
	)

	for {
		n, err = io.CopyN(&m.b, r, m.max+1)
		if err != nil && err != io.EOF {
			return 0, err
		}

		m.size += n

		if err == io.EOF {
			err = nil
			break
		}

		if n > m.max {
			// too big, write to disk and flush buffer
			file, err := ioutil.TempFile("", FilenamePrefix)
			if err != nil {
				return 0, err
			}

			n, err = io.Copy(file, io.MultiReader(&m.b, r))
			if err != nil {
				file.Close()
				os.Remove(file.Name())
				return 0, err
			}
			m.b.Reset()
			m.tmpfile = file.Name()
			m.file = file
			m.file.Seek(0, 0)
			m.size = n
			break

		} else {
			m.max -= n
		}
	}

	return m.size, err
}

func (m *MemoryConstrainedBuffer) Len() int64 {
	return m.size
}

func (m *MemoryConstrainedBuffer) Seek(offset int64, whence int) (int64, error) {
	if err := m.open(); err != nil {
		return 0, err
	}
	return m.file.Seek(offset, whence)
}

func (m *MemoryConstrainedBuffer) Remove() (err error) {
	if m.file == nil && m.tmpfile == "" {
		return nil
	}
	if m.file != nil {
		err = m.file.Close()
	}
	if m.tmpfile != "" {
		err = os.Remove(m.tmpfile)
	}
	return err
}

func (m *MemoryConstrainedBuffer) Close() error {
	m.b.Reset()
	if m.file == nil {
		return nil
	}
	err := m.file.Close()
	if m.removeOnClose {
		err = m.Remove()
	}
	return err
}

type sectionReadCloser struct {
	*io.SectionReader
}

func (rc sectionReadCloser) Close() error {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"

	"go.mozilla.org/autograph/signer"
)

// authBackend is an interface for adding and finding HAWK users and
// their permissions
type authBackend interface {
	addAuth(*authorization) error
	addMonitoringAuth(string) error
	getAuthByID(id string) (authorization, error)
	getSignerID(userid, keyid string) (int, error)
	makeSignerIndex([]signer.Signer) error
}

// inMemoryBackend is an authBackend that loads a config and stores
// that auth info in memory
type inMemoryBackend struct {
	auths       map[string]authorization
	signerIndex map[string]int
}

// newInMemoryAuthBackend returns an empty inMemoryBackend
func newInMemoryAuthBackend() (backend *inMemoryBackend) {
	return &inMemoryBackend{
		auths:       make(map[string]authorization),
		signerIndex: make(map[string]int),
	}
}

// addAuth adds an authorization to the auth map or errors
func (b *inMemoryBackend) addAuth(auth *authorization) (err error) {
	_, getAuthErr := b.getAuthByID(auth.ID)
	switch getAuthErr {
	case nil:
		return errors.Errorf("authorization id '%s' already defined, duplicates are not permitted", auth.ID)
	case ErrAuthNotFound:
		// this is what we want
	default:
		return errors.Wrapf(getAuthErr, "error finding auth with id '%s'", auth.ID)
	}
	if auth.HawkTimestampValidity != "" {
		auth.hawkMaxTimestampSkew, err = time.ParseDuration(auth.HawkTimestampValidity)
		if err != nil {
			return err
		}
	} else {
		auth.hawkMaxTimestampSkew = time.Minute
	}
	b.auths[auth.ID] = *auth
	return nil
}

// getAuthByID returns an authorization if it exists or nil. Call
// addAuthorizations and addMonitoring first
func (b *inMemoryBackend) getAuthByID(id string) (authorization, error) {
	if auth, ok := b.auths[id]; ok {
		return auth, nil
	}
	return authorization{}, ErrAuthNotFound
}

// addMonitoringAuth adds an authorization to enable the
// tools/autograph-monitor
func (b *inMemoryBackend) addMonitoringAuth(monitorKey string) error {
	_, err := b.getAuthByID(monitorAuthID)
	switch err {
	case ErrAuthNotFound:
	case nil:
		return errors.Errorf("user 'monitor' is reserved for monitoring, duplication is not permitted")
	default:
		return errors.Errorf("error fetching 'monitor' auth: %q", err)
	}
	return b.addAuth(&authorization{
		ID:                    monitorAuthID,
		Key:                   monitorKey,
		HawkTimestampValidity: "1m",
		hawkMaxTimestampSkew:  time.Minute,
	})
}

// getSignerId returns the signer identifier for the user. If a keyid
// is specified, the corresponding signer is returned. If no signer is
// found, an error is returned and the signer identifier is set to -1.
func (b *inMemoryBackend) getSignerID(userid, keyid string) (int, error) {
	tag := userid + "+" + keyid
	if _, ok := b.signerIndex[tag]; !ok {
		if keyid == "" {
			return -1, errors.Errorf("%q does not have a default signing key", userid)
		}
		return -1, errors.Errorf("%s is not authorized to sign with key ID %s", userid, keyid)
	}
	return b.signerIndex[tag], nil
}

// makeSignerIndex creates a map of authorization IDs and signer IDs to
// quickly locate a signer based on the user requesting the signature.
func (b *inMemoryBackend) makeSignerIndex(signers []signer.Signer) error {
	// add an entry for each authid+signerid pair
	for id, auth := range b.auths {
		if id == monitorAuthID {
			// the "monitor" authorization is a special case
			// that doesn't need a signer index
			continue
		}
		// if the authorization has no signer configured, error out
		if len(auth.Signers) < 1 {
			return errors.Errorf("auth id %q must have at least one signer configured", id)
		}
		for _, sid := range auth.Signers {
			// make sure the sid is valid
			sidExists := false

			for pos, s := range signers {
				if sid == s.Config().ID {
					sidExists = true
					log.Printf("Mapping auth id %q and signer id %q to signer %d with hawk ts validity %s", auth.ID, s.Config().ID, pos, auth.hawkMaxTimestampSkew)
					tag := auth.ID + "+" + s.Config().ID
					b.signerIndex[tag] = pos
				}
			}

			if !sidExists {
				return errors.Errorf("in auth id %q, signer id %q was not found in the list of known signers", auth.ID, sid)
			}
		}
		// add a default entry for the signer, such that if none is provided in
		// the signing request, the default is used
		for pos, signer := range signers {
			if auth.Signers[0] == signer.Config().ID {
				log.Printf("Mapping auth id %q to default signer %d with hawk ts validity %s", auth.ID, pos, auth.hawkMaxTimestampSkew)
				tag := auth.ID + "+"
				b.signerIndex[tag] = pos
				break
			}
		}
	}
	return nil
}
<commit_msg>unnest outer loop of makeSignerIndex<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"

	"go.mozilla.org/autograph/signer"
)

// authBackend is an interface for adding and finding HAWK users and
// their permissions
type authBackend interface {
	addAuth(*authorization) error
	addMonitoringAuth(string) error
	getAuthByID(id string) (authorization, error)
	getSignerID(userid, keyid string) (int, error)
	makeSignerIndex([]signer.Signer) error
}

// inMemoryBackend is an authBackend that loads a config and stores
// that auth info in memory
type inMemoryBackend struct {
	auths       map[string]authorization
	signerIndex map[string]int
}

// newInMemoryAuthBackend returns an empty inMemoryBackend
func newInMemoryAuthBackend() (backend *inMemoryBackend) {
	return &inMemoryBackend{
		auths:       make(map[string]authorization),
		signerIndex: make(map[string]int),
	}
}

// addAuth adds an authorization to the auth map or errors
func (b *inMemoryBackend) addAuth(auth *authorization) (err error) {
	_, getAuthErr := b.getAuthByID(auth.ID)
	switch getAuthErr {
	case nil:
		return errors.Errorf("authorization id '%s' already defined, duplicates are not permitted", auth.ID)
	case ErrAuthNotFound:
		// this is what we want
	default:
		return errors.Wrapf(getAuthErr, "error finding auth with id '%s'", auth.ID)
	}
	if auth.HawkTimestampValidity != "" {
		auth.hawkMaxTimestampSkew, err = time.ParseDuration(auth.HawkTimestampValidity)
		if err != nil {
			return err
		}
	} else {
		auth.hawkMaxTimestampSkew = time.Minute
	}
	b.auths[auth.ID] = *auth
	return nil
}

// getAuthByID returns an authorization if it exists or nil. Call
// addAuthorizations and addMonitoring first
func (b *inMemoryBackend) getAuthByID(id string) (authorization, error) {
	if auth, ok := b.auths[id]; ok {
		return auth, nil
	}
	return authorization{}, ErrAuthNotFound
}

// addMonitoringAuth adds an authorization to enable the
// tools/autograph-monitor
func (b *inMemoryBackend) addMonitoringAuth(monitorKey string) error {
	_, err := b.getAuthByID(monitorAuthID)
	switch err {
	case ErrAuthNotFound:
	case nil:
		return errors.Errorf("user 'monitor' is reserved for monitoring, duplication is not permitted")
	default:
		return errors.Errorf("error fetching 'monitor' auth: %q", err)
	}
	return b.addAuth(&authorization{
		ID:                    monitorAuthID,
		Key:                   monitorKey,
		HawkTimestampValidity: "1m",
		hawkMaxTimestampSkew:  time.Minute,
	})
}

// getSignerId returns the signer identifier for the user. If a keyid
// is specified, the corresponding signer is returned. If no signer is
// found, an error is returned and the signer identifier is set to -1.
func (b *inMemoryBackend) getSignerID(userid, keyid string) (int, error) {
	tag := getSignerIndexTag(userid, keyid)
	if _, ok := b.signerIndex[tag]; !ok {
		if keyid == "" {
			return -1, errors.Errorf("%q does not have a default signing key", userid)
		}
		return -1, errors.Errorf("%s is not authorized to sign with key ID %s", userid, keyid)
	}
	return b.signerIndex[tag], nil
}

// getSignerIndexTag returns the tag to lookup the signer for a hawk user
func getSignerIndexTag(authID, signerID string) string {
	return fmt.Sprintf("%s+%s", authID, signerID)
}

// addAuthToSignerIndex
func (b *inMemoryBackend) addAuthToSignerIndex(auth authorization, signers []signer.Signer) error {
	// the "monitor" authorization is doesn't need a signer index
	if auth.ID == monitorAuthID {
		return nil
	}
	// authorization must have a signer configured
	if len(auth.Signers) < 1 {
		return errors.Errorf("auth id %q must have at least one signer configured", auth.ID)
	}
	// add an authid+signerid entry for each signer the auth grants access to
	for _, sid := range auth.Signers {
		// make sure the sid is valid
		sidExists := false

		for pos, s := range signers {
			if sid == s.Config().ID {
				sidExists = true
				log.Printf("Mapping auth id %q and signer id %q to signer %d with hawk ts validity %s", auth.ID, s.Config().ID, pos, auth.hawkMaxTimestampSkew)
				b.signerIndex[getSignerIndexTag(auth.ID, s.Config().ID)] = pos
			}
		}

		if !sidExists {
			return errors.Errorf("in auth id %q, signer id %q was not found in the list of known signers", auth.ID, sid)
		}
	}
	// add a default entry for the signer, such that if none is provided in
	// the signing request, the default is used
	for pos, signer := range signers {
		if auth.Signers[0] == signer.Config().ID {
			log.Printf("Mapping auth id %q to default signer %d with hawk ts validity %s", auth.ID, pos, auth.hawkMaxTimestampSkew)
			tag := auth.ID + "+"
			b.signerIndex[tag] = pos
			break
		}
	}
	return nil
}

// makeSignerIndex creates a map of authorization IDs and signer IDs to
// quickly locate a signer based on the user requesting the signature.
func (b *inMemoryBackend) makeSignerIndex(signers []signer.Signer) error {
	for _, auth := range b.auths {
		err := b.addAuthToSignerIndex(auth, signers)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package pgx

import (
	"bytes"
	"encoding/binary"
)

// MessageReader is a helper that reads values from a PostgreSQL message.
type MessageReader struct {
	buf *bytes.Buffer
}

func newMessageReader(buf *bytes.Buffer) *MessageReader {
	return &MessageReader{buf: buf}
}

func (r *MessageReader) ReadByte() byte {
	b, err := r.buf.ReadByte()
	if err != nil {
		panic("Unable to read byte")
	}
	return b
}

func (r *MessageReader) ReadInt16() int16 {
	return int16(binary.BigEndian.Uint16(r.buf.Next(2)))
}

func (r *MessageReader) ReadInt32() int32 {
	return int32(binary.BigEndian.Uint32(r.buf.Next(4)))
}

func (r *MessageReader) ReadInt64() int64 {
	return int64(binary.BigEndian.Uint64(r.buf.Next(8)))
}

func (r *MessageReader) ReadOid() Oid {
	return Oid(binary.BigEndian.Uint32(r.buf.Next(4)))
}

// ReadString reads a null terminated string
func (r *MessageReader) ReadString() string {
	b, err := r.buf.ReadBytes(0)
	if err != nil {
		panic("Unable to read string")
	}
	return string(b[:len(b)-1])
}

// ReadByteString reads count bytes and return as string
func (r *MessageReader) ReadByteString(count int32) string {
	return string(r.buf.Next(int(count)))
}
<commit_msg>Use deferred error handling for pgx.MessageReader<commit_after>package pgx

import (
	"bytes"
	"encoding/binary"
	"fmt"
)

// MessageReader is a helper that reads values from a PostgreSQL message.
// To avoid verbose error handling it internally records errors and no-ops
// any calls that occur after an error. At the end of a sequence of reads
// the Err field should be checked to see if any errors occurred.
type MessageReader struct {
	buf *bytes.Buffer
	Err error
}

func newMessageReader(buf *bytes.Buffer) *MessageReader {
	return &MessageReader{buf: buf}
}

func (r *MessageReader) ReadByte() (b byte) {
	if r.Err != nil {
		return
	}

	b, r.Err = r.buf.ReadByte()
	return
}

func (r *MessageReader) ReadInt16() (n int16) {
	if r.Err != nil {
		return
	}

	size := 2
	b := r.buf.Next(size)
	if len(b) != size {
		r.Err = fmt.Errorf("Unable to read %d bytes, only read %d", size, len(b))
	}

	return int16(binary.BigEndian.Uint16(b))
}

func (r *MessageReader) ReadInt32() (n int32) {
	if r.Err != nil {
		return
	}

	size := 4
	b := r.buf.Next(size)
	if len(b) != size {
		r.Err = fmt.Errorf("Unable to read %d bytes, only read %d", size, len(b))
	}

	return int32(binary.BigEndian.Uint32(b))
}

func (r *MessageReader) ReadInt64() (n int64) {
	if r.Err != nil {
		return
	}

	size := 8
	b := r.buf.Next(size)
	if len(b) != size {
		r.Err = fmt.Errorf("Unable to read %d bytes, only read %d", size, len(b))
	}

	return int64(binary.BigEndian.Uint64(b))
}

func (r *MessageReader) ReadOid() (oid Oid) {
	if r.Err != nil {
		return
	}

	size := 4
	b := r.buf.Next(size)
	if len(b) != size {
		r.Err = fmt.Errorf("Unable to read %d bytes, only read %d", size, len(b))
	}

	return Oid(binary.BigEndian.Uint32(b))
}

// ReadString reads a null terminated string
func (r *MessageReader) ReadString() (s string) {
	if r.Err != nil {
		return
	}

	var b []byte
	b, r.Err = r.buf.ReadBytes(0)
	if r.Err != nil {
		return
	}

	return string(b[:len(b)-1])
}

// ReadByteString reads count bytes and return as string
func (r *MessageReader) ReadByteString(count int32) (s string) {
	if r.Err != nil {
		return
	}

	size := int(count)
	b := r.buf.Next(size)
	if len(b) != size {
		r.Err = fmt.Errorf("Unable to read %d bytes, only read %d", size, len(b))
	}

	return string(b)
}
<|endoftext|>


<commit_before>// constants
package nlog

var pool = NewBufferPool()

// Level type
type Level uint8

// These are the different logging levels. You can set the logging level to log
// on your instance of logger, obtained with `logrus.New()`.
const (
	FatalLevel Level = iota
	// PanicLevel level, highest level of severity. Logs and then calls panic with the
	// message passed to Debug, Info, ...
	PanicLevel
	// ErrorLevel level. Logs. Used for errors that should definitely be noted.
	// Commonly used for hooks to send errors to an error tracking service.
	ErrorLevel
	// WarnLevel level. Non-critical entries that deserve eyes.
	WarnLevel
	// InfoLevel level. General operational entries about what's going on inside the
	// application.
	InfoLevel
	// DebugLevel level. Usually only enabled when debugging. Very verbose logging.
	DebugLevel

	lastIndexLevel
)

var levelString = []string{"FATA", "PANI", "ERRO", "WARN", "INFO", "DEBU"}
var levelColor = []int{31, 31, 31, 33, 34, 37}
var levelStringLower = []string{"fatal", "panic", "error", "warn", "info", "debug"}

var isTerminal bool

func init() {
	isTerminal = checkIsTerminal()
}
<commit_msg>add string to level<commit_after>// constants
package nlog

var pool = NewBufferPool()

// Level type
type Level uint8

// These are the different logging levels. You can set the logging level to log
// on your instance of logger, obtained with `logrus.New()`.
const (
	FatalLevel Level = iota
	// PanicLevel level, highest level of severity. Logs and then calls panic with the
	// message passed to Debug, Info, ...
	PanicLevel
	// ErrorLevel level. Logs. Used for errors that should definitely be noted.
	// Commonly used for hooks to send errors to an error tracking service.
	ErrorLevel
	// WarnLevel level. Non-critical entries that deserve eyes.
	WarnLevel
	// InfoLevel level. General operational entries about what's going on inside the
	// application.
	InfoLevel
	// DebugLevel level. Usually only enabled when debugging. Very verbose logging.
	DebugLevel

	lastIndexLevel
)

var levelString = []string{"FATA", "PANI", "ERRO", "WARN", "INFO", "DEBU"}
var levelColor = []int{31, 31, 31, 33, 34, 37}
var levelStringLower = []string{"fatal", "panic", "error", "warn", "info", "debug"}

var isTerminal bool

func init() {
	isTerminal = checkIsTerminal()
}

func StringToLevel(level string) Level {
	switch level {
	case "panic":
		return PanicLevel
	case "error":
		return ErrorLevel
	case "warn":
		return WarnLevel
	case "info":
		return InfoLevel
	case "debug":
		return DebugLevel
	}
	return FatalLevel
}
<|endoftext|>


<commit_before>package end2end_carbon

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"syscall"
	"testing"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/grafana/metrictank/stacktest/docker"
	"github.com/grafana/metrictank/stacktest/fakemetrics"
	"github.com/grafana/metrictank/stacktest/grafana"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stacktest/track"
)

// TODO: cleanup when ctrl-C go test (teardown all containers)

var tracker *track.Tracker
var fm *fakemetrics.FakeMetrics

const metricsPerSecond = 1000

func TestMain(m *testing.M) {
	fmt.Println("launching docker-dev stack...")
	cmd := exec.Command(docker.Path("docker/launch.sh"), "docker-dev")
	var err error

	tracker, err = track.NewTracker(cmd, false, false, "launch-stdout", "launch-stderr")
	if err != nil {
		log.Fatal(err)
	}

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	retcode := m.Run()
	fm.Close()

	fmt.Println("stopping docker-compose stack...")
	cmd.Process.Signal(syscall.SIGINT)
	// note: even when we don't care about the output, it's best to consume it before calling cmd.Wait()
	// even though the cmd.Wait docs say it will wait for stdout/stderr copying to complete
	// however the docs for cmd.StdoutPipe say "it is incorrect to call Wait before all reads from the pipe have completed"
	tracker.Wait()
	if err := cmd.Wait(); err != nil {
		// 130 means ctrl-C (interrupt) which is what we want
		if err.Error() == "exit status 130" {
			os.Exit(retcode)
		}
		log.Printf("ERROR: could not cleanly shutdown running docker-compose command: %s", err)
		retcode = 1
	}

	os.Exit(retcode)
}

func TestStartup(t *testing.T) {
	matchers := []track.Matcher{
		{Str: "metrictank.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank.*carbon-in: listening on.*2003"},
		{Str: "grafana.*Initializing HTTP Server.*:3000"},
	}
	select {
	case <-tracker.Match(matchers):
		fmt.Println("stack now running.")
		fmt.Println("Go to http://localhost:3000 (and login as admin:admin) to see what's going on")
	case <-time.After(time.Second * 70):
		grafana.PostAnnotation("TestStartup:FAIL")
		t.Fatal("timed out while waiting for all metrictank instances to come up")
	}
}

func TestBaseIngestWorkload(t *testing.T) {
	grafana.PostAnnotation("TestBaseIngestWorkload:begin")

	fm = fakemetrics.NewCarbon(metricsPerSecond)

	suc6, resp := graphite.RetryGraphite8080("perSecond(metrictank.stats.docker-env.*.input.carbon.metrics_received.counter32)", "-8s", 18, func(resp graphite.Response) bool {
		exp := []string{
			"perSecond(metrictank.stats.docker-env.default.input.carbon.metrics_received.counter32)",
		}
		a := graphite.ValidateTargets(exp)(resp)
		b := graphite.ValidatorLenNulls(1, 8)(resp)
		c := graphite.ValidatorAvgWindowed(8, graphite.Ge(metricsPerSecond))(resp)
		log.Printf("condition target names %t - condition len & nulls %t - condition avg value %t", a, b, c)
		return a && b && c
	})
	if !suc6 {
		grafana.PostAnnotation("TestBaseIngestWorkload:FAIL")
		t.Fatalf("cluster did not reach a state where the MT instance processes at least %d points per second. last response was: %s", metricsPerSecond, spew.Sdump(resp))
	}
}
<commit_msg>better logging<commit_after>package end2end_carbon

import (
	"log"
	"os"
	"os/exec"
	"syscall"
	"testing"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/grafana/metrictank/stacktest/docker"
	"github.com/grafana/metrictank/stacktest/fakemetrics"
	"github.com/grafana/metrictank/stacktest/grafana"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stacktest/track"
)

// TODO: cleanup when ctrl-C go test (teardown all containers)

var tracker *track.Tracker
var fm *fakemetrics.FakeMetrics

const metricsPerSecond = 1000

func TestMain(m *testing.M) {
	log.Println("launching docker-dev stack...")
	cmd := exec.Command(docker.Path("docker/launch.sh"), "docker-dev")
	var err error

	tracker, err = track.NewTracker(cmd, false, false, "launch-stdout", "launch-stderr")
	if err != nil {
		log.Fatal(err)
	}

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	retcode := m.Run()
	fm.Close()

	log.Println("stopping docker-compose stack...")
	cmd.Process.Signal(syscall.SIGINT)
	// note: even when we don't care about the output, it's best to consume it before calling cmd.Wait()
	// even though the cmd.Wait docs say it will wait for stdout/stderr copying to complete
	// however the docs for cmd.StdoutPipe say "it is incorrect to call Wait before all reads from the pipe have completed"
	tracker.Wait()
	err = cmd.Wait()

	// 130 means ctrl-C (interrupt) which is what we want
	if err != nil && err.Error() != "exit status 130" {
		log.Printf("ERROR: could not cleanly shutdown running docker-compose command: %s", err)
		retcode = 1
	} else {
		log.Println("docker-compose stack is shut down")
	}

	os.Exit(retcode)
}

func TestStartup(t *testing.T) {
	matchers := []track.Matcher{
		{Str: "metrictank.*metricIndex initialized.*starting data consumption$"},
		{Str: "metrictank.*carbon-in: listening on.*2003"},
		{Str: "grafana.*Initializing HTTP Server.*:3000"},
	}
	select {
	case <-tracker.Match(matchers):
		log.Println("stack now running.")
		log.Println("Go to http://localhost:3000 (and login as admin:admin) to see what's going on")
	case <-time.After(time.Second * 70):
		grafana.PostAnnotation("TestStartup:FAIL")
		t.Fatal("timed out while waiting for all metrictank instances to come up")
	}
}

func TestBaseIngestWorkload(t *testing.T) {
	grafana.PostAnnotation("TestBaseIngestWorkload:begin")

	fm = fakemetrics.NewCarbon(metricsPerSecond)

	suc6, resp := graphite.RetryGraphite8080("perSecond(metrictank.stats.docker-env.*.input.carbon.metrics_received.counter32)", "-8s", 18, func(resp graphite.Response) bool {
		exp := []string{
			"perSecond(metrictank.stats.docker-env.default.input.carbon.metrics_received.counter32)",
		}
		a := graphite.ValidateTargets(exp)(resp)
		b := graphite.ValidatorLenNulls(1, 8)(resp)
		c := graphite.ValidatorAvgWindowed(8, graphite.Ge(metricsPerSecond))(resp)
		log.Printf("condition target names %t - condition len & nulls %t - condition avg value %t", a, b, c)
		return a && b && c
	})
	if !suc6 {
		grafana.PostAnnotation("TestBaseIngestWorkload:FAIL")
		t.Fatalf("cluster did not reach a state where the MT instance processes at least %d points per second. last response was: %s", metricsPerSecond, spew.Sdump(resp))
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"database/sql"

	// Import the postgres database driver.
	_ "github.com/lib/pq"
	"github.com/matrix-org/dendrite/mediaapi/types"
	"github.com/matrix-org/gomatrixserverlib"
)

// A Database is used to store metadata about a repository of media files.
type Database struct {
	statements statements
	db         *sql.DB
}

// Open a postgres database.
func Open(dataSourceName string) (*Database, error) {
	var d Database
	var err error
	if d.db, err = sql.Open("postgres", dataSourceName); err != nil {
		return nil, err
	}
	if err = d.statements.prepare(d.db); err != nil {
		return nil, err
	}
	return &d, nil
}

// StoreMediaMetadata inserts the metadata about the uploaded media into the database.
// Returns an error if the combination of MediaID and Origin are not unique in the table.
func (d *Database) StoreMediaMetadata(mediaMetadata *types.MediaMetadata) error {
	return d.statements.insertMedia(mediaMetadata)
}

// GetMediaMetadata returns metadata about media stored on this server. The media could
// have been uploaded to this server or fetched from another server and cached here.
// Returns sql.ErrNoRows if there is no metadata associated with this media.
func (d *Database) GetMediaMetadata(mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) {
	return d.statements.selectMedia(mediaID, mediaOrigin)
}
<commit_msg>mediaapi/storage: Simplify descriptions<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"database/sql"

	// Import the postgres database driver.
	_ "github.com/lib/pq"
	"github.com/matrix-org/dendrite/mediaapi/types"
	"github.com/matrix-org/gomatrixserverlib"
)

// Database is used to store metadata about a repository of media files.
type Database struct {
	statements statements
	db         *sql.DB
}

// Open opens a postgres database.
func Open(dataSourceName string) (*Database, error) {
	var d Database
	var err error
	if d.db, err = sql.Open("postgres", dataSourceName); err != nil {
		return nil, err
	}
	if err = d.statements.prepare(d.db); err != nil {
		return nil, err
	}
	return &d, nil
}

// StoreMediaMetadata inserts the metadata about the uploaded media into the database.
// Returns an error if the combination of MediaID and Origin are not unique in the table.
func (d *Database) StoreMediaMetadata(mediaMetadata *types.MediaMetadata) error {
	return d.statements.insertMedia(mediaMetadata)
}

// GetMediaMetadata returns metadata about media stored on this server.
// The media could have been uploaded to this server or fetched from another server and cached here.
// Returns sql.ErrNoRows if there is no metadata associated with this media.
func (d *Database) GetMediaMetadata(mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) {
	return d.statements.selectMedia(mediaID, mediaOrigin)
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshcmd "github.com/cloudfoundry/bosh-utils/fileutil"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	boshsys "github.com/cloudfoundry/bosh-utils/system"
	"github.com/codegangsta/cli"
	"github.com/voelzmo/bosh-release-info/output"
	"github.com/voelzmo/bosh-release-info/release"
)

func FileListCommand(fs boshsys.FileSystem, comp boshcmd.Compressor, logger boshlog.Logger) cli.Command {
	command := cli.Command{
		Name:  "file-list",
		Usage: "lists all files in all packages in this release",
		Action: func(c *cli.Context) {
			if len(c.Args()) != 1 {
				err := bosherr.Error("that's not how it works. provide the name to the release tarball!")
				output.Fail(err, logger)
			}
			releasePath := c.Args()[0]
			relasePathSplit := strings.Split(releasePath, "/")
			releaseName := relasePathSplit[len(relasePathSplit)-1]
			fmt.Printf("Info for release: %s\n", releaseName)
			fmt.Printf("\n")

			tmpDir, err := ioutil.TempDir("", releaseName)
			if err != nil {
				output.Fail(bosherr.WrapError(err, "Failed creating temporary directory:"), logger)
			}
			defer os.RemoveAll(tmpDir)

			reader := release.NewReader(releasePath, tmpDir, fs, comp)

			manifest, err := reader.Read()
			if err != nil {
				output.Fail(bosherr.WrapError(err, "Failed reading release:"), logger)
			}
			fmt.Printf("Release name: %s\n", manifest.Name)
			fmt.Printf("\n")

			for _, pkg := range manifest.Packages {
				pkgSpecFiles, err := reader.ReadPackageSpecs(pkg.Name)
				if err != nil {
					output.Fail(bosherr.WrapError(err, "Failed reading package spec files:"), logger)
				}

				fmt.Printf("Files for package '%s': %s", pkg.Name, pkgSpecFiles)

				fmt.Printf("\n")

			}
		},
	}
	return command
}
<commit_msg>find licenses in packages<commit_after>package commands

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshcmd "github.com/cloudfoundry/bosh-utils/fileutil"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	boshsys "github.com/cloudfoundry/bosh-utils/system"
	"github.com/codegangsta/cli"
	"github.com/voelzmo/bosh-release-info/output"
	"github.com/voelzmo/bosh-release-info/release"
)

func FileListCommand(fs boshsys.FileSystem, comp boshcmd.Compressor, logger boshlog.Logger) cli.Command {
	command := cli.Command{
		Name:  "file-list",
		Usage: "lists all files in all packages in this release",
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "sort-by-type",
				Usage: "sort the files according to their filetype",
			},
		},
		Action: func(c *cli.Context) {
			if len(c.Args()) != 1 {
				err := bosherr.Error("that's not how it works. provide the name to the release tarball!")
				output.Fail(err, logger)
			}
			releasePath := c.Args()[0]
			sortByType := c.Bool("sort-by-type")

			// relasePathSplit := strings.Split(releasePath, "/")
			// releaseName := relasePathSplit[len(relasePathSplit)-1]
			releaseName := path.Base(releasePath)
			fmt.Printf("Info for release: %s\n", releaseName)
			fmt.Printf("\n")

			tmpDir, err := ioutil.TempDir("", releaseName)
			if err != nil {
				output.Fail(bosherr.WrapError(err, "Failed creating temporary directory:"), logger)
			}
			defer os.RemoveAll(tmpDir)

			reader := release.NewReader(releasePath, tmpDir, fs, comp)

			manifest, err := reader.Read()
			if err != nil {
				output.Fail(bosherr.WrapError(err, "Failed reading release:"), logger)
			}
			fmt.Printf("Release name: %s\n", manifest.Name)
			fmt.Printf("\n")

			for _, pkg := range manifest.Packages {
				pkgSpecFiles, err := reader.ReadPackageSpecs(pkg.Name)
				if err != nil {
					output.Fail(bosherr.WrapError(err, "Failed reading package spec files:"), logger)
				}

				if sortByType {
					filesInPackage := strings.Split(pkgSpecFiles, "\n")
					filesByType := make(map[string][]string)
					for _, fileWithPath := range filesInPackage {
						file := path.Base(fileWithPath)
						re, _ := regexp.Compile(".*(LICENSE|license|License)[\\.md]??")
						if re.MatchString(fileWithPath) {
							fileSplit := strings.Split(file, ".")
							fileType := fileSplit[len(fileSplit)-1]
							filesByType[fileType] = append(filesByType[fileType], fmt.Sprintf("%s:%s", manifest.Name, fileWithPath))
						} else {

						}

					}
					for typeName, files := range filesByType {
						fmt.Printf("Files for type '%s'\n: %s", typeName, strings.Join(files, "\n"))
						fmt.Printf("\n")
					}

				} else {
					fmt.Printf("Files for package '%s': %s", pkg.Name, pkgSpecFiles)
					fmt.Printf("\n")
				}
			}
		},
	}
	return command
}
<|endoftext|>


<commit_before>package cloudwatch

import (
	"encoding/json"
	"time"

	"github.com/apex/go-apex"
)

// Event represents a CloudWatch Event
type Event struct {
	ID         string          `json:"id"`
	DetailType string          `json:"detail-type"`
	Source     string          `json:"source"`
	Account    string          `json:"account"`
	Time       time.Time       `json:"time"`
	Region     string          `json:"region"`
	Resources  []string        `json:"resources"`
	Detail     json.RawMessage `json:"detail"`
}

// AutoScalingGroupDetail of the triggered event
type AutoScalingGroupDetail struct {
	ActivityID           string            `json:"ActivityId"`
	AutoScalingGroupName string            `json:"AutoScalingGroupName"`
	Cause                string            `json:"Cause"`
	Details              map[string]string `json:"Details"`
	EC2InstanceID        string            `json:"EC2InstanceId"`
	RequestID            string            `json:"RequestId"`
	StatusCode           string            `json:"StatusCode"`

	StartTime time.Time `json:"StartTime"`
	EndTime   time.Time `json:"EndTime"`
}

// EC2Detail of the triggered event
type EC2Detail struct {
	InstanceID string `json:"instance-id"`
	State      string `json:"state"`
}

// APIDetail of the triggered event
// This is useful for API or Console events
type APIDetail struct {
	EventID      string    `json:"eventID"`
	EventName    string    `json:"eventName"`
	EventSource  string    `json:"eventSource"`
	EventTime    time.Time `json:"eventTime"`
	EventType    string    `json:"eventType"`
	EventVersion string    `json:"eventVersion"`

	AWSRegion           string            `json:"awsRegion"`
	AdditionalEventData map[string]string `json:"additionalEventData,omitempty"`
	RequestParams       interface{}       `json:"requestParameters"`
	ResponseElements    map[string]string `json:"responseElements,omitempty"`
	SourceIPAddress     string            `json:"sourceIPAddress"`
	UserAgent           string            `json:"userAgent"`
	UserIdentity        UserIdentity      `json:"userIdentity,omitempty"`
}

type UserIdentity struct {
	Type           string            `json:"type,omitempty"`
	PrincipleID    string            `json:"principalId,omitempty"`
	ARN            string            `json:"arn,omitempty"`
	AccountID      string            `json:"accountId,omitempty"`
	SessionContext map[string]string `json:"sessionContext,omitempty"`
}

// Handler handles CloudWatch Events
type Handler interface {
	HandleCloudWatcEvent(*Event, *apex.Context) error
}

// HandlerFunc unmarshals CloudWatch Events before passing control.
type HandlerFunc func(*Event, *apex.Context) error

// Handle implements apex.Handler.
func (h HandlerFunc) Handle(data json.RawMessage, ctx *apex.Context) (interface{}, error) {
	var event Event

	if err := json.Unmarshal(data, &event); err != nil {
		return nil, err
	}

	return nil, h(&event, ctx)
}

// HandleFunc handles CloudWatch Events with callback function.
func HandleFunc(h HandlerFunc) {
	apex.Handle(h)
}

// Handle CloudWatch Events with handler.
func Handle(h Handler) {
	HandleFunc(HandlerFunc(h.HandleCloudWatcEvent))
}
<commit_msg>Fixed a small spelling errors<commit_after>package cloudwatch

import (
	"encoding/json"
	"time"

	"github.com/apex/go-apex"
)

// Event represents a CloudWatch Event
type Event struct {
	ID         string          `json:"id"`
	DetailType string          `json:"detail-type"`
	Source     string          `json:"source"`
	Account    string          `json:"account"`
	Time       time.Time       `json:"time"`
	Region     string          `json:"region"`
	Resources  []string        `json:"resources"`
	Detail     json.RawMessage `json:"detail"`
}

// AutoScalingGroupDetail of the triggered event
type AutoScalingGroupDetail struct {
	ActivityID           string            `json:"ActivityId"`
	AutoScalingGroupName string            `json:"AutoScalingGroupName"`
	Cause                string            `json:"Cause"`
	Details              map[string]string `json:"Details"`
	EC2InstanceID        string            `json:"EC2InstanceId"`
	RequestID            string            `json:"RequestId"`
	StatusCode           string            `json:"StatusCode"`

	StartTime time.Time `json:"StartTime"`
	EndTime   time.Time `json:"EndTime"`
}

// EC2Detail of the triggered event
type EC2Detail struct {
	InstanceID string `json:"instance-id"`
	State      string `json:"state"`
}

// APIDetail of the triggered event
// This is useful for API or Console events
type APIDetail struct {
	EventID      string    `json:"eventID"`
	EventName    string    `json:"eventName"`
	EventSource  string    `json:"eventSource"`
	EventTime    time.Time `json:"eventTime"`
	EventType    string    `json:"eventType"`
	EventVersion string    `json:"eventVersion"`

	AWSRegion           string            `json:"awsRegion"`
	AdditionalEventData map[string]string `json:"additionalEventData,omitempty"`
	RequestParams       interface{}       `json:"requestParameters"`
	ResponseElements    map[string]string `json:"responseElements,omitempty"`
	SourceIPAddress     string            `json:"sourceIPAddress"`
	UserAgent           string            `json:"userAgent"`
	UserIdentity        UserIdentity      `json:"userIdentity,omitempty"`
}

type UserIdentity struct {
	Type           string            `json:"type,omitempty"`
	PrincipleID    string            `json:"principalId,omitempty"`
	ARN            string            `json:"arn,omitempty"`
	AccountID      string            `json:"accountId,omitempty"`
	SessionContext map[string]string `json:"sessionContext,omitempty"`
}

// Handler handles CloudWatch Events
type Handler interface {
	HandleCloudWatchEvent(*Event, *apex.Context) error
}

// HandlerFunc unmarshals CloudWatch Events before passing control.
type HandlerFunc func(*Event, *apex.Context) error

// Handle implements apex.Handler.
func (h HandlerFunc) Handle(data json.RawMessage, ctx *apex.Context) (interface{}, error) {
	var event Event

	if err := json.Unmarshal(data, &event); err != nil {
		return nil, err
	}

	return nil, h(&event, ctx)
}

// HandleFunc handles CloudWatch Events with callback function.
func HandleFunc(h HandlerFunc) {
	apex.Handle(h)
}

// Handle CloudWatch Events with handler.
func Handle(h Handler) {
	HandleFunc(HandlerFunc(h.HandleCloudWatchEvent))
}
<|endoftext|>


<commit_before>package logger

import (
	"log"
	"os"
	"testing"
)

var logger *log.Logger
var filePath string
var fileName string

func init() {
	filePath = "./testLog/"
	fileName = "hello.txt"
	logger = log.New(os.Stderr,
		"Test Log :: ",
		log.Ldate|log.Ltime|log.Lshortfile)
}

// checking if file created is exist
func TestCreateLogFile(t *testing.T) {
	//checking file location
	_, err := os.Stat(filePath + fileName)
	if err != nil {
		t.Errorf("Expecting the file exist got = %v\n", err)
	}
}

func TestWriteLog(t *testing.T) {
	// set file and print some error to the error message to file.
	inputData := "data log here"

	file, err := createLogFile(filePath, fileName)
	if err != nil {
		t.Error(err)
	}
	defer file.Close()

	logger.SetOutput(file)
	logger.Println(inputData)
}
<commit_msg>delete file first if exist then test write file<commit_after>package logger

import (
	"fmt"
	"log"
	"os"
	"testing"
)

var logger *log.Logger
var filePath string
var fileName string

func init() {
	filePath = "./testLog/"
	fileName = "hello.txt"
	logger = log.New(os.Stderr,
		"Test Log :: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	// remove the file first if exist
	_, err := os.Stat(filePath + fileName)
	if err != nil {
		// file is not exist, then continue to the test
		return
	}

	os.Remove(filePath + fileName)

}

func TestWriteLog(t *testing.T) {
	// set file and print some error to the error message to file.
	inputData := "data log here"

	file, err := createLogFile(filePath, fileName)
	if err != nil {
		t.Error(err)
	}
	defer file.Close()

	logger.SetOutput(file)
	logger.Println(inputData)
}

func TestCheckfile(t *testing.T) {
	file, err := os.OpenFile(filePath+fileName, os.O_RDWR, os.ModePerm)
	if err != nil {
		panic(err)
	}

	result := make([]byte, 100)
	file.Read(result)

	fmt.Println(result)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"
	"net/http"

	"cloud.google.com/go/storage"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
)

func main() {

	ctx := context.Background()

	transCfg := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // ignore expired SSL certificates
	}
	httpClient := &http.Client{Transport: transCfg}

	// [START setup]
	client, err := storage.NewClient(ctx, option.WithEndpoint("https://0.0.0.0:4443/storage/v1/"), option.WithHTTPClient(httpClient))
	if err != nil {
		log.Fatal(err)
	}

	buckets, err := list(client, "test")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("buckets: %+v\n", buckets)
}

func list(client *storage.Client, projectID string) ([]string, error) {
	ctx := context.Background()
	// [START list_buckets]
	var buckets []string
	it := client.Buckets(ctx, projectID)
	for {
		battrs, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return nil, err
		}
		buckets = append(buckets, battrs.Name)
	}
	// [END list_buckets]
	return buckets, nil
}
<commit_msg>examples/go: cleanup<commit_after>package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"
	"net/http"

	"cloud.google.com/go/storage"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
)

func main() {
	transCfg := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // ignore expired SSL certificates
	}
	httpClient := &http.Client{Transport: transCfg}
	client, err := storage.NewClient(context.TODO(), option.WithEndpoint("https://0.0.0.0:4443/storage/v1/"), option.WithHTTPClient(httpClient))
	if err != nil {
		log.Fatal(err)
	}
	buckets, err := list(client, "test")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("buckets: %+v\n", buckets)
}

func list(client *storage.Client, projectID string) ([]string, error) {
	var buckets []string
	it := client.Buckets(context.TODO(), projectID)
	for {
		battrs, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return nil, err
		}
		buckets = append(buckets, battrs.Name)
	}
	return buckets, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"bytes"
	"fmt"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/glog"
	githubapi "github.com/google/go-github/github"
	"github.com/spf13/cobra"

	"k8s.io/contrib/mungegithub/features"
	"k8s.io/contrib/mungegithub/github"
	c "k8s.io/contrib/mungegithub/mungers/matchers/comment"
	"k8s.io/kubernetes/pkg/util/sets"
)

const (
	approvalNotificationName = "ApprovalNotifier"
	approveCommand           = "approve"
	cancel                   = "cancel"
	ownersFileName           = "OWNERS"
)

// ApprovalHandler will try to add "approved" label once
// all files of change has been approved by approvers.
type ApprovalHandler struct {
	features *features.Features
}

func init() {
	h := &ApprovalHandler{}
	RegisterMungerOrDie(h)
}

// Name is the name usable in --pr-mungers
func (*ApprovalHandler) Name() string { return "approval-handler" }

// RequiredFeatures is a slice of 'features' that must be provided
func (*ApprovalHandler) RequiredFeatures() []string {
	return []string{features.RepoFeatureName, features.AliasesFeature}
}

// Initialize will initialize the munger
func (h *ApprovalHandler) Initialize(config *github.Config, features *features.Features) error {
	h.features = features
	return nil
}

// EachLoop is called at the start of every munge loop
func (*ApprovalHandler) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (*ApprovalHandler) AddFlags(cmd *cobra.Command, config *github.Config) {}

// Munge is the workhorse the will actually make updates to the PR
// The algorithm goes as:
// - Initially, we build an approverSet
//   - Go through all comments after latest commit.
//	- If anyone said "/approve", add them to approverSet.
// - Then, for each file, we see if any approver of this file is in approverSet and keep track of files without approval
//   - An approver of a file is defined as:
//     - Someone listed as an "approver" in an OWNERS file in the files directory OR
//     - in one of the file's parent directorie
// - Iff all files have been approved, the bot will add the "approved" label.
// - Iff a cancel command is found, that reviewer will be removed from the approverSet
// 	and the munger will remove the approved label if it has been applied
func (h *ApprovalHandler) Munge(obj *github.MungeObject) {
	if !obj.IsPR() {
		return
	}
	files, ok := obj.ListFiles()
	if !ok {
		return
	}

	comments, ok := getCommentsAfterLastModified(obj)

	if !ok {
		return
	}

	ownersMap := h.getApprovedOwners(files, createApproverSet(comments))

	if err := h.updateNotification(obj, ownersMap); err != nil {
		return
	}

	for _, approverSet := range ownersMap {
		if approverSet.Len() == 0 {
			if obj.HasLabel(approvedLabel) {
				obj.RemoveLabel(approvedLabel)
			}
			return
		}
	}

	if !obj.HasLabel(approvedLabel) {
		obj.AddLabel(approvedLabel)
	}
}

func (h *ApprovalHandler) updateNotification(obj *github.MungeObject, ownersMap map[string]sets.String) error {
	notificationMatcher := c.MungerNotificationName(approvalNotificationName)
	comments, ok := obj.ListComments()
	if !ok {
		return fmt.Errorf("Unable to ListComments for %d", obj.Number())
	}

	notifications := c.FilterComments(comments, notificationMatcher)
	latestNotification := notifications.GetLast()
	if latestNotification == nil {
		body := h.getMessage(obj, ownersMap)
		obj.WriteComment(body)
	}

	latestApprove := c.FilterComments(comments, c.CommandName(approveCommand)).GetLast()
	if latestApprove == nil || latestApprove.CreatedAt == nil {
		// there was already a bot notification and nothing has changed since
		// or we wouldn't tell when the latestApproval occurred
		return nil
	}
	if latestApprove.CreatedAt.After(*latestNotification.CreatedAt) {
		// if someone approved since the last comment, we should update the comment
		glog.Infof("Latest approve was after last time notified")
		body := h.getMessage(obj, ownersMap)
		return obj.EditComment(latestApprove, body)
	}
	lastModified, ok := obj.LastModifiedTime()
	if !ok {
		return fmt.Errorf("Unable to get LastModifiedTime for %d", obj.Number())
	}
	if latestNotification.CreatedAt.Before(*lastModified) {
		// the PR was modified After our last notification, so we should update the approvers notification
		// i.e. People that have formerly approved haven't necessarily approved of new changes
		glog.Infof("PR Modified After Last Notification")
		body := h.getMessage(obj, ownersMap)
		return obj.EditComment(latestApprove, body)
	}
	return nil
}

// findPeopleToApprove Takes the Owners Files that Are Needed for the PR and chooses a good
// subset of Approvers that are guaranteed to cover all of them (exact cover)
// This is a greedy approximation and not guaranteed to find the minimum number of OWNERS
func (h ApprovalHandler) findPeopleToApprove(ownersPaths sets.String, prAuthor string) sets.String {

	// approverCount contains a map: person -> set of relevant OWNERS file they are in
	approverCount := make(map[string]sets.String)
	for ownersFile := range ownersPaths {
		// LeafApprovers removes the last part of a path for dirs and files, so we append owners to the path
		for approver := range h.features.Repos.LeafApprovers(filepath.Join(ownersFile, ownersFileName)) {
			if approver == prAuthor {
				// don't add the author of the PR to the list of candidates that can approve
				continue
			}
			if _, ok := approverCount[approver]; ok {
				approverCount[approver].Insert(ownersFile)
			} else {
				approverCount[approver] = sets.NewString(ownersFile)
			}
		}
	}

	copyOfFiles := sets.NewString()
	for fn := range ownersPaths {
		copyOfFiles.Insert(fn)
	}

	approverGroup := sets.NewString()
	var bestPerson string
	for copyOfFiles.Len() > 0 {
		maxCovered := 0
		for k, v := range approverCount {
			if v.Intersection(copyOfFiles).Len() > maxCovered {
				maxCovered = len(v)
				bestPerson = k
			}
		}

		approverGroup.Insert(bestPerson)
		toDelete := sets.NewString()
		// remove all files in the directories that our approver approved AND
		// in the subdirectories that s/he approved.  HasPrefix finds subdirs
		for fn := range copyOfFiles {
			for approvedFile := range approverCount[bestPerson] {
				if strings.HasPrefix(fn, approvedFile) {
					toDelete.Insert(fn)
				}

			}
		}
		copyOfFiles.Delete(toDelete.List()...)
	}
	return approverGroup
}

// removeSubdirs takes a list of directories as an input and returns a set of directories with all
// subdirectories removed.  E.g. [/a,/a/b/c,/d/e,/d/e/f] -> [/a, /d/e]
func removeSubdirs(dirList []string) sets.String {
	toDel := sets.String{}
	for i := 0; i < len(dirList)-1; i++ {
		for j := i + 1; j < len(dirList); j++ {
			// ex /a/b has prefix /a so if remove /a/b since its already covered
			if strings.HasPrefix(dirList[i], dirList[j]) {
				toDel.Insert(dirList[i])
			} else if strings.HasPrefix(dirList[j], dirList[i]) {
				toDel.Insert(dirList[j])
			}
		}
	}
	finalSet := sets.NewString(dirList...)
	finalSet.Delete(toDel.List()...)
	return finalSet
}

// getMessage returns the comment body that we want the approval-handler to display on PRs
// The comment shows:
// 	- a list of approvers files (and links) needed to get the PR approved
// 	- a list of approvers files with strikethroughs that already have an approver's approval
// 	- a suggested list of people from each OWNERS files that can fully approve the PR
// 	- how an approver can indicate their approval
// 	- how an approver can cancel their approval
func (h *ApprovalHandler) getMessage(obj *github.MungeObject, ownersMap map[string]sets.String) string {
	// sort the keys so we always display OWNERS files in same order
	sliceOfKeys := make([]string, len(ownersMap))
	i := 0
	for path := range ownersMap {
		sliceOfKeys[i] = path
		i++
	}
	sort.Strings(sliceOfKeys)

	unapprovedOwners := sets.NewString()
	context := bytes.NewBufferString("")
	for _, path := range sliceOfKeys {
		approverSet := ownersMap[path]
		if approverSet.Len() == 0 {
			fullOwnersPath := filepath.Join(path, ownersFileName)
			link := fmt.Sprintf("https://github.com/%s/%s/blob/master/%v", obj.Org(), obj.Project(), fullOwnersPath)
			context.WriteString(fmt.Sprintf("- **[%s](%s)** \n", fullOwnersPath, link))
			unapprovedOwners.Insert(path)
		} else {
			context.WriteString(fmt.Sprintf("- ~~%s~~ [%v]\n", path, strings.Join(approverSet.List(), ",")))
		}
	}
	context.WriteString("\n")
	if unapprovedOwners.Len() > 0 {
		context.WriteString("We suggest the following people:\n")
		context.WriteString("cc ")
		toBeAssigned := h.findPeopleToApprove(unapprovedOwners, *obj.Issue.User.Login)
		for person := range toBeAssigned {
			context.WriteString("@" + person + " ")
		}
	}
	context.WriteString("\n You can indicate your approval by writing `/approve` in a comment")
	context.WriteString("\n You can cancel your approval by writing `/approve cancel` in a comment")
	notif := c.Notification{approvalNotificationName, "Needs approval from an approver in each of these OWNERS Files:\n", context.String()}
	return notif.String()
}

// createApproverSet iterates through the list of comments on a PR
// and identifies all of the people that have said /approve and adds
// them to the approverSet.  The function uses the latest approve or cancel comment
// to determine the Users intention
func createApproverSet(comments []*githubapi.IssueComment) sets.String {
	approverSet := sets.NewString()

	approverMatcher := c.CommandName(approveCommand)
	for _, comment := range c.FilterComments(comments, approverMatcher) {
		cmd := c.ParseCommand(comment)
		if cmd.Arguments == cancel {
			approverSet.Delete(*comment.User.Login)
		} else {
			approverSet.Insert(*comment.User.Login)
		}
	}
	return approverSet
}

// getApprovedOwners finds all the relevant OWNERS files for the PRs and identifies all the people from them
// that have approved the PR.  For all files that have not been approved, it finds the minimum number of owners files
// that cover all of them.  E.g. If /a/b/c.txt and /a/d.txt need approval, it will only indicate that an approval from
// someone in /a/OWNERS is needed
func (h ApprovalHandler) getApprovedOwners(files []*githubapi.CommitFile, approverSet sets.String) map[string]sets.String {
	ownersApprovers := make(map[string]sets.String)
	// TODO: go through the files starting at the top of the tree
	needsApproval := sets.NewString()
	for _, file := range files {
		fileOwners := h.features.Repos.Approvers(*file.Filename)
		ownersFile := h.features.Repos.FindOwnersForPath(*file.Filename)
		hasApproved := fileOwners.Intersection(approverSet)
		if len(hasApproved) != 0 {
			ownersApprovers[ownersFile] = hasApproved
		} else {
			needsApproval.Insert(ownersFile)
		}

	}
	needsApproval = removeSubdirs(needsApproval.List())
	for fn := range needsApproval {
		ownersApprovers[fn] = sets.NewString()
	}
	return ownersApprovers
}

// gets the comments since the obj was last changed.  If we can't figure out when the object was last changed
// return all the comments on the issue
func getCommentsAfterLastModified(obj *github.MungeObject) ([]*githubapi.IssueComment, bool) {
	comments, ok := obj.ListComments()
	if !ok {
		return comments, ok
	}
	lastModified, ok := obj.LastModifiedTime()
	if !ok {
		return comments, ok
	}
	return c.FilterComments(comments, c.CreatedAfter(*lastModified)), true
}
<commit_msg>mungegithub: Write consistent approval OWNERS path<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"bytes"
	"fmt"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/glog"
	githubapi "github.com/google/go-github/github"
	"github.com/spf13/cobra"

	"k8s.io/contrib/mungegithub/features"
	"k8s.io/contrib/mungegithub/github"
	c "k8s.io/contrib/mungegithub/mungers/matchers/comment"
	"k8s.io/kubernetes/pkg/util/sets"
)

const (
	approvalNotificationName = "ApprovalNotifier"
	approveCommand           = "approve"
	cancel                   = "cancel"
	ownersFileName           = "OWNERS"
)

// ApprovalHandler will try to add "approved" label once
// all files of change has been approved by approvers.
type ApprovalHandler struct {
	features *features.Features
}

func init() {
	h := &ApprovalHandler{}
	RegisterMungerOrDie(h)
}

// Name is the name usable in --pr-mungers
func (*ApprovalHandler) Name() string { return "approval-handler" }

// RequiredFeatures is a slice of 'features' that must be provided
func (*ApprovalHandler) RequiredFeatures() []string {
	return []string{features.RepoFeatureName, features.AliasesFeature}
}

// Initialize will initialize the munger
func (h *ApprovalHandler) Initialize(config *github.Config, features *features.Features) error {
	h.features = features
	return nil
}

// EachLoop is called at the start of every munge loop
func (*ApprovalHandler) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (*ApprovalHandler) AddFlags(cmd *cobra.Command, config *github.Config) {}

// Munge is the workhorse the will actually make updates to the PR
// The algorithm goes as:
// - Initially, we build an approverSet
//   - Go through all comments after latest commit.
//	- If anyone said "/approve", add them to approverSet.
// - Then, for each file, we see if any approver of this file is in approverSet and keep track of files without approval
//   - An approver of a file is defined as:
//     - Someone listed as an "approver" in an OWNERS file in the files directory OR
//     - in one of the file's parent directorie
// - Iff all files have been approved, the bot will add the "approved" label.
// - Iff a cancel command is found, that reviewer will be removed from the approverSet
// 	and the munger will remove the approved label if it has been applied
func (h *ApprovalHandler) Munge(obj *github.MungeObject) {
	if !obj.IsPR() {
		return
	}
	files, ok := obj.ListFiles()
	if !ok {
		return
	}

	comments, ok := getCommentsAfterLastModified(obj)

	if !ok {
		return
	}

	ownersMap := h.getApprovedOwners(files, createApproverSet(comments))

	if err := h.updateNotification(obj, ownersMap); err != nil {
		return
	}

	for _, approverSet := range ownersMap {
		if approverSet.Len() == 0 {
			if obj.HasLabel(approvedLabel) {
				obj.RemoveLabel(approvedLabel)
			}
			return
		}
	}

	if !obj.HasLabel(approvedLabel) {
		obj.AddLabel(approvedLabel)
	}
}

func (h *ApprovalHandler) updateNotification(obj *github.MungeObject, ownersMap map[string]sets.String) error {
	notificationMatcher := c.MungerNotificationName(approvalNotificationName)
	comments, ok := obj.ListComments()
	if !ok {
		return fmt.Errorf("Unable to ListComments for %d", obj.Number())
	}

	notifications := c.FilterComments(comments, notificationMatcher)
	latestNotification := notifications.GetLast()
	if latestNotification == nil {
		body := h.getMessage(obj, ownersMap)
		obj.WriteComment(body)
	}

	latestApprove := c.FilterComments(comments, c.CommandName(approveCommand)).GetLast()
	if latestApprove == nil || latestApprove.CreatedAt == nil {
		// there was already a bot notification and nothing has changed since
		// or we wouldn't tell when the latestApproval occurred
		return nil
	}
	if latestApprove.CreatedAt.After(*latestNotification.CreatedAt) {
		// if someone approved since the last comment, we should update the comment
		glog.Infof("Latest approve was after last time notified")
		body := h.getMessage(obj, ownersMap)
		return obj.EditComment(latestApprove, body)
	}
	lastModified, ok := obj.LastModifiedTime()
	if !ok {
		return fmt.Errorf("Unable to get LastModifiedTime for %d", obj.Number())
	}
	if latestNotification.CreatedAt.Before(*lastModified) {
		// the PR was modified After our last notification, so we should update the approvers notification
		// i.e. People that have formerly approved haven't necessarily approved of new changes
		glog.Infof("PR Modified After Last Notification")
		body := h.getMessage(obj, ownersMap)
		return obj.EditComment(latestApprove, body)
	}
	return nil
}

// findPeopleToApprove Takes the Owners Files that Are Needed for the PR and chooses a good
// subset of Approvers that are guaranteed to cover all of them (exact cover)
// This is a greedy approximation and not guaranteed to find the minimum number of OWNERS
func (h ApprovalHandler) findPeopleToApprove(ownersPaths sets.String, prAuthor string) sets.String {

	// approverCount contains a map: person -> set of relevant OWNERS file they are in
	approverCount := make(map[string]sets.String)
	for ownersFile := range ownersPaths {
		// LeafApprovers removes the last part of a path for dirs and files, so we append owners to the path
		for approver := range h.features.Repos.LeafApprovers(filepath.Join(ownersFile, ownersFileName)) {
			if approver == prAuthor {
				// don't add the author of the PR to the list of candidates that can approve
				continue
			}
			if _, ok := approverCount[approver]; ok {
				approverCount[approver].Insert(ownersFile)
			} else {
				approverCount[approver] = sets.NewString(ownersFile)
			}
		}
	}

	copyOfFiles := sets.NewString()
	for fn := range ownersPaths {
		copyOfFiles.Insert(fn)
	}

	approverGroup := sets.NewString()
	var bestPerson string
	for copyOfFiles.Len() > 0 {
		maxCovered := 0
		for k, v := range approverCount {
			if v.Intersection(copyOfFiles).Len() > maxCovered {
				maxCovered = len(v)
				bestPerson = k
			}
		}

		approverGroup.Insert(bestPerson)
		toDelete := sets.NewString()
		// remove all files in the directories that our approver approved AND
		// in the subdirectories that s/he approved.  HasPrefix finds subdirs
		for fn := range copyOfFiles {
			for approvedFile := range approverCount[bestPerson] {
				if strings.HasPrefix(fn, approvedFile) {
					toDelete.Insert(fn)
				}

			}
		}
		copyOfFiles.Delete(toDelete.List()...)
	}
	return approverGroup
}

// removeSubdirs takes a list of directories as an input and returns a set of directories with all
// subdirectories removed.  E.g. [/a,/a/b/c,/d/e,/d/e/f] -> [/a, /d/e]
func removeSubdirs(dirList []string) sets.String {
	toDel := sets.String{}
	for i := 0; i < len(dirList)-1; i++ {
		for j := i + 1; j < len(dirList); j++ {
			// ex /a/b has prefix /a so if remove /a/b since its already covered
			if strings.HasPrefix(dirList[i], dirList[j]) {
				toDel.Insert(dirList[i])
			} else if strings.HasPrefix(dirList[j], dirList[i]) {
				toDel.Insert(dirList[j])
			}
		}
	}
	finalSet := sets.NewString(dirList...)
	finalSet.Delete(toDel.List()...)
	return finalSet
}

// getMessage returns the comment body that we want the approval-handler to display on PRs
// The comment shows:
// 	- a list of approvers files (and links) needed to get the PR approved
// 	- a list of approvers files with strikethroughs that already have an approver's approval
// 	- a suggested list of people from each OWNERS files that can fully approve the PR
// 	- how an approver can indicate their approval
// 	- how an approver can cancel their approval
func (h *ApprovalHandler) getMessage(obj *github.MungeObject, ownersMap map[string]sets.String) string {
	// sort the keys so we always display OWNERS files in same order
	sliceOfKeys := make([]string, len(ownersMap))
	i := 0
	for path := range ownersMap {
		sliceOfKeys[i] = path
		i++
	}
	sort.Strings(sliceOfKeys)

	unapprovedOwners := sets.NewString()
	context := bytes.NewBufferString("")
	for _, path := range sliceOfKeys {
		approverSet := ownersMap[path]
		fullOwnersPath := filepath.Join(path, ownersFileName)
		link := fmt.Sprintf("https://github.com/%s/%s/blob/master/%v", obj.Org(), obj.Project(), fullOwnersPath)

		if approverSet.Len() == 0 {
			context.WriteString(fmt.Sprintf("- **[%s](%s)** \n", fullOwnersPath, link))
			unapprovedOwners.Insert(path)
		} else {
			context.WriteString(fmt.Sprintf("- ~~[%s](%s)~~ [%v]\n", fullOwnersPath, link, strings.Join(approverSet.List(), ",")))
		}
	}
	context.WriteString("\n")
	if unapprovedOwners.Len() > 0 {
		context.WriteString("We suggest the following people:\n")
		context.WriteString("cc ")
		toBeAssigned := h.findPeopleToApprove(unapprovedOwners, *obj.Issue.User.Login)
		for person := range toBeAssigned {
			context.WriteString("@" + person + " ")
		}
	}
	context.WriteString("\n You can indicate your approval by writing `/approve` in a comment")
	context.WriteString("\n You can cancel your approval by writing `/approve cancel` in a comment")
	notif := c.Notification{approvalNotificationName, "Needs approval from an approver in each of these OWNERS Files:\n", context.String()}
	return notif.String()
}

// createApproverSet iterates through the list of comments on a PR
// and identifies all of the people that have said /approve and adds
// them to the approverSet.  The function uses the latest approve or cancel comment
// to determine the Users intention
func createApproverSet(comments []*githubapi.IssueComment) sets.String {
	approverSet := sets.NewString()

	approverMatcher := c.CommandName(approveCommand)
	for _, comment := range c.FilterComments(comments, approverMatcher) {
		cmd := c.ParseCommand(comment)
		if cmd.Arguments == cancel {
			approverSet.Delete(*comment.User.Login)
		} else {
			approverSet.Insert(*comment.User.Login)
		}
	}
	return approverSet
}

// getApprovedOwners finds all the relevant OWNERS files for the PRs and identifies all the people from them
// that have approved the PR.  For all files that have not been approved, it finds the minimum number of owners files
// that cover all of them.  E.g. If /a/b/c.txt and /a/d.txt need approval, it will only indicate that an approval from
// someone in /a/OWNERS is needed
func (h ApprovalHandler) getApprovedOwners(files []*githubapi.CommitFile, approverSet sets.String) map[string]sets.String {
	ownersApprovers := make(map[string]sets.String)
	// TODO: go through the files starting at the top of the tree
	needsApproval := sets.NewString()
	for _, file := range files {
		fileOwners := h.features.Repos.Approvers(*file.Filename)
		ownersFile := h.features.Repos.FindOwnersForPath(*file.Filename)
		hasApproved := fileOwners.Intersection(approverSet)
		if len(hasApproved) != 0 {
			ownersApprovers[ownersFile] = hasApproved
		} else {
			needsApproval.Insert(ownersFile)
		}

	}
	needsApproval = removeSubdirs(needsApproval.List())
	for fn := range needsApproval {
		ownersApprovers[fn] = sets.NewString()
	}
	return ownersApprovers
}

// gets the comments since the obj was last changed.  If we can't figure out when the object was last changed
// return all the comments on the issue
func getCommentsAfterLastModified(obj *github.MungeObject) ([]*githubapi.IssueComment, bool) {
	comments, ok := obj.ListComments()
	if !ok {
		return comments, ok
	}
	lastModified, ok := obj.LastModifiedTime()
	if !ok {
		return comments, ok
	}
	return c.FilterComments(comments, c.CreatedAfter(*lastModified)), true
}
<|endoftext|>