<commit_before>// Package grace allows for gracefully waiting for a listener to
// finish serving it's active requests.
package grace

import (
	"errors"
	"fmt"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

var (
	// This error is returned by Inherits() when we're not inheriting any fds.
	ErrNotInheriting = errors.New("no inherited listeners")

	// This error is returned by Listener.Accept() when Close is in progress.
	ErrAlreadyClosed = errors.New("already closed")

	// Time in the past to trigger immediate deadline.
	timeInPast = time.Date(1983, time.November, 6, 0, 0, 0, 0, time.UTC)
)

const (
	// Used to indicate a graceful restart in the new process.
	envCountKey = "LISTEN_FDS"

	// The error returned by the standard library when the socket is closed.
	errClosed = "use of closed network connection"
)

// A Listener providing a graceful Close process and can be sent
// across processes using the underlying File descriptor.
type Listener interface {
	net.Listener

	// Will return the underlying file representing this Listener.
	File() (f *os.File, err error)
}

type listener struct {
	Listener
	closed      bool
	closedMutex sync.RWMutex
	wg          sync.WaitGroup
}

// Allows for us to notice when the connection is closed.
type conn struct {
	net.Conn
	wg *sync.WaitGroup
}

func (c conn) Close() error {
	defer c.wg.Done()
	return c.Conn.Close()
}

// Wraps an existing File listener to provide a graceful Close() process.
func NewListener(l Listener) Listener {
	return &listener{Listener: l}
}

func (l *listener) Close() error {
	l.closedMutex.Lock()
	l.closed = true
	l.closedMutex.Unlock()

	var err error
	// Init provided sockets dont actually close so we trigger Accept to return
	// by setting the deadline.
	if os.Getppid() == 1 {
		if ld, ok := l.Listener.(interface {
			SetDeadline(t time.Time) error
		}); ok {
			ld.SetDeadline(timeInPast)
		} else {
			fmt.Fprintln(os.Stderr, "init activated server did not have SetDeadline")
		}
	} else {
		err = l.Listener.Close()
	}
	l.wg.Wait()
	return err
}

func (l *listener) Accept() (c net.Conn, err error) {
	// Presume we'll accept and decrement in defer if we don't. If we did this
	// after a successful accept we would have a race condition where we may end
	// up incorrectly shutting down between the time we do a successful accept
	// and the increment.
	l.wg.Add(1)
	defer func() {
		// If we didn't accept, we decrement our presumptuous count above.
		if c == nil {
			l.wg.Done()
		}
	}()

	l.closedMutex.RLock()
	if l.closed {
		l.closedMutex.RUnlock()
		return nil, ErrAlreadyClosed
	}
	l.closedMutex.RUnlock()

	c, err = l.Listener.Accept()
	if err != nil {
		if strings.HasSuffix(err.Error(), errClosed) {
			return nil, ErrAlreadyClosed
		}

		// We use SetDeadline above to trigger Accept to return when we're trying
		// to handoff to a child as part of our restart process. In this scenario
		// we want to treat the timeout the same as a Close.
		if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
			l.closedMutex.RLock()
			if l.closed {
				l.closedMutex.RUnlock()
				return nil, ErrAlreadyClosed
			}
			l.closedMutex.RUnlock()
		}
		return nil, err
	}
	return conn{Conn: c, wg: &l.wg}, nil
}

// Wait for signals to gracefully terminate or restart the process.
func Wait(listeners []Listener) (err error) {
	ch := make(chan os.Signal, 2)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGUSR2)
	for {
		sig := <-ch
		switch sig {
		case syscall.SIGTERM:
			var wg sync.WaitGroup
			wg.Add(len(listeners))
			for _, l := range listeners {
				go func(l Listener) {
					cErr := l.Close()
					if cErr != nil {
						err = cErr
					}
					wg.Done()
				}(l)
			}
			wg.Wait()
			return
		case syscall.SIGUSR2:
			rErr := Restart(listeners)
			if rErr != nil {
				return rErr
			}
		}
	}
}

// Try to inherit listeners from the parent process.
func Inherit() (listeners []Listener, err error) {
	countStr := os.Getenv(envCountKey)
	if countStr == "" {
		return nil, ErrNotInheriting
	}
	count, err := strconv.Atoi(countStr)
	if err != nil {
		return nil, err
	}
	// If we are inheriting, the listeners will begin at fd 3
	for i := 3; i < 3+count; i++ {
		file := os.NewFile(uintptr(i), "listener")
		tmp, err := net.FileListener(file)
		file.Close()
		if err != nil {
			return nil, err
		}
		l := tmp.(*net.TCPListener)
		listeners = append(listeners, NewListener(l))
	}
	return
}

// Start the Close process in the parent. This does not wait for the
// parent to close and simply sends it the TERM signal.
func CloseParent() error {
	ppid := os.Getppid()
	if ppid == 1 { // init provided sockets, for example systemd
		return nil
	}
	return syscall.Kill(ppid, syscall.SIGTERM)
}

// Restart the process passing the given listeners to the new process.
func Restart(listeners []Listener) (err error) {
	if len(listeners) == 0 {
		return errors.New("restart must be given listeners.")
	}
	files := make([]*os.File, len(listeners))
	for i, l := range listeners {
		files[i], err = l.File()
		if err != nil {
			return err
		}
		defer files[i].Close()
		syscall.CloseOnExec(int(files[i].Fd()))
	}
	argv0, err := exec.LookPath(os.Args[0])
	if err != nil {
		return err
	}
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	allFiles := append([]*os.File{os.Stdin, os.Stdout, os.Stderr}, files...)
	allFiles = append(allFiles, nil)
	_, err = os.StartProcess(argv0, os.Args, &os.ProcAttr{
		Dir:   wd,
		Env:   append(os.Environ(), fmt.Sprintf("%s=%d", envCountKey, len(files))),
		Files: allFiles,
	})
	return err
}
<commit_msg>move inline interface out for readability<commit_after>// Package grace allows for gracefully waiting for a listener to
// finish serving it's active requests.
package grace

import (
	"errors"
	"fmt"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

var (
	// This error is returned by Inherits() when we're not inheriting any fds.
	ErrNotInheriting = errors.New("no inherited listeners")

	// This error is returned by Listener.Accept() when Close is in progress.
	ErrAlreadyClosed = errors.New("already closed")

	// Time in the past to trigger immediate deadline.
	timeInPast = time.Date(1983, time.November, 6, 0, 0, 0, 0, time.UTC)
)

const (
	// Used to indicate a graceful restart in the new process.
	envCountKey = "LISTEN_FDS"

	// The error returned by the standard library when the socket is closed.
	errClosed = "use of closed network connection"
)

// A Listener providing a graceful Close process and can be sent
// across processes using the underlying File descriptor.
type Listener interface {
	net.Listener

	// Will return the underlying file representing this Listener.
	File() (f *os.File, err error)
}

type listener struct {
	Listener
	closed      bool
	closedMutex sync.RWMutex
	wg          sync.WaitGroup
}

type deadliner interface {
	SetDeadline(t time.Time) error
}

// Allows for us to notice when the connection is closed.
type conn struct {
	net.Conn
	wg *sync.WaitGroup
}

func (c conn) Close() error {
	defer c.wg.Done()
	return c.Conn.Close()
}

// Wraps an existing File listener to provide a graceful Close() process.
func NewListener(l Listener) Listener {
	return &listener{Listener: l}
}

func (l *listener) Close() error {
	l.closedMutex.Lock()
	l.closed = true
	l.closedMutex.Unlock()

	var err error
	// Init provided sockets dont actually close so we trigger Accept to return
	// by setting the deadline.
	if os.Getppid() == 1 {
		if ld, ok := l.Listener.(deadliner); ok {
			ld.SetDeadline(timeInPast)
		} else {
			fmt.Fprintln(os.Stderr, "init activated server did not have SetDeadline")
		}
	} else {
		err = l.Listener.Close()
	}
	l.wg.Wait()
	return err
}

func (l *listener) Accept() (c net.Conn, err error) {
	// Presume we'll accept and decrement in defer if we don't. If we did this
	// after a successful accept we would have a race condition where we may end
	// up incorrectly shutting down between the time we do a successful accept
	// and the increment.
	l.wg.Add(1)
	defer func() {
		// If we didn't accept, we decrement our presumptuous count above.
		if c == nil {
			l.wg.Done()
		}
	}()

	l.closedMutex.RLock()
	if l.closed {
		l.closedMutex.RUnlock()
		return nil, ErrAlreadyClosed
	}
	l.closedMutex.RUnlock()

	c, err = l.Listener.Accept()
	if err != nil {
		if strings.HasSuffix(err.Error(), errClosed) {
			return nil, ErrAlreadyClosed
		}

		// We use SetDeadline above to trigger Accept to return when we're trying
		// to handoff to a child as part of our restart process. In this scenario
		// we want to treat the timeout the same as a Close.
		if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
			l.closedMutex.RLock()
			if l.closed {
				l.closedMutex.RUnlock()
				return nil, ErrAlreadyClosed
			}
			l.closedMutex.RUnlock()
		}
		return nil, err
	}
	return conn{Conn: c, wg: &l.wg}, nil
}

// Wait for signals to gracefully terminate or restart the process.
func Wait(listeners []Listener) (err error) {
	ch := make(chan os.Signal, 2)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGUSR2)
	for {
		sig := <-ch
		switch sig {
		case syscall.SIGTERM:
			var wg sync.WaitGroup
			wg.Add(len(listeners))
			for _, l := range listeners {
				go func(l Listener) {
					cErr := l.Close()
					if cErr != nil {
						err = cErr
					}
					wg.Done()
				}(l)
			}
			wg.Wait()
			return
		case syscall.SIGUSR2:
			rErr := Restart(listeners)
			if rErr != nil {
				return rErr
			}
		}
	}
}

// Try to inherit listeners from the parent process.
func Inherit() (listeners []Listener, err error) {
	countStr := os.Getenv(envCountKey)
	if countStr == "" {
		return nil, ErrNotInheriting
	}
	count, err := strconv.Atoi(countStr)
	if err != nil {
		return nil, err
	}
	// If we are inheriting, the listeners will begin at fd 3
	for i := 3; i < 3+count; i++ {
		file := os.NewFile(uintptr(i), "listener")
		tmp, err := net.FileListener(file)
		file.Close()
		if err != nil {
			return nil, err
		}
		l := tmp.(*net.TCPListener)
		listeners = append(listeners, NewListener(l))
	}
	return
}

// Start the Close process in the parent. This does not wait for the
// parent to close and simply sends it the TERM signal.
func CloseParent() error {
	ppid := os.Getppid()
	if ppid == 1 { // init provided sockets, for example systemd
		return nil
	}
	return syscall.Kill(ppid, syscall.SIGTERM)
}

// Restart the process passing the given listeners to the new process.
func Restart(listeners []Listener) (err error) {
	if len(listeners) == 0 {
		return errors.New("restart must be given listeners.")
	}
	files := make([]*os.File, len(listeners))
	for i, l := range listeners {
		files[i], err = l.File()
		if err != nil {
			return err
		}
		defer files[i].Close()
		syscall.CloseOnExec(int(files[i].Fd()))
	}
	argv0, err := exec.LookPath(os.Args[0])
	if err != nil {
		return err
	}
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	allFiles := append([]*os.File{os.Stdin, os.Stdout, os.Stderr}, files...)
	allFiles = append(allFiles, nil)
	_, err = os.StartProcess(argv0, os.Args, &os.ProcAttr{
		Dir:   wd,
		Env:   append(os.Environ(), fmt.Sprintf("%s=%d", envCountKey, len(files))),
		Files: allFiles,
	})
	return err
}
<|endoftext|>


<commit_before>package store4

import (
	"bytes"
	"sort"
)

// TripleCallbackFn is the function signature used to implement
// callback functions that receive a triple.
//
// Used with calls to Graph's ForEach and ForEachWith.
type TripleCallbackFn func(s, p, o string)

// TripleTestFn is the function signature used to implement
// callback functions performing triple tests.
// A response of true means that the test has been passed.
//
// Used with calls to Graph's Every, EveryWith, Some and SomeWith.
type TripleTestFn func(s, p, o string) bool

// Graph is a convenience façade that simply
// proxies calls to its associated QuadStore.
type Graph struct {
	Name      string
	QuadStore *QuadStore
}

// Graph returns a proxy-façade that provides a triple-based API for working with graphs within the store.
func (s *QuadStore) Graph(name string) *Graph {
	return &Graph{
		Name:      name,
		QuadStore: s,
	}
}

// NewGraph returns an unnamed graph with a
// newly created QuadStore as its backing.
func NewGraph() *Graph {
	return NewQuadStore().Graph("")
}

func adaptTripleCallbackFn(fn TripleCallbackFn) QuadCallbackFn {
	return func(s, p, o, g string) {
		fn(s, p, o)
	}
}

func adaptTripleTestFn(fn TripleTestFn) QuadTestFn {
	return func(s, p, o, g string) bool {
		return fn(s, p, o)
	}
}

// Add a triple to the graph.
// Returns true if the triple was a new triple,
// or false if the triple already existed.
//
// If any of the given terms are "*" (an asterisk),
// then this method will panic. (The asterisk is reserved
// for wildcard operations throughout the API).
func (g *Graph) Add(subject, predicate, object string) bool {
	return g.QuadStore.Add(subject, predicate, object, g.Name)
}

// Count returns a count of triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) Count(subject, predicate, object string) uint64 {
	return g.QuadStore.Count(subject, predicate, object, g.Name)
}

// Every tests whether all triples in the graph pass the test
// implemented by the given function.
//
// The given callback is
// executed once for each triple present in the graph until
// Every finds one where the callback returns false. If such
// an element is found, iteration is immediately halted and
// Every returns false. Otherwise, if the callback returns
// true for all triples, then Every returns true.
//
// Acting like the 'for all' quantifier in maths, it should
// be noted that Every returns true for an empty graph.
func (g *Graph) Every(fn TripleTestFn) bool {
	return g.QuadStore.EveryWith("*", "*", "*", g.Name, adaptTripleTestFn(fn))
}

// EveryWith tests whether all triples in the graph that match the
// given terms pass the test implemented by the given function.
//
// The given callback is
// executed once for each matching triple in the graph until
// EveryWith finds one where the callback returns false. If such
// an element is found, iteration is immediately halted and
// EveryWith returns false. Otherwise, if the callback returns
// true for all triples, then EveryWith returns true.
//
// Acting like the 'for all' quantifier in maths, it should
// be noted that EveryWith returns true for an empty graph.
// By extension, if the given parameters cause the iteration
// set to be empty, then EveryWith also returns true.
func (g *Graph) EveryWith(subject, predicate, object string, fn TripleTestFn) bool {
	return g.QuadStore.EveryWith(subject, predicate, object, g.Name, adaptTripleTestFn(fn))
}

// FindObjects returns a list of distinct object terms for all triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) FindObjects(subject, predicate string) []string {
	return g.QuadStore.FindObjects(subject, predicate, g.Name)
}

// FindPredicates returns a list of distinct predicate terms for all triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) FindPredicates(subject, object string) []string {
	return g.QuadStore.FindPredicates(subject, object, g.Name)
}

// FindSubjects returns a list of distinct subject terms for all triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) FindSubjects(predicate, object string) []string {
	return g.QuadStore.FindSubjects(predicate, object, g.Name)
}

// ForEach executes the given callback once for each triple in the graph.
func (g *Graph) ForEach(fn TripleCallbackFn) {
	g.QuadStore.ForEachWith("*", "*", "*", g.Name, adaptTripleCallbackFn(fn))
}

// ForEachWith executes the given callback once for each triple in the graph
// that matches the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) ForEachWith(subject, predicate, object string, fn TripleCallbackFn) {
	g.QuadStore.ForEachWith(subject, predicate, object, g.Name, adaptTripleCallbackFn(fn))
}

// ForObjects executes the given callback once for each distinct object term
// for all triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) ForObjects(subject, predicate string, fn StringCallbackFn) {
	g.QuadStore.ForObjects(subject, predicate, g.Name, fn)
}

// ForPredicates executes the given callback once for each distinct predicate term
// for all triples in the graph that graph the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) ForPredicates(subject, object string, fn StringCallbackFn) {
	g.QuadStore.ForPredicates(subject, object, g.Name, fn)
}

// ForSubjects executes the given callback once for each distinct subject term
// for all triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) ForSubjects(predicate, object string, fn StringCallbackFn) {
	g.QuadStore.ForSubjects(predicate, object, g.Name, fn)
}

// Removes triples from the graph. Returns true if triples were removed,
// or false if no matching triples exist.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) Remove(subject, predicate, object string) bool {
	return g.QuadStore.Remove(subject, predicate, object, g.Name)
}

// Size returns the total count of triples in the graph.
func (g *Graph) Size() uint64 {
	gimpl, ok := g.QuadStore.graphs[g.Name]
	if !ok {
		return 0
	}
	return gimpl.size
}

// Some tests whether some triple in the graph passes the test
// implemented by the given function.
//
// The given callback is
// executed once for each triple present in the graph until
// Some finds one where the callback returns true. If such
// an element is found, iteration is immediately halted and
// Some returns true. Otherwise, if the callback returns
// false for all triples, then Some returns false.
func (g *Graph) Some(fn TripleTestFn) bool {
	return g.QuadStore.SomeWith("*", "*", "*", g.Name, adaptTripleTestFn(fn))
}

// SomeWith tests whether some triple matching the given pattern
// passes the test implemented by the given function.
//
// The given callback is
// executed once for each triple matching the given pattern until
// SomeWith finds one where the callback returns true. If such
// an element is found, iteration is immediately halted and
// SomeWith returns true. Otherwise, if the callback returns
// false for all triples, then SomeWith returns false.
func (g *Graph) SomeWith(subject, predicate, object string, fn TripleTestFn) bool {
	return g.QuadStore.SomeWith(subject, predicate, object, g.Name, adaptTripleTestFn(fn))
}

// String returns the contents of the graph in a human-readable format.
func (g *Graph) String() string {
	var buf bytes.Buffer
	name := g.Name
	if len(name) > 0 {
		buf.WriteString(name)
		buf.WriteByte('\n')
	}
	subjects := g.FindSubjects("*", "*")
	sort.Strings(subjects)
	for _, subject := range subjects {
		predicates := g.FindPredicates(subject, "*")
		sort.Strings(predicates)
		for _, predicate := range predicates {
			objects := g.FindObjects(subject, predicate)
			sort.Strings(objects)
			for _, object := range objects {
				buf.WriteByte('[')
				buf.WriteString(subject)
				buf.WriteByte(' ')
				buf.WriteString(predicate)
				buf.WriteByte(' ')
				buf.WriteString(object)
				buf.WriteByte(']')
				buf.WriteByte('\n')
			}
		}
	}
	return buf.String()
}
<commit_msg>Make NewGraph take initialisation data<commit_after>package store4

import (
	"bytes"
	"fmt"
	"sort"
)

// TripleCallbackFn is the function signature used to implement
// callback functions that receive a triple.
//
// Used with calls to Graph's ForEach and ForEachWith.
type TripleCallbackFn func(s, p, o string)

// TripleTestFn is the function signature used to implement
// callback functions performing triple tests.
// A response of true means that the test has been passed.
//
// Used with calls to Graph's Every, EveryWith, Some and SomeWith.
type TripleTestFn func(s, p, o string) bool

// Graph is a convenience façade that simply
// proxies calls to its associated QuadStore.
type Graph struct {
	Name      string
	QuadStore *QuadStore
}

// Graph returns a proxy-façade that provides a triple-based API for working with graphs within the store.
func (s *QuadStore) Graph(name string) *Graph {
	return &Graph{
		Name:      name,
		QuadStore: s,
	}
}

// NewGraph returns an unnamed graph with a
// newly created QuadStore as its backing.
func NewGraph(args ...interface{}) *Graph {
	g := NewQuadStore().Graph("")

	for _, arg := range args {
		switch arg := arg.(type) {
		default:
			panic(fmt.Sprintf("unexpected type %T\n", arg))
		case [3]string:
			// Single string triple.
			g.Add(arg[0], arg[1], arg[2])
		case [][3]string:
			// Slice of string triples.
			for _, q := range arg {
				g.Add(q[0], q[1], q[2])
			}
		}
	}

	return g
}

func adaptTripleCallbackFn(fn TripleCallbackFn) QuadCallbackFn {
	return func(s, p, o, g string) {
		fn(s, p, o)
	}
}

func adaptTripleTestFn(fn TripleTestFn) QuadTestFn {
	return func(s, p, o, g string) bool {
		return fn(s, p, o)
	}
}

// Add a triple to the graph.
// Returns true if the triple was a new triple,
// or false if the triple already existed.
//
// If any of the given terms are "*" (an asterisk),
// then this method will panic. (The asterisk is reserved
// for wildcard operations throughout the API).
func (g *Graph) Add(subject, predicate, object string) bool {
	return g.QuadStore.Add(subject, predicate, object, g.Name)
}

// Count returns a count of triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) Count(subject, predicate, object string) uint64 {
	return g.QuadStore.Count(subject, predicate, object, g.Name)
}

// Every tests whether all triples in the graph pass the test
// implemented by the given function.
//
// The given callback is
// executed once for each triple present in the graph until
// Every finds one where the callback returns false. If such
// an element is found, iteration is immediately halted and
// Every returns false. Otherwise, if the callback returns
// true for all triples, then Every returns true.
//
// Acting like the 'for all' quantifier in maths, it should
// be noted that Every returns true for an empty graph.
func (g *Graph) Every(fn TripleTestFn) bool {
	return g.QuadStore.EveryWith("*", "*", "*", g.Name, adaptTripleTestFn(fn))
}

// EveryWith tests whether all triples in the graph that match the
// given terms pass the test implemented by the given function.
//
// The given callback is
// executed once for each matching triple in the graph until
// EveryWith finds one where the callback returns false. If such
// an element is found, iteration is immediately halted and
// EveryWith returns false. Otherwise, if the callback returns
// true for all triples, then EveryWith returns true.
//
// Acting like the 'for all' quantifier in maths, it should
// be noted that EveryWith returns true for an empty graph.
// By extension, if the given parameters cause the iteration
// set to be empty, then EveryWith also returns true.
func (g *Graph) EveryWith(subject, predicate, object string, fn TripleTestFn) bool {
	return g.QuadStore.EveryWith(subject, predicate, object, g.Name, adaptTripleTestFn(fn))
}

// FindObjects returns a list of distinct object terms for all triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) FindObjects(subject, predicate string) []string {
	return g.QuadStore.FindObjects(subject, predicate, g.Name)
}

// FindPredicates returns a list of distinct predicate terms for all triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) FindPredicates(subject, object string) []string {
	return g.QuadStore.FindPredicates(subject, object, g.Name)
}

// FindSubjects returns a list of distinct subject terms for all triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) FindSubjects(predicate, object string) []string {
	return g.QuadStore.FindSubjects(predicate, object, g.Name)
}

// ForEach executes the given callback once for each triple in the graph.
func (g *Graph) ForEach(fn TripleCallbackFn) {
	g.QuadStore.ForEachWith("*", "*", "*", g.Name, adaptTripleCallbackFn(fn))
}

// ForEachWith executes the given callback once for each triple in the graph
// that matches the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) ForEachWith(subject, predicate, object string, fn TripleCallbackFn) {
	g.QuadStore.ForEachWith(subject, predicate, object, g.Name, adaptTripleCallbackFn(fn))
}

// ForObjects executes the given callback once for each distinct object term
// for all triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) ForObjects(subject, predicate string, fn StringCallbackFn) {
	g.QuadStore.ForObjects(subject, predicate, g.Name, fn)
}

// ForPredicates executes the given callback once for each distinct predicate term
// for all triples in the graph that graph the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) ForPredicates(subject, object string, fn StringCallbackFn) {
	g.QuadStore.ForPredicates(subject, object, g.Name, fn)
}

// ForSubjects executes the given callback once for each distinct subject term
// for all triples in the graph that match the given pattern.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) ForSubjects(predicate, object string, fn StringCallbackFn) {
	g.QuadStore.ForSubjects(predicate, object, g.Name, fn)
}

// Removes triples from the graph. Returns true if triples were removed,
// or false if no matching triples exist.
//
// Passing "*" (an asterisk) for any parameter acts as a
// match-everything wildcard for that term.
func (g *Graph) Remove(subject, predicate, object string) bool {
	return g.QuadStore.Remove(subject, predicate, object, g.Name)
}

// Size returns the total count of triples in the graph.
func (g *Graph) Size() uint64 {
	gimpl, ok := g.QuadStore.graphs[g.Name]
	if !ok {
		return 0
	}
	return gimpl.size
}

// Some tests whether some triple in the graph passes the test
// implemented by the given function.
//
// The given callback is
// executed once for each triple present in the graph until
// Some finds one where the callback returns true. If such
// an element is found, iteration is immediately halted and
// Some returns true. Otherwise, if the callback returns
// false for all triples, then Some returns false.
func (g *Graph) Some(fn TripleTestFn) bool {
	return g.QuadStore.SomeWith("*", "*", "*", g.Name, adaptTripleTestFn(fn))
}

// SomeWith tests whether some triple matching the given pattern
// passes the test implemented by the given function.
//
// The given callback is
// executed once for each triple matching the given pattern until
// SomeWith finds one where the callback returns true. If such
// an element is found, iteration is immediately halted and
// SomeWith returns true. Otherwise, if the callback returns
// false for all triples, then SomeWith returns false.
func (g *Graph) SomeWith(subject, predicate, object string, fn TripleTestFn) bool {
	return g.QuadStore.SomeWith(subject, predicate, object, g.Name, adaptTripleTestFn(fn))
}

// String returns the contents of the graph in a human-readable format.
func (g *Graph) String() string {
	var buf bytes.Buffer
	name := g.Name
	if len(name) > 0 {
		buf.WriteString(name)
		buf.WriteByte('\n')
	}
	subjects := g.FindSubjects("*", "*")
	sort.Strings(subjects)
	for _, subject := range subjects {
		predicates := g.FindPredicates(subject, "*")
		sort.Strings(predicates)
		for _, predicate := range predicates {
			objects := g.FindObjects(subject, predicate)
			sort.Strings(objects)
			for _, object := range objects {
				buf.WriteByte('[')
				buf.WriteString(subject)
				buf.WriteByte(' ')
				buf.WriteString(predicate)
				buf.WriteByte(' ')
				buf.WriteString(object)
				buf.WriteByte(']')
				buf.WriteByte('\n')
			}
		}
	}
	return buf.String()
}
<|endoftext|>


<commit_before>package echo

import (
	"path"
)

type (
	// Group is a set of sub-routes for a specified route. It can be used for inner
	// routes that share a common middleware or functionality that should be separate
	// from the parent echo instance while still inheriting from it.
	Group struct {
		prefix     string
		middleware []MiddlewareFunc
		echo       *Echo
	}
)

// Use implements `Echo#Use()` for sub-routes within the Group.
func (g *Group) Use(middleware ...MiddlewareFunc) {
	g.middleware = append(g.middleware, middleware...)
	// Allow all requests to reach the group as they might get dropped if router
	// doesn't find a match, making none of the group middleware process.
	g.echo.Any(path.Clean(g.prefix+"/*"), func(c Context) error {
		return NotFoundHandler(c)
	}, g.middleware...)
}

// CONNECT implements `Echo#CONNECT()` for sub-routes within the Group.
func (g *Group) CONNECT(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.add(CONNECT, path, h, m...)
}

// DELETE implements `Echo#DELETE()` for sub-routes within the Group.
func (g *Group) DELETE(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.add(DELETE, path, h, m...)
}

// GET implements `Echo#GET()` for sub-routes within the Group.
func (g *Group) GET(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.add(GET, path, h, m...)
}

// HEAD implements `Echo#HEAD()` for sub-routes within the Group.
func (g *Group) HEAD(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.add(HEAD, path, h, m...)
}

// OPTIONS implements `Echo#OPTIONS()` for sub-routes within the Group.
func (g *Group) OPTIONS(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.add(OPTIONS, path, h, m...)
}

// PATCH implements `Echo#PATCH()` for sub-routes within the Group.
func (g *Group) PATCH(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.add(PATCH, path, h, m...)
}

// POST implements `Echo#POST()` for sub-routes within the Group.
func (g *Group) POST(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.add(POST, path, h, m...)
}

// PUT implements `Echo#PUT()` for sub-routes within the Group.
func (g *Group) PUT(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.add(PUT, path, h, m...)
}

// TRACE implements `Echo#TRACE()` for sub-routes within the Group.
func (g *Group) TRACE(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.add(TRACE, path, h, m...)
}

// Any implements `Echo#Any()` for sub-routes within the Group.
func (g *Group) Any(path string, handler HandlerFunc, middleware ...MiddlewareFunc) {
	for _, m := range methods {
		g.add(m, path, handler, middleware...)
	}
}

// Match implements `Echo#Match()` for sub-routes within the Group.
func (g *Group) Match(methods []string, path string, handler HandlerFunc, middleware ...MiddlewareFunc) {
	for _, m := range methods {
		g.add(m, path, handler, middleware...)
	}
}

// Group creates a new sub-group with prefix and optional sub-group-level middleware.
func (g *Group) Group(prefix string, middleware ...MiddlewareFunc) *Group {
	m := []MiddlewareFunc{}
	m = append(m, g.middleware...)
	m = append(m, middleware...)
	return g.echo.Group(g.prefix+prefix, m...)
}

// Static implements `Echo#Static()` for sub-routes within the Group.
func (g *Group) Static(prefix, root string) {
	static(g, prefix, root)
}

// File implements `Echo#File()` for sub-routes within the Group.
func (g *Group) File(path, file string) {
	g.echo.File(g.prefix+path, file)
}

func (g *Group) add(method, path string, handler HandlerFunc, middleware ...MiddlewareFunc) *Route {
	// Combine into a new slice to avoid accidentally passing the same slice for
	// multiple routes, which would lead to later add() calls overwriting the
	// middleware from earlier calls.
	m := []MiddlewareFunc{}
	m = append(m, g.middleware...)
	m = append(m, middleware...)
	return g.echo.Add(method, g.prefix+path, handler, m...)
}
<commit_msg>Expose group.add() method for dynamic route registration (#985)<commit_after>package echo

import (
	"path"
)

type (
	// Group is a set of sub-routes for a specified route. It can be used for inner
	// routes that share a common middleware or functionality that should be separate
	// from the parent echo instance while still inheriting from it.
	Group struct {
		prefix     string
		middleware []MiddlewareFunc
		echo       *Echo
	}
)

// Use implements `Echo#Use()` for sub-routes within the Group.
func (g *Group) Use(middleware ...MiddlewareFunc) {
	g.middleware = append(g.middleware, middleware...)
	// Allow all requests to reach the group as they might get dropped if router
	// doesn't find a match, making none of the group middleware process.
	g.echo.Any(path.Clean(g.prefix+"/*"), func(c Context) error {
		return NotFoundHandler(c)
	}, g.middleware...)
}

// CONNECT implements `Echo#CONNECT()` for sub-routes within the Group.
func (g *Group) CONNECT(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.Add(CONNECT, path, h, m...)
}

// DELETE implements `Echo#DELETE()` for sub-routes within the Group.
func (g *Group) DELETE(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.Add(DELETE, path, h, m...)
}

// GET implements `Echo#GET()` for sub-routes within the Group.
func (g *Group) GET(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.Add(GET, path, h, m...)
}

// HEAD implements `Echo#HEAD()` for sub-routes within the Group.
func (g *Group) HEAD(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.Add(HEAD, path, h, m...)
}

// OPTIONS implements `Echo#OPTIONS()` for sub-routes within the Group.
func (g *Group) OPTIONS(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.Add(OPTIONS, path, h, m...)
}

// PATCH implements `Echo#PATCH()` for sub-routes within the Group.
func (g *Group) PATCH(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.Add(PATCH, path, h, m...)
}

// POST implements `Echo#POST()` for sub-routes within the Group.
func (g *Group) POST(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.Add(POST, path, h, m...)
}

// PUT implements `Echo#PUT()` for sub-routes within the Group.
func (g *Group) PUT(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.Add(PUT, path, h, m...)
}

// TRACE implements `Echo#TRACE()` for sub-routes within the Group.
func (g *Group) TRACE(path string, h HandlerFunc, m ...MiddlewareFunc) *Route {
	return g.Add(TRACE, path, h, m...)
}

// Any implements `Echo#Any()` for sub-routes within the Group.
func (g *Group) Any(path string, handler HandlerFunc, middleware ...MiddlewareFunc) {
	for _, m := range methods {
		g.Add(m, path, handler, middleware...)
	}
}

// Match implements `Echo#Match()` for sub-routes within the Group.
func (g *Group) Match(methods []string, path string, handler HandlerFunc, middleware ...MiddlewareFunc) {
	for _, m := range methods {
		g.Add(m, path, handler, middleware...)
	}
}

// Group creates a new sub-group with prefix and optional sub-group-level middleware.
func (g *Group) Group(prefix string, middleware ...MiddlewareFunc) *Group {
	m := []MiddlewareFunc{}
	m = append(m, g.middleware...)
	m = append(m, middleware...)
	return g.echo.Group(g.prefix+prefix, m...)
}

// Static implements `Echo#Static()` for sub-routes within the Group.
func (g *Group) Static(prefix, root string) {
	static(g, prefix, root)
}

// File implements `Echo#File()` for sub-routes within the Group.
func (g *Group) File(path, file string) {
	g.echo.File(g.prefix+path, file)
}

// Add implements `Echo#Add()` for sub-routes within the Group.
func (g *Group) Add(method, path string, handler HandlerFunc, middleware ...MiddlewareFunc) *Route {
	// Combine into a new slice to avoid accidentally passing the same slice for
	// multiple routes, which would lead to later add() calls overwriting the
	// middleware from earlier calls.
	m := []MiddlewareFunc{}
	m = append(m, g.middleware...)
	m = append(m, middleware...)
	return g.echo.Add(method, g.prefix+path, handler, m...)
}
<|endoftext|>


<commit_before>package forecaster

import (
	"context"
	"time"
)

var (
	accurate = fake{
		name: "accurate",
		d:    delay{0.2, time.Second},
	}
	quick = fake{
		name: "quick",
		d:    delay{0.01, 200 * time.Millisecond},
	}
)

type fake struct {
	name string
	d    delay
}

func (s *fake) Forecast(ctx context.Context) (*Response, error) {
	respCh := make(chan Response)
	go func() {
		resp := Response{s.name, doNothing(s.d.delay())}
		select {
		case <-ctx.Done():
		case respCh <- resp:
		}
	}()
	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case resp := <-respCh:
		return &resp, nil
	}
}

func doNothing(delay time.Duration) time.Time {
	if delay > 0 {
		time.Sleep(delay)
	}
	return time.Now()
}
<commit_msg>added comments to forecaster example<commit_after>// Package forecaster allows to test Fallback approach.
//
// The main idea is to have two different providers with a balance between an accurate data and quick data retrieving.
// Usually, quick load gives dirty data. Dirty data allows to avoid unreliable response.
//
// It will work if a system was ready to take a low rate of dirty data. It's a sponsor requirement.
package forecaster

import (
	"context"
	"time"
)

var (
	// accurate works slowly
	accurate = fake{
		name: "accurate",
		d:    delay{0.2, time.Second},
	}
	// quick returns dirty data
	quick = fake{
		name: "quick",
		d:    delay{0.01, 200 * time.Millisecond},
	}
)

type fake struct {
	name string
	d    delay
}

func (s *fake) Forecast(ctx context.Context) (*Response, error) {
	respCh := make(chan Response)
	go func() {
		resp := Response{s.name, doNothing(s.d.delay())}
		select {
		case <-ctx.Done():
		case respCh <- resp:
		}
	}()
	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case resp := <-respCh:
		return &resp, nil
	}
}

func doNothing(delay time.Duration) time.Time {
	if delay > 0 {
		time.Sleep(delay)
	}
	return time.Now()
}
<|endoftext|>


<commit_before>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build example

package main

import (
	"fmt"
	"log"
	"math"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/ebitenutil"
)

const (
	screenWidth  = 640
	screenHeight = 640
	maxIt        = 128
)

var (
	offscreen    *ebiten.Image
	offscreenPix []byte
	palette      [maxIt]byte
)

func init() {
	offscreen, _ = ebiten.NewImage(screenWidth, screenHeight, ebiten.FilterNearest)
	offscreenPix = make([]byte, screenWidth*screenHeight*4)
	for i := range palette {
		c := byte(math.Sqrt(float64(i)/float64(len(palette))) * 0xff)
		palette[i] = c
	}
}

func color(it int) (r, g, b byte) {
	if it == maxIt {
		return 0, 0, 0
	}
	c := palette[it]
	return c, c, c
}

func updateOffscreen(centerX, centerY, size float64) {
	for j := 0; j < screenHeight; j++ {
		for i := 0; i < screenHeight; i++ {
			x := float64(i)*size/screenWidth - size/2 + centerX
			y := (screenHeight-float64(j))*size/screenHeight - size/2 + centerY
			c := complex(x, y)
			z := c
			it := 0
			for ; it < maxIt; it++ {
				nz := z*z + c
				if real(nz)*real(nz)+imag(nz)*imag(nz) > 4 {
					break
				}
				z = nz
			}
			r, g, b := color(it)
			p := 4 * (i + j*screenWidth)
			offscreenPix[p] = r
			offscreenPix[p+1] = g
			offscreenPix[p+2] = b
			offscreenPix[p+3] = 0xff
		}
	}
	offscreen.ReplacePixels(offscreenPix)
}

func init() {
	// Now it is not feasible to call updateOffscreen every frame due to performance.
	updateOffscreen(-0.75, 0.25, 2)
}

func update(screen *ebiten.Image) error {
	if ebiten.IsRunningSlowly() {
		return nil
	}

	screen.DrawImage(offscreen, nil)
	ebitenutil.DebugPrint(screen, fmt.Sprintf("FPS: %0.2f", ebiten.CurrentFPS()))
	return nil
}

func main() {
	if err := ebiten.Run(update, screenWidth, screenHeight, 1, "Mandelbrot (Ebiten Demo)"); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>examples/mandelbrot: Fix colors<commit_after>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build example

package main

import (
	"fmt"
	"log"
	"math"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/ebitenutil"
)

const (
	screenWidth  = 640
	screenHeight = 640
	maxIt        = 128
)

var (
	offscreen    *ebiten.Image
	offscreenPix []byte
	palette      [maxIt]byte
)

func init() {
	offscreen, _ = ebiten.NewImage(screenWidth, screenHeight, ebiten.FilterNearest)
	offscreenPix = make([]byte, screenWidth*screenHeight*4)
	for i := range palette {
		c := byte(math.Sqrt(float64(i)/float64(len(palette))) * 0xff)
		palette[i] = c
	}
}

func color(it int) (r, g, b byte) {
	if it == maxIt {
		return 0xff, 0xff, 0xff
	}
	c := palette[it]
	return c, c, c
}

func updateOffscreen(centerX, centerY, size float64) {
	for j := 0; j < screenHeight; j++ {
		for i := 0; i < screenHeight; i++ {
			x := float64(i)*size/screenWidth - size/2 + centerX
			y := (screenHeight-float64(j))*size/screenHeight - size/2 + centerY
			c := complex(x, y)
			z := c
			it := 0
			for ; it < maxIt; it++ {
				nz := z*z + c
				if real(nz)*real(nz)+imag(nz)*imag(nz) > 4 {
					break
				}
				z = nz
			}
			r, g, b := color(it)
			p := 4 * (i + j*screenWidth)
			offscreenPix[p] = r
			offscreenPix[p+1] = g
			offscreenPix[p+2] = b
			offscreenPix[p+3] = 0xff
		}
	}
	offscreen.ReplacePixels(offscreenPix)
}

func init() {
	// Now it is not feasible to call updateOffscreen every frame due to performance.
	updateOffscreen(-0.75, 0.25, 2)
}

func update(screen *ebiten.Image) error {
	if ebiten.IsRunningSlowly() {
		return nil
	}

	screen.DrawImage(offscreen, nil)
	ebitenutil.DebugPrint(screen, fmt.Sprintf("FPS: %0.2f", ebiten.CurrentFPS()))
	return nil
}

func main() {
	if err := ebiten.Run(update, screenWidth, screenHeight, 1, "Mandelbrot (Ebiten Demo)"); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2014-2015, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package query

import (
	"encoding/json"
	"fmt"
	"sync"

	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/percona-agent/instance"
	"github.com/percona/percona-agent/mysql"
	"github.com/percona/percona-agent/pct"
	mysqlExec "github.com/percona/percona-agent/query/mysql"
)

const (
	SERVICE_NAME = "query"
)

type Manager struct {
	logger       *pct.Logger
	instanceRepo *instance.Repo
	connFactory  mysql.ConnectionFactory
	// --
	running bool
	sync.Mutex
	status *pct.Status
}

func NewManager(logger *pct.Logger, instanceRepo *instance.Repo, connFactory mysql.ConnectionFactory) *Manager {
	m := &Manager{
		logger:       logger,
		instanceRepo: instanceRepo,
		connFactory:  connFactory,
		// --
		status: pct.NewStatus([]string{SERVICE_NAME}),
	}
	return m
}

/////////////////////////////////////////////////////////////////////////////
// Interface
/////////////////////////////////////////////////////////////////////////////

func (m *Manager) Start() error {
	m.Lock()
	defer m.Unlock()
	if m.running {
		return pct.ServiceIsRunningError{Service: SERVICE_NAME}
	}
	m.running = true
	m.logger.Info("Started")
	m.status.Update(SERVICE_NAME, "Idle")
	return nil
}

func (m *Manager) Stop() error {
	// Let user stop this tool in case they don't want agent executing queries.
	m.Lock()
	defer m.Unlock()
	if !m.running {
		return nil
	}
	m.running = false
	m.logger.Info("Stopped")
	m.status.Update(SERVICE_NAME, "Stopped")
	return nil
}

func (m *Manager) Handle(cmd *proto.Cmd) *proto.Reply {
	m.Lock()
	defer m.Unlock()

	// Don't query if this tool is stopped.
	if !m.running {
		return cmd.Reply(nil, pct.ServiceIsNotRunningError{})
	}

	m.status.UpdateRe(SERVICE_NAME, "Handling", cmd)
	defer m.status.Update(SERVICE_NAME, "Idle")

	// See which type of subsystem this query is for. Right now we only support
	// MySQL, but this abstraction will make adding other subsystems easy.
	si := &proto.ServiceInstance{}
	if err := json.Unmarshal(cmd.Data, si); err != nil {
		return cmd.Reply(nil, err)
	}

	switch si.Service {
	case "mysql":
		return m.handleMySQLQuery(cmd, si)
	default:
		return cmd.Reply(nil, pct.UnknownCmdError{si.Service})
	}
}

func (m *Manager) Status() map[string]string {
	return m.status.All()
}

func (m *Manager) GetConfig() ([]proto.AgentConfig, []error) {
	return nil, nil
}

// --------------------------------------------------------------------------

func (m *Manager) handleMySQLQuery(cmd *proto.Cmd, si *proto.ServiceInstance) *proto.Reply {
	m.logger.Debug("handleMySQLQuery:call")
	defer m.logger.Debug("handleMySQLQuery:return")

	// Connect to MySQL.
	mysqlIt := &proto.MySQLInstance{}
	if err := m.instanceRepo.Get(si.Service, si.InstanceId, mysqlIt); err != nil {
		return cmd.Reply(nil, err)
	}
	conn := m.connFactory.Make(mysqlIt.DSN)
	if err := conn.Connect(1); err != nil {
		return cmd.Reply(nil, fmt.Errorf("Cannot connect to MySQL: %s", err))
	}
	defer conn.Close()

	// Create a MySQL query executor to do the actual work.
	e := mysqlExec.NewQueryExecutor(conn)

	// Get the instance name, e.g. mysql-db01, to make status human-readable.
	instanceName := m.instanceRepo.Name(si.Service, si.InstanceId)

	// Execute the query.
	m.logger.Debug(cmd.Cmd + ":" + instanceName)
	switch cmd.Cmd {
	case "Explain":
		m.status.Update(SERVICE_NAME, "EXPLAIN query on "+instanceName)
		q := &proto.ExplainQuery{}
		if err := json.Unmarshal(cmd.Data, q); err != nil {
			return cmd.Reply(nil, err)
		}
		res, err := e.Explain(q.Db, q.Query)
		if err != nil {
			return cmd.Reply(nil, fmt.Errorf("EXPLAIN failed: %s", err))
		}
		return cmd.Reply(res, nil)
	case "TableInfo":
		m.status.Update(SERVICE_NAME, "Table Info queries on "+instanceName)
		tableInfo := &proto.TableInfoQuery{}
		if err := json.Unmarshal(cmd.Data, tableInfo); err != nil {
			return cmd.Reply(nil, err)
		}
		res, err := e.TableInfo(tableInfo)
		if err != nil {
			return cmd.Reply(nil, fmt.Errorf("Table Info failed: %s", err))
		}
		return cmd.Reply(res, nil)
	default:
		return cmd.Reply(nil, pct.UnknownCmdError{Cmd: cmd.Cmd})
	}
}
<commit_msg>Fix err type: UnknownServiceInstanceError not UnknownCmdError.<commit_after>/*
   Copyright (c) 2014-2015, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package query

import (
	"encoding/json"
	"fmt"
	"sync"

	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/percona-agent/instance"
	"github.com/percona/percona-agent/mysql"
	"github.com/percona/percona-agent/pct"
	mysqlExec "github.com/percona/percona-agent/query/mysql"
)

const (
	SERVICE_NAME = "query"
)

type Manager struct {
	logger       *pct.Logger
	instanceRepo *instance.Repo
	connFactory  mysql.ConnectionFactory
	// --
	running bool
	sync.Mutex
	status *pct.Status
}

func NewManager(logger *pct.Logger, instanceRepo *instance.Repo, connFactory mysql.ConnectionFactory) *Manager {
	m := &Manager{
		logger:       logger,
		instanceRepo: instanceRepo,
		connFactory:  connFactory,
		// --
		status: pct.NewStatus([]string{SERVICE_NAME}),
	}
	return m
}

/////////////////////////////////////////////////////////////////////////////
// Interface
/////////////////////////////////////////////////////////////////////////////

func (m *Manager) Start() error {
	m.Lock()
	defer m.Unlock()
	if m.running {
		return pct.ServiceIsRunningError{Service: SERVICE_NAME}
	}
	m.running = true
	m.logger.Info("Started")
	m.status.Update(SERVICE_NAME, "Idle")
	return nil
}

func (m *Manager) Stop() error {
	// Let user stop this tool in case they don't want agent executing queries.
	m.Lock()
	defer m.Unlock()
	if !m.running {
		return nil
	}
	m.running = false
	m.logger.Info("Stopped")
	m.status.Update(SERVICE_NAME, "Stopped")
	return nil
}

func (m *Manager) Handle(cmd *proto.Cmd) *proto.Reply {
	m.Lock()
	defer m.Unlock()

	// Don't query if this tool is stopped.
	if !m.running {
		return cmd.Reply(nil, pct.ServiceIsNotRunningError{})
	}

	m.status.UpdateRe(SERVICE_NAME, "Handling", cmd)
	defer m.status.Update(SERVICE_NAME, "Idle")

	// See which type of subsystem this query is for. Right now we only support
	// MySQL, but this abstraction will make adding other subsystems easy.
	si := &proto.ServiceInstance{}
	if err := json.Unmarshal(cmd.Data, si); err != nil {
		return cmd.Reply(nil, err)
	}

	switch si.Service {
	case "mysql":
		return m.handleMySQLQuery(cmd, si)
	default:
		return cmd.Reply(nil, pct.UnknownServiceInstanceError{si.Service, si.Id})
	}
}

func (m *Manager) Status() map[string]string {
	return m.status.All()
}

func (m *Manager) GetConfig() ([]proto.AgentConfig, []error) {
	return nil, nil
}

// --------------------------------------------------------------------------

func (m *Manager) handleMySQLQuery(cmd *proto.Cmd, si *proto.ServiceInstance) *proto.Reply {
	m.logger.Debug("handleMySQLQuery:call")
	defer m.logger.Debug("handleMySQLQuery:return")

	// Connect to MySQL.
	mysqlIt := &proto.MySQLInstance{}
	if err := m.instanceRepo.Get(si.Service, si.InstanceId, mysqlIt); err != nil {
		return cmd.Reply(nil, err)
	}
	conn := m.connFactory.Make(mysqlIt.DSN)
	if err := conn.Connect(1); err != nil {
		return cmd.Reply(nil, fmt.Errorf("Cannot connect to MySQL: %s", err))
	}
	defer conn.Close()

	// Create a MySQL query executor to do the actual work.
	e := mysqlExec.NewQueryExecutor(conn)

	// Get the instance name, e.g. mysql-db01, to make status human-readable.
	instanceName := m.instanceRepo.Name(si.Service, si.InstanceId)

	// Execute the query.
	m.logger.Debug(cmd.Cmd + ":" + instanceName)
	switch cmd.Cmd {
	case "Explain":
		m.status.Update(SERVICE_NAME, "EXPLAIN query on "+instanceName)
		q := &proto.ExplainQuery{}
		if err := json.Unmarshal(cmd.Data, q); err != nil {
			return cmd.Reply(nil, err)
		}
		res, err := e.Explain(q.Db, q.Query)
		if err != nil {
			return cmd.Reply(nil, fmt.Errorf("EXPLAIN failed: %s", err))
		}
		return cmd.Reply(res, nil)
	case "TableInfo":
		m.status.Update(SERVICE_NAME, "Table Info queries on "+instanceName)
		tableInfo := &proto.TableInfoQuery{}
		if err := json.Unmarshal(cmd.Data, tableInfo); err != nil {
			return cmd.Reply(nil, err)
		}
		res, err := e.TableInfo(tableInfo)
		if err != nil {
			return cmd.Reply(nil, fmt.Errorf("Table Info failed: %s", err))
		}
		return cmd.Reply(res, nil)
	default:
		return cmd.Reply(nil, pct.UnknownCmdError{Cmd: cmd.Cmd})
	}
}
<|endoftext|>


<commit_before>package grpc_test

import (
	"context"
	"fmt"
	"net"
	"testing"

	"google.golang.org/grpc"

	test "github.com/go-kit/kit/transport/grpc/_grpc_test"
	"github.com/go-kit/kit/transport/grpc/_grpc_test/pb"
)

const (
	hostPort string = "localhost:8002"
)

func TestGRPCClient(t *testing.T) {
	var (
		server  = grpc.NewServer()
		service = test.NewService()
	)

	sc, err := net.Listen("tcp", hostPort)
	if err != nil {
		t.Fatalf("unable to listen: %+v", err)
	}
	defer server.GracefulStop()

	go func() {
		pb.RegisterTestServer(server, test.NewBinding(service))
		_ = server.Serve(sc)
	}()

	cc, err := grpc.Dial(hostPort, grpc.WithInsecure())
	if err != nil {
		t.Fatalf("unable to Dial: %+v", err)
	}

	client := test.NewClient(cc)

	var (
		a   = "the answer to life the universe and everything"
		b   = int64(42)
		cID = "request-1"
		ctx = test.SetCorrelationID(context.Background(), cID)
	)

	responseCTX, v, err := client.Test(ctx, a, b)

	if want, have := fmt.Sprintf("%s = %d", a, b), v; want != have {
		t.Fatalf("want %q, have %q", want, have)
	}

	if want, have := cID, test.GetConsumedCorrelationID(responseCTX); want != have {
		t.Fatalf("want %q, have %q", want, have)
	}
}
<commit_msg>Fix swallowed error in grpc_test.<commit_after>package grpc_test

import (
	"context"
	"fmt"
	"net"
	"testing"

	"google.golang.org/grpc"

	test "github.com/go-kit/kit/transport/grpc/_grpc_test"
	"github.com/go-kit/kit/transport/grpc/_grpc_test/pb"
)

const (
	hostPort string = "localhost:8002"
)

func TestGRPCClient(t *testing.T) {
	var (
		server  = grpc.NewServer()
		service = test.NewService()
	)

	sc, err := net.Listen("tcp", hostPort)
	if err != nil {
		t.Fatalf("unable to listen: %+v", err)
	}
	defer server.GracefulStop()

	go func() {
		pb.RegisterTestServer(server, test.NewBinding(service))
		_ = server.Serve(sc)
	}()

	cc, err := grpc.Dial(hostPort, grpc.WithInsecure())
	if err != nil {
		t.Fatalf("unable to Dial: %+v", err)
	}

	client := test.NewClient(cc)

	var (
		a   = "the answer to life the universe and everything"
		b   = int64(42)
		cID = "request-1"
		ctx = test.SetCorrelationID(context.Background(), cID)
	)

	responseCTX, v, err := client.Test(ctx, a, b)
	if err != nil {
		t.Fatalf("unable to Test: %+v", err)
	}
	if want, have := fmt.Sprintf("%s = %d", a, b), v; want != have {
		t.Fatalf("want %q, have %q", want, have)
	}

	if want, have := cID, test.GetConsumedCorrelationID(responseCTX); want != have {
		t.Fatalf("want %q, have %q", want, have)
	}
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "math"

func main() {
	fmt.Println("Hello, ??")
	log2()
}

func log2() {
fmt.Println(math.Log2(2))
}<commit_msg>changes<commit_after>package main

import "fmt"
import "math"

func main() {
	fmt.Println("Hello, ??")
<<<<<<< HEAD
	log2()
}

func log2() {
fmt.Println(math.Log2(2))
}
=======
	fmt.Println("Heisann")
}
>>>>>>> 565d01d9da9a85e1ce39beefffbc2b8ccbfa3ca1
<|endoftext|>


<commit_before>package types

import (
	"crypto/ecdsa"
	"errors"
	"fmt"
	"math/big"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/crypto/secp256k1"
	"github.com/ethereum/go-ethereum/rlp"
)

func IsContractAddr(addr []byte) bool {
	return len(addr) == 0
}

type Transaction struct {
	AccountNonce uint64
	Price        *big.Int
	GasLimit     *big.Int
	Recipient    *common.Address // nil means contract creation
	Amount       *big.Int
	Payload      []byte
	V            byte
	R, S         []byte
}

func NewContractCreationTx(amount, gasLimit, gasPrice *big.Int, data []byte) *Transaction {
	return &Transaction{Recipient: nil, Amount: amount, GasLimit: gasLimit, Price: gasPrice, Payload: data}
}

func NewTransactionMessage(to common.Address, amount, gasAmount, gasPrice *big.Int, data []byte) *Transaction {
	return &Transaction{Recipient: &to, Amount: amount, GasLimit: gasAmount, Price: gasPrice, Payload: data}
}

func NewTransactionFromBytes(data []byte) *Transaction {
	// TODO: remove this function if possible. callers would
	// much better off decoding into transaction directly.
	// it's not that hard.
	tx := new(Transaction)
	rlp.DecodeBytes(data, tx)
	return tx
}

func (tx *Transaction) Hash() common.Hash {
	return rlpHash([]interface{}{
		tx.AccountNonce, tx.Price, tx.GasLimit, tx.Recipient, tx.Amount, tx.Payload,
	})
}

func (self *Transaction) Data() []byte {
	return self.Payload
}

func (self *Transaction) Gas() *big.Int {
	return self.GasLimit
}

func (self *Transaction) GasPrice() *big.Int {
	return self.Price
}

func (self *Transaction) Value() *big.Int {
	return self.Amount
}

func (self *Transaction) Nonce() uint64 {
	return self.AccountNonce
}

func (self *Transaction) SetNonce(AccountNonce uint64) {
	self.AccountNonce = AccountNonce
}

func (self *Transaction) From() (common.Address, error) {
	pubkey := self.PublicKey()
	if len(pubkey) == 0 || pubkey[0] != 4 {
		return common.Address{}, errors.New("invalid public key")
	}
	var addr common.Address
	copy(addr[:], crypto.Sha3(pubkey[1:]))
	return addr, nil
}

// To returns the recipient of the transaction.
// If transaction is a contract creation (with no recipient address)
// To returns nil.
func (tx *Transaction) To() *common.Address {
	return tx.Recipient
}

func (tx *Transaction) Curve() (v byte, r []byte, s []byte) {
	v = byte(tx.V)
	r = common.LeftPadBytes(tx.R, 32)
	s = common.LeftPadBytes(tx.S, 32)
	return
}

func (tx *Transaction) Signature(key []byte) []byte {
	hash := tx.Hash()
	sig, _ := secp256k1.Sign(hash[:], key)
	return sig
}

func (tx *Transaction) PublicKey() []byte {
	hash := tx.Hash()
	v, r, s := tx.Curve()
	sig := append(r, s...)
	sig = append(sig, v-27)

	//pubkey := crypto.Ecrecover(append(hash, sig...))
	pubkey, _ := secp256k1.RecoverPubkey(hash[:], sig)
	return pubkey
}

func (tx *Transaction) SetSignatureValues(sig []byte) error {
	tx.R = sig[:32]
	tx.S = sig[32:64]
	tx.V = sig[64] + 27
	return nil
}

func (tx *Transaction) SignECDSA(prv *ecdsa.PrivateKey) error {
	h := tx.Hash()
	sig, err := crypto.Sign(h[:], prv)
	if err != nil {
		return err
	}
	tx.SetSignatureValues(sig)
	return nil
}

// TODO: remove
func (tx *Transaction) RlpData() interface{} {
	data := []interface{}{tx.AccountNonce, tx.Price, tx.GasLimit, tx.Recipient, tx.Amount, tx.Payload}
	return append(data, tx.V, new(big.Int).SetBytes(tx.R).Bytes(), new(big.Int).SetBytes(tx.S).Bytes())
}

func (tx *Transaction) String() string {
	var from, to string
	if f, err := tx.From(); err != nil {
		from = "[invalid sender]"
	} else {
		from = fmt.Sprintf("%x", f[:])
	}
	if t := tx.To(); t == nil {
		to = "[contract creation]"
	} else {
		to = fmt.Sprintf("%x", t[:])
	}
	enc, _ := rlp.EncodeToBytes(tx)
	return fmt.Sprintf(`
	TX(%x)
	Contract: %v
	From:     %s
	To:       %s
	Nonce:    %v
	GasPrice: %v
	GasLimit  %v
	Value:    %v
	Data:     0x%x
	V:        0x%x
	R:        0x%x
	S:        0x%x
	Hex:      %x
`,
		tx.Hash(),
		len(tx.Recipient) == 0,
		from,
		to,
		tx.AccountNonce,
		tx.Price,
		tx.GasLimit,
		tx.Amount,
		tx.Payload,
		tx.V,
		tx.R,
		tx.S,
		enc,
	)
}

// Transaction slice type for basic sorting
type Transactions []*Transaction

// TODO: remove
func (self Transactions) RlpData() interface{} {
	// Marshal the transactions of this block
	enc := make([]interface{}, len(self))
	for i, tx := range self {
		// Cast it to a string (safe)
		enc[i] = tx.RlpData()
	}

	return enc
}

func (s Transactions) Len() int      { return len(s) }
func (s Transactions) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

func (s Transactions) GetRlp(i int) []byte {
	enc, _ := rlp.EncodeToBytes(s[i])
	return enc
}

type TxByNonce struct{ Transactions }

func (s TxByNonce) Less(i, j int) bool {
	return s.Transactions[i].AccountNonce < s.Transactions[j].AccountNonce
}
<commit_msg>Fixed incorrect recipient derived<commit_after>package types

import (
	"crypto/ecdsa"
	"errors"
	"fmt"
	"math/big"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/crypto/secp256k1"
	"github.com/ethereum/go-ethereum/rlp"
)

func IsContractAddr(addr []byte) bool {
	return len(addr) == 0
}

type Transaction struct {
	AccountNonce uint64
	Price        *big.Int
	GasLimit     *big.Int
	Recipient    *common.Address // nil means contract creation
	Amount       *big.Int
	Payload      []byte
	V            byte
	R, S         []byte
}

func NewContractCreationTx(amount, gasLimit, gasPrice *big.Int, data []byte) *Transaction {
	return &Transaction{Recipient: nil, Amount: amount, GasLimit: gasLimit, Price: gasPrice, Payload: data}
}

func NewTransactionMessage(to common.Address, amount, gasAmount, gasPrice *big.Int, data []byte) *Transaction {
	return &Transaction{Recipient: &to, Amount: amount, GasLimit: gasAmount, Price: gasPrice, Payload: data}
}

func NewTransactionFromBytes(data []byte) *Transaction {
	// TODO: remove this function if possible. callers would
	// much better off decoding into transaction directly.
	// it's not that hard.
	tx := new(Transaction)
	rlp.DecodeBytes(data, tx)
	return tx
}

func (tx *Transaction) Hash() common.Hash {
	return rlpHash([]interface{}{
		tx.AccountNonce, tx.Price, tx.GasLimit, tx.Recipient, tx.Amount, tx.Payload,
	})
}

func (self *Transaction) Data() []byte {
	return self.Payload
}

func (self *Transaction) Gas() *big.Int {
	return self.GasLimit
}

func (self *Transaction) GasPrice() *big.Int {
	return self.Price
}

func (self *Transaction) Value() *big.Int {
	return self.Amount
}

func (self *Transaction) Nonce() uint64 {
	return self.AccountNonce
}

func (self *Transaction) SetNonce(AccountNonce uint64) {
	self.AccountNonce = AccountNonce
}

func (self *Transaction) From() (common.Address, error) {
	pubkey := self.PublicKey()
	if len(pubkey) == 0 || pubkey[0] != 4 {
		return common.Address{}, errors.New("invalid public key")
	}
	var addr common.Address
	copy(addr[:], crypto.Sha3(pubkey[1:])[12:])
	return addr, nil
}

// To returns the recipient of the transaction.
// If transaction is a contract creation (with no recipient address)
// To returns nil.
func (tx *Transaction) To() *common.Address {
	return tx.Recipient
}

func (tx *Transaction) Curve() (v byte, r []byte, s []byte) {
	v = byte(tx.V)
	r = common.LeftPadBytes(tx.R, 32)
	s = common.LeftPadBytes(tx.S, 32)
	return
}

func (tx *Transaction) Signature(key []byte) []byte {
	hash := tx.Hash()
	sig, _ := secp256k1.Sign(hash[:], key)
	return sig
}

func (tx *Transaction) PublicKey() []byte {
	hash := tx.Hash()
	v, r, s := tx.Curve()
	sig := append(r, s...)
	sig = append(sig, v-27)

	//pubkey := crypto.Ecrecover(append(hash, sig...))
	pubkey, _ := secp256k1.RecoverPubkey(hash[:], sig)
	return pubkey
}

func (tx *Transaction) SetSignatureValues(sig []byte) error {
	tx.R = sig[:32]
	tx.S = sig[32:64]
	tx.V = sig[64] + 27
	return nil
}

func (tx *Transaction) SignECDSA(prv *ecdsa.PrivateKey) error {
	h := tx.Hash()
	sig, err := crypto.Sign(h[:], prv)
	if err != nil {
		return err
	}
	tx.SetSignatureValues(sig)
	return nil
}

// TODO: remove
func (tx *Transaction) RlpData() interface{} {
	data := []interface{}{tx.AccountNonce, tx.Price, tx.GasLimit, tx.Recipient, tx.Amount, tx.Payload}
	return append(data, tx.V, new(big.Int).SetBytes(tx.R).Bytes(), new(big.Int).SetBytes(tx.S).Bytes())
}

func (tx *Transaction) String() string {
	var from, to string
	if f, err := tx.From(); err != nil {
		from = "[invalid sender]"
	} else {
		from = fmt.Sprintf("%x", f[:])
	}
	if t := tx.To(); t == nil {
		to = "[contract creation]"
	} else {
		to = fmt.Sprintf("%x", t[:])
	}
	enc, _ := rlp.EncodeToBytes(tx)
	return fmt.Sprintf(`
	TX(%x)
	Contract: %v
	From:     %s
	To:       %s
	Nonce:    %v
	GasPrice: %v
	GasLimit  %v
	Value:    %v
	Data:     0x%x
	V:        0x%x
	R:        0x%x
	S:        0x%x
	Hex:      %x
`,
		tx.Hash(),
		len(tx.Recipient) == 0,
		from,
		to,
		tx.AccountNonce,
		tx.Price,
		tx.GasLimit,
		tx.Amount,
		tx.Payload,
		tx.V,
		tx.R,
		tx.S,
		enc,
	)
}

// Transaction slice type for basic sorting
type Transactions []*Transaction

// TODO: remove
func (self Transactions) RlpData() interface{} {
	// Marshal the transactions of this block
	enc := make([]interface{}, len(self))
	for i, tx := range self {
		// Cast it to a string (safe)
		enc[i] = tx.RlpData()
	}

	return enc
}

func (s Transactions) Len() int      { return len(s) }
func (s Transactions) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

func (s Transactions) GetRlp(i int) []byte {
	enc, _ := rlp.EncodeToBytes(s[i])
	return enc
}

type TxByNonce struct{ Transactions }

func (s TxByNonce) Less(i, j int) bool {
	return s.Transactions[i].AccountNonce < s.Transactions[j].AccountNonce
}
<|endoftext|>


<commit_before>package reader

import (
	"encoding/binary"
	"io"
	"reflect"
	"sync"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/marshal"
	"github.com/xitongsys/parquet-go/source"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/parquet"
)

type ParquetReader struct {
	SchemaHandler *schema.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         source.ParquetFile

	ColumnBuffers map[string]*ColumnBufferType
}

//Create a parquet reader
func NewParquetReader(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetReader, error) {
	var err error
	res := new(ParquetReader)
	res.NP = np
	res.PFile = pFile
	if err = res.ReadFooter(); err != nil {
		return nil, err
	}
	res.ColumnBuffers = make(map[string]*ColumnBufferType)

	if obj != nil {
		if res.SchemaHandler, err = schema.NewSchemaHandlerFromStruct(obj); err != nil {
			return res, err
		}
		res.RenameSchema()

		for i := 0; i < len(res.SchemaHandler.SchemaElements); i++ {
			schema := res.SchemaHandler.SchemaElements[i]
			if schema.GetNumChildren() == 0 {
				pathStr := res.SchemaHandler.IndexMap[int32(i)]
				if res.ColumnBuffers[pathStr], err = NewColumnBuffer(pFile, res.Footer, res.SchemaHandler, pathStr); err != nil {
					return res, err
				}
			}
		}
	}
	return res, nil
}

func (self *ParquetReader) SetSchemaHandlerFromJSON(jsonSchema string) error {
	var err error
	if self.SchemaHandler, err = schema.NewSchemaHandlerFromJSON(jsonSchema); err != nil {
		return err
	}
	self.RenameSchema()
	for i := 0; i < len(self.SchemaHandler.SchemaElements); i++ {
		schemaElement := self.SchemaHandler.SchemaElements[i]
		if schemaElement.GetNumChildren() == 0 {
			pathStr := self.SchemaHandler.IndexMap[int32(i)]
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}
	return nil
}

//Rename schema name to inname
func (self *ParquetReader) RenameSchema() {
	for i := 0; i < len(self.SchemaHandler.Infos); i++ {
		self.Footer.Schema[i].Name = self.SchemaHandler.Infos[i].InName
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			exPath := make([]string, 0)
			exPath = append(exPath, self.SchemaHandler.GetRootName())
			exPath = append(exPath, chunk.MetaData.GetPathInSchema()...)
			exPathStr := common.PathToStr(exPath)

			inPathStr := self.SchemaHandler.ExPathToInPath[exPathStr]
			inPath := common.StrToPath(inPathStr)[1:]
			chunk.MetaData.PathInSchema = inPath
		}
	}
}

func (self *ParquetReader) GetNumRows() int64 {
	return self.Footer.GetNumRows()
}

//Get the footer size
func (self *ParquetReader) GetFooterSize() (uint32, error) {
	var err error
	buf := make([]byte, 4)
	if _, err = self.PFile.Seek(-8, io.SeekEnd); err != nil {
		return 0, err
	}
	if _, err = self.PFile.Read(buf); err != nil {
		return 0, err
	}
	size := binary.LittleEndian.Uint32(buf)
	return size, err
}

//Read footer from parquet file
func (self *ParquetReader) ReadFooter() error {
	size, err := self.GetFooterSize()
	if err != nil {
		return err
	}
	if _, err = self.PFile.Seek(-(int64)(8+size), io.SeekEnd); err != nil {
		return err
	}
	self.Footer = parquet.NewFileMetaData()
	pf := thrift.NewTCompactProtocolFactory()
	protocol := pf.GetProtocol(thrift.NewStreamTransportR(self.PFile))
	return self.Footer.Read(protocol)
}

//Skip rows of parquet file
func (self *ParquetReader) SkipRows(num int64) error {
	var err error
	if num <= 0 {
		return nil
	}
	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.SchemaHandler.ValueColumns))
	stopChan := make(chan int)

	for _, pathStr := range self.SchemaHandler.ValueColumns {
		if _, ok := self.ColumnBuffers[pathStr]; !ok {
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					cb.SkipRows(int64(num))
					doneChan <- 0
				}
			}
		}()
	}

	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}

	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}
	return err
}

//Read rows of parquet file and unmarshal all to dst
func (self *ParquetReader) Read(dstInterface interface{}) error {
	return self.read(dstInterface, "")
}

//Read rows of parquet file and unmarshal all to dst
func (self *ParquetReader) ReadPartial(dstInterface interface{}, prefixPath string) error {
	prefixPath, err := self.SchemaHandler.ConvertToInPathStr(prefixPath)
	if err != nil {
		return err
	}

	return self.read(dstInterface, prefixPath)
}

//Read rows of parquet file
func (self *ParquetReader) read(dstInterface interface{}, prefixPath string) error {
	var err error
	tmap := make(map[string]*layout.Table)
	locker := new(sync.Mutex)
	ot := reflect.TypeOf(dstInterface).Elem().Elem()
	num := reflect.ValueOf(dstInterface).Elem().Len()
	if num <= 0 {
		return nil
	}

	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.ColumnBuffers))
	stopChan := make(chan int)

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					table, _ := cb.ReadRows(int64(num))
					locker.Lock()
					if _, ok := tmap[pathStr]; ok {
						tmap[pathStr].Merge(table)
					} else {
						tmap[pathStr] = layout.NewTableFromTable(table)
						tmap[pathStr].Merge(table)
					}
					locker.Unlock()
					doneChan <- 0
				}
			}
		}()
	}
	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}
	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}

	dstList := make([]interface{}, self.NP)
	delta := (int64(num) + self.NP - 1) / self.NP

	doneChan = make(chan int)
	for c := int64(0); c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > int64(num) {
			end = int64(num)
		}
		if bgn >= int64(num) {
			bgn, end = int64(num), int64(num)
		}
		go func(b, e, index int) {
			dstList[index] = reflect.New(reflect.SliceOf(ot)).Interface()
			if err2 := marshal.Unmarshal(&tmap, b, e, dstList[index], self.SchemaHandler, prefixPath); err2 != nil {
				err = err2
			}
			doneChan <- 0
		}(int(bgn), int(end), int(c))
	}
	for c := int64(0); c < self.NP; c++ {
		<-doneChan
	}

	resTmp := reflect.MakeSlice(reflect.SliceOf(ot), 0, num)
	for _, dst := range dstList {
		resTmp = reflect.AppendSlice(resTmp, reflect.ValueOf(dst).Elem())
	}
	reflect.ValueOf(dstInterface).Elem().Set(resTmp)
	return err
}

//Stop Read
func (self *ParquetReader) ReadStop() {
	for _, cb := range self.ColumnBuffers {
		if cb != nil {
			cb.PFile.Close()
		}
	}
}
<commit_msg>updating<commit_after>package reader

import (
	"encoding/binary"
	"io"
	"reflect"
	"sync"
	"strings"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/marshal"
	"github.com/xitongsys/parquet-go/source"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/parquet"
)

type ParquetReader struct {
	SchemaHandler *schema.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         source.ParquetFile

	ColumnBuffers map[string]*ColumnBufferType
}

//Create a parquet reader
func NewParquetReader(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetReader, error) {
	var err error
	res := new(ParquetReader)
	res.NP = np
	res.PFile = pFile
	if err = res.ReadFooter(); err != nil {
		return nil, err
	}
	res.ColumnBuffers = make(map[string]*ColumnBufferType)

	if obj != nil {
		if res.SchemaHandler, err = schema.NewSchemaHandlerFromStruct(obj); err != nil {
			return res, err
		}
		res.RenameSchema()

		for i := 0; i < len(res.SchemaHandler.SchemaElements); i++ {
			schema := res.SchemaHandler.SchemaElements[i]
			if schema.GetNumChildren() == 0 {
				pathStr := res.SchemaHandler.IndexMap[int32(i)]
				if res.ColumnBuffers[pathStr], err = NewColumnBuffer(pFile, res.Footer, res.SchemaHandler, pathStr); err != nil {
					return res, err
				}
			}
		}
	}
	return res, nil
}

func (self *ParquetReader) SetSchemaHandlerFromJSON(jsonSchema string) error {
	var err error
	if self.SchemaHandler, err = schema.NewSchemaHandlerFromJSON(jsonSchema); err != nil {
		return err
	}
	self.RenameSchema()
	for i := 0; i < len(self.SchemaHandler.SchemaElements); i++ {
		schemaElement := self.SchemaHandler.SchemaElements[i]
		if schemaElement.GetNumChildren() == 0 {
			pathStr := self.SchemaHandler.IndexMap[int32(i)]
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}
	return nil
}

//Rename schema name to inname
func (self *ParquetReader) RenameSchema() {
	for i := 0; i < len(self.SchemaHandler.Infos); i++ {
		self.Footer.Schema[i].Name = self.SchemaHandler.Infos[i].InName
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			exPath := make([]string, 0)
			exPath = append(exPath, self.SchemaHandler.GetRootName())
			exPath = append(exPath, chunk.MetaData.GetPathInSchema()...)
			exPathStr := common.PathToStr(exPath)

			inPathStr := self.SchemaHandler.ExPathToInPath[exPathStr]
			inPath := common.StrToPath(inPathStr)[1:]
			chunk.MetaData.PathInSchema = inPath
		}
	}
}

func (self *ParquetReader) GetNumRows() int64 {
	return self.Footer.GetNumRows()
}

//Get the footer size
func (self *ParquetReader) GetFooterSize() (uint32, error) {
	var err error
	buf := make([]byte, 4)
	if _, err = self.PFile.Seek(-8, io.SeekEnd); err != nil {
		return 0, err
	}
	if _, err = self.PFile.Read(buf); err != nil {
		return 0, err
	}
	size := binary.LittleEndian.Uint32(buf)
	return size, err
}

//Read footer from parquet file
func (self *ParquetReader) ReadFooter() error {
	size, err := self.GetFooterSize()
	if err != nil {
		return err
	}
	if _, err = self.PFile.Seek(-(int64)(8+size), io.SeekEnd); err != nil {
		return err
	}
	self.Footer = parquet.NewFileMetaData()
	pf := thrift.NewTCompactProtocolFactory()
	protocol := pf.GetProtocol(thrift.NewStreamTransportR(self.PFile))
	return self.Footer.Read(protocol)
}

//Skip rows of parquet file
func (self *ParquetReader) SkipRows(num int64) error {
	var err error
	if num <= 0 {
		return nil
	}
	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.SchemaHandler.ValueColumns))
	stopChan := make(chan int)

	for _, pathStr := range self.SchemaHandler.ValueColumns {
		if _, ok := self.ColumnBuffers[pathStr]; !ok {
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					cb.SkipRows(int64(num))
					doneChan <- 0
				}
			}
		}()
	}

	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}

	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}
	return err
}

//Read rows of parquet file and unmarshal all to dst
func (self *ParquetReader) Read(dstInterface interface{}) error {
	return self.read(dstInterface, "")
}

//Read rows of parquet file and unmarshal all to dst
func (self *ParquetReader) ReadPartial(dstInterface interface{}, prefixPath string) error {
	prefixPath, err := self.SchemaHandler.ConvertToInPathStr(prefixPath)
	if err != nil {
		return err
	}

	return self.read(dstInterface, prefixPath)
}

//Read rows of parquet file
func (self *ParquetReader) read(dstInterface interface{}, prefixPath string) error {
	var err error
	tmap := make(map[string]*layout.Table)
	locker := new(sync.Mutex)
	ot := reflect.TypeOf(dstInterface).Elem().Elem()
	num := reflect.ValueOf(dstInterface).Elem().Len()
	if num <= 0 {
		return nil
	}

	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.ColumnBuffers))
	stopChan := make(chan int)

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					table, _ := cb.ReadRows(int64(num))
					locker.Lock()
					if _, ok := tmap[pathStr]; ok {
						tmap[pathStr].Merge(table)
					} else {
						tmap[pathStr] = layout.NewTableFromTable(table)
						tmap[pathStr].Merge(table)
					}
					locker.Unlock()
					doneChan <- 0
				}
			}
		}()
	}

	readNum := 0
	for key, _ := range self.ColumnBuffers {
		if strings.HasPrefix(key, prefixPath) {
			taskChan <- key
			readNum++
		}
	}
	for i := 0; i < readNum; i++ {
		<-doneChan
	}

	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}

	dstList := make([]interface{}, self.NP)
	delta := (int64(num) + self.NP - 1) / self.NP

	doneChan = make(chan int)
	for c := int64(0); c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > int64(num) {
			end = int64(num)
		}
		if bgn >= int64(num) {
			bgn, end = int64(num), int64(num)
		}
		go func(b, e, index int) {
			dstList[index] = reflect.New(reflect.SliceOf(ot)).Interface()
			if err2 := marshal.Unmarshal(&tmap, b, e, dstList[index], self.SchemaHandler, prefixPath); err2 != nil {
				err = err2
			}
			doneChan <- 0
		}(int(bgn), int(end), int(c))
	}
	for c := int64(0); c < self.NP; c++ {
		<-doneChan
	}

	resTmp := reflect.MakeSlice(reflect.SliceOf(ot), 0, num)
	for _, dst := range dstList {
		resTmp = reflect.AppendSlice(resTmp, reflect.ValueOf(dst).Elem())
	}
	reflect.ValueOf(dstInterface).Elem().Set(resTmp)
	return err
}

//Stop Read
func (self *ParquetReader) ReadStop() {
	for _, cb := range self.ColumnBuffers {
		if cb != nil {
			cb.PFile.Close()
		}
	}
}
<|endoftext|>


<commit_before>package notificationcenter

import (
	"context"
	"reflect"

	"github.com/pkg/errors"

	"github.com/geniusrabbit/notificationcenter/v2/decoder"
)

var errInvalidReturnType = errors.New("invalid return types")

// ReceiverFrom converts income handler type to Receiver interface
func ReceiverFrom(handler any) Receiver {
	switch h := handler.(type) {
	case Receiver:
		return h
	case func() error:
		return FuncReceiver(func(msg Message) error { h(); return msg.Ack() })
	case func(msg Message) error:
		return FuncReceiver(h)
	case func(ctx context.Context, msg Message) error:
		return FuncReceiver(func(msg Message) error { return h(msg.Context(), msg) })
	default:
		return ExtFuncReceiver(h)
	}
}

var (
	errorType   = reflect.TypeOf((*error)(nil)).Elem()
	contextType = reflect.TypeOf((*context.Context)(nil)).Elem()
	msgType     = reflect.TypeOf((*Message)(nil)).Elem()
)

// ExtFuncReceiver wraps function argument with arbitrary input data type
func ExtFuncReceiver(f any, decs ...decoder.Decoder) Receiver {
	fv := reflect.ValueOf(f)
	if fv.Kind() != reflect.Func {
		panic("argument must be a function")
	}
	dec := decoder.JSON
	if len(decs) > 0 && decs[0] != nil {
		dec = decs[0]
	}
	var (
		ft        = fv.Type()
		argMapper = make([]func(Message) (reflect.Value, error), 0, ft.NumIn())
		retMapper = make([]func(reflect.Value) error, 0, ft.NumOut())
	)
	for i := 0; i < ft.NumIn(); i++ {
		inType := ft.In(i)
		switch inType {
		case contextType:
			argMapper = append(argMapper, func(msg Message) (reflect.Value, error) {
				return reflect.ValueOf(msg.Context()), nil
			})
		case msgType:
			argMapper = append(argMapper, func(msg Message) (reflect.Value, error) {
				return reflect.ValueOf(msg), nil
			})
		default:
			argMapper = append(argMapper, func(msg Message) (reflect.Value, error) {
				newValue, newValueI := newValue(inType)
				err := dec(msg.Body(), newValueI)
				return newValue, err
			})
		}
	}
	for i := 0; i < ft.NumOut(); i++ {
		outType := ft.Out(i)
		switch outType {
		case errorType:
			retMapper = append(retMapper, func(v reflect.Value) error {
				if v.IsNil() {
					return nil
				}
				return v.Interface().(error)
			})
		default:
			panic(errInvalidReturnType)
		}
	}
	return FuncReceiver(func(msg Message) error {
		args := make([]reflect.Value, 0, len(argMapper))
		for _, fm := range argMapper {
			arg, err := fm(msg)
			if err != nil {
				return err
			}
			args = append(args, arg)
		}
		retVals := fv.Call(args)
		for i, fr := range retMapper {
			if err := fr(retVals[i]); err != nil {
				return err
			}
		}
		return nil
	})
}

func newValue(t reflect.Type) (reflect.Value, any) {
	if t.Kind() == reflect.Ptr {
		return newValue(t.Elem())
	}
	v := reflect.New(t)
	i := v.Interface()
	return v, i
}
<commit_msg>Fix lint errors<commit_after>package notificationcenter

import (
	"context"
	"reflect"

	"github.com/pkg/errors"

	"github.com/geniusrabbit/notificationcenter/v2/decoder"
)

var errInvalidReturnType = errors.New("invalid return types")

// ReceiverFrom converts income handler type to Receiver interface
func ReceiverFrom(handler any) Receiver {
	switch h := handler.(type) {
	case Receiver:
		return h
	case func() error:
		return FuncReceiver(func(msg Message) error {
			if err := h(); err != nil {
				return err
			}
			return msg.Ack()
		})
	case func(msg Message) error:
		return FuncReceiver(h)
	case func(ctx context.Context, msg Message) error:
		return FuncReceiver(func(msg Message) error { return h(msg.Context(), msg) })
	default:
		return ExtFuncReceiver(h)
	}
}

var (
	errorType   = reflect.TypeOf((*error)(nil)).Elem()
	contextType = reflect.TypeOf((*context.Context)(nil)).Elem()
	msgType     = reflect.TypeOf((*Message)(nil)).Elem()
)

// ExtFuncReceiver wraps function argument with arbitrary input data type
func ExtFuncReceiver(f any, decs ...decoder.Decoder) Receiver {
	fv := reflect.ValueOf(f)
	if fv.Kind() != reflect.Func {
		panic("argument must be a function")
	}
	dec := decoder.JSON
	if len(decs) > 0 && decs[0] != nil {
		dec = decs[0]
	}
	var (
		ft        = fv.Type()
		argMapper = make([]func(Message) (reflect.Value, error), 0, ft.NumIn())
		retMapper = make([]func(reflect.Value) error, 0, ft.NumOut())
	)
	for i := 0; i < ft.NumIn(); i++ {
		inType := ft.In(i)
		switch inType {
		case contextType:
			argMapper = append(argMapper, func(msg Message) (reflect.Value, error) {
				return reflect.ValueOf(msg.Context()), nil
			})
		case msgType:
			argMapper = append(argMapper, func(msg Message) (reflect.Value, error) {
				return reflect.ValueOf(msg), nil
			})
		default:
			argMapper = append(argMapper, func(msg Message) (reflect.Value, error) {
				newValue, newValueI := newValue(inType)
				err := dec(msg.Body(), newValueI)
				return newValue, err
			})
		}
	}
	for i := 0; i < ft.NumOut(); i++ {
		outType := ft.Out(i)
		switch outType {
		case errorType:
			retMapper = append(retMapper, func(v reflect.Value) error {
				if v.IsNil() {
					return nil
				}
				return v.Interface().(error)
			})
		default:
			panic(errInvalidReturnType)
		}
	}
	return FuncReceiver(func(msg Message) error {
		args := make([]reflect.Value, 0, len(argMapper))
		for _, fm := range argMapper {
			arg, err := fm(msg)
			if err != nil {
				return err
			}
			args = append(args, arg)
		}
		retVals := fv.Call(args)
		for i, fr := range retMapper {
			if err := fr(retVals[i]); err != nil {
				return err
			}
		}
		return nil
	})
}

func newValue(t reflect.Type) (reflect.Value, any) {
	if t.Kind() == reflect.Ptr {
		return newValue(t.Elem())
	}
	v := reflect.New(t)
	i := v.Interface()
	return v, i
}
<|endoftext|>


<commit_before>package refctx

import (
	"context"
	"sync/atomic"
)

// RefCtr cancels a context when no references are held
type RefCtr struct {
	cancel context.CancelFunc
	refcnt int32
}

// Incr increments the refcount
func (r *RefCtr) Incr() { r.Add(1) }

// Add i refcounts
func (r *RefCtr) Add(i int32) {
	if v := atomic.AddInt32(&r.refcnt, i); v <= 0 {
		r.cancel()
	}
}

// Decr decrements the refcount
func (r *RefCtr) Decr() { r.Add(-1) }

// WithRefCount derives a context that will be cancelled when all references are
// freed.
func WithRefCount(c context.Context) (context.Context, *RefCtr) {
	c, cancel := context.WithCancel(c)
	return c, &RefCtr{cancel: cancel}
}
<commit_msg>Use uint32 for Refctr WithRefCount now accepts a ctx.Doner Add ContextWithRefcount, which accepts a context.Context<commit_after>package refctx

import (
	"context"
	"sync/atomic"

	"github.com/SentimensRG/ctx"
)

// RefCtr cancels a context when no references are held
type RefCtr struct {
	cancel func()
	refcnt uint32
}

// Incr increments the refcount
func (r *RefCtr) Incr() { r.Add(1) }

// Add i refcounts
func (r *RefCtr) Add(i uint32) {
	if v := atomic.AddUint32(&r.refcnt, i); v == 0 {
		r.cancel()
	}
}

// Decr decrements the refcount
func (r *RefCtr) Decr() { atomic.AddUint32(&r.refcnt, ^uint32(0)) }

// WithRefCount derives a ctx.C that will be cancelled when all references are
// freed
func WithRefCount(d ctx.Doner) (ctx.C, *RefCtr) {
	ch, cancel := ctx.WithCancel(d)
	return ch, &RefCtr{cancel: cancel}
}

// ContextWithRefCount derives a context that will be cancelled when all
// references are freed.
func ContextWithRefCount(c context.Context) (context.Context, *RefCtr) {
	c, cancel := context.WithCancel(c)
	return c, &RefCtr{cancel: cancel}
}
<|endoftext|>


<commit_before>package checksum_generator

import (
	"crypto/sha1"
	"encoding/hex"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

type ChecksumRecord struct {
	Checksum string    `json:"checksum"`
	ModTime  time.Time `json:"mod_time"`
}

type DirectoryManifest struct {
	Path      string                    `json:"path"`
	CreatedAt time.Time                 `json:"created_at"`
	Entries   map[string]ChecksumRecord `json:"entries"`
}

func FileChecksum(file string) ChecksumRecord {
	fi, err := os.Stat(file)
	check(err)

	sum := generateChecksum(file)
	return ChecksumRecord{
		Checksum: sum,
		ModTime:  fi.ModTime(),
	}
}

func GenerateDirectoryManifest(path string) DirectoryManifest {
	return DirectoryManifest{
		Path:      path,
		CreatedAt: time.Now(),
		Entries:   directoryChecksums(path),
	}
}

// Private functions

func generateChecksum(file string) string {
	data, err := ioutil.ReadFile(file)
	check(err)

	sum := sha1.Sum(data)
	return hex.EncodeToString(sum[:])
}

func directoryChecksums(path string) map[string]ChecksumRecord {
	records := map[string]ChecksumRecord{}
	filepath.Walk(path, func(entryPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() {
			var relPath string
			relPath, err = filepath.Rel(path, entryPath)
			check(err)
			records[relPath] = FileChecksum(entryPath)
		}

		return nil
	})
	return records
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<commit_msg>Use mod time from filepath walker<commit_after>package checksum_generator

import (
	"crypto/sha1"
	"encoding/hex"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

type ChecksumRecord struct {
	Checksum string    `json:"checksum"`
	ModTime  time.Time `json:"mod_time"`
}

type DirectoryManifest struct {
	Path      string                    `json:"path"`
	CreatedAt time.Time                 `json:"created_at"`
	Entries   map[string]ChecksumRecord `json:"entries"`
}

func FileChecksum(file string) ChecksumRecord {
	fi, err := os.Stat(file)
	check(err)

	sum := generateChecksum(file)
	return ChecksumRecord{
		Checksum: sum,
		ModTime:  fi.ModTime(),
	}
}

func GenerateDirectoryManifest(path string) DirectoryManifest {
	return DirectoryManifest{
		Path:      path,
		CreatedAt: time.Now(),
		Entries:   directoryChecksums(path),
	}
}

// Private functions

func generateChecksum(file string) string {
	data, err := ioutil.ReadFile(file)
	check(err)

	sum := sha1.Sum(data)
	return hex.EncodeToString(sum[:])
}

func directoryChecksums(path string) map[string]ChecksumRecord {
	records := map[string]ChecksumRecord{}
	filepath.Walk(path, func(entryPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() {
			var relPath string
			relPath, err = filepath.Rel(path, entryPath)
			check(err)
			records[relPath] = ChecksumRecord{
				Checksum: generateChecksum(entryPath),
				ModTime:  info.ModTime(),
			}
		}

		return nil
	})
	return records
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<|endoftext|>