<commit_before>package main

// copyright(c) 2014, Jason E. Aten
//
// goq : a simple queueing system in go; qsub replacement.
//

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"time"
)

// pass by value to avoid races
func (w *Worker) Shepard(jobPtr *Job) {

	// to avoid data races, make a copy of the job value
	j := *jobPtr

	// reset our input channel
	w.DrainTellShepPidKilled()

	go func() {
		myPid := 0

		defer func() {
			// if myPid set then ShepSaysJobStarted already signalled.
			// otherwise indicate error:
			if myPid == 0 {
				w.ShepSaysJobStarted <- 0
			}
			WPrintf("end of SHEP: just before w.ShepSaysJobDone <- j\n")
			w.ShepSaysJobDone <- &j
			WPrintf("end of SHEP: just after w.ShepSaysJobDone <- j\n")
		}()

		jid := j.Id
		dir := j.Dir
		cmd := j.Cmd
		args := j.Args
		env := j.Env
		if j.Out == nil {
			j.Out = make([]string, 0)
		}
		var origdir string
		var err error
		origdir, err = os.Getwd()
		if err != nil {
			panic(err)
		}

		if dir != "" {
			err = os.Chdir(dir)
			if err != nil {
				j.Out = append(j.Out, fmt.Sprintf("Shepard got error trying to move to submit directory with os.Chdir('%s'): %s", dir, err))
				return
			}
			// go back to our starting dir at the end of shepding this job.
			defer os.Chdir(origdir)
		}

		c := exec.Command(cmd, args...)
		c.Dir = dir
		c.Env = env

		var oe bytes.Buffer
		c.Stdout = &oe
		c.Stderr = &oe

		//	oe, err = c.CombinedOutput()
		err = c.Start()
		if err != nil {
			j.Out = append(j.Out, fmt.Sprintf("Shepard finds non-nil err on trying to Start() cmd '%s' in dir '%s': %s", cmd, dir, err))
			return
		}

		// no error/return should be possible between
		// setting myPid and w.ShepSaysJobStarted <- myPid
		myPid = c.Process.Pid
		j.Pid = int64(myPid)
		VPrintf("\n SHEP Shepard goroutine about to block on w.ShepSaysJobStarted <- j\n")
		w.ShepSaysJobStarted <- myPid
		VPrintf("\n SHEP Shepard goroutine about to block on c.Wait()\n")

		// this c.Wait() can be 15-20 seconds *slooooow*, so also wait on TellShepPidKilled
		//  to speed things up.
		err = nil
		// waitDone is buffered so this next short goro can exit immediately after c.Wait() finishes.
		// i.e. if TellShepPidKilled arrives first, then there will never be a receiver, so
		// without the buffering the channel and goroutine would be blocked, uncollectable, waiting-forever garbage/leak.
		waitDone := make(chan error, 1)
		go func() {
			err = c.Wait()
			waitDone <- err
		}()

		// back in shep goroutine:
		select {
		case err = <-waitDone:
		case killedPid := <-w.TellShepPidKilled:
			if killedPid != myPid {
				panic(fmt.Sprintf("SHEP error: mismatch in myPid(%d) vs killedPid(%d) received on w.TellShepPidKilled", myPid, killedPid))
			}
			j.Cancelled = true
			WPrintf("\n SHEP got notice from w.TellShepPidKilled, setting j.Cancelled = true\n")
		}

		// Now set j.Out based on which of the two cases we just saw:
		//  Either we saw w.TellShepPidKilled, in which case j.Cancelled == true and we want to exit quickly.
		//  Otherwise, we had a normal or fast c.Wait() exit, and we want to gather and send output on j.Out.
		//
		if j.Cancelled {
			j.Out = append(j.Out, fmt.Sprintf("cancelled/killed: job %d / pid %d ; cmd '%s' in dir '%s' on worker '%s' at '%s'", jid, myPid, cmd, dir, j.Workeraddr, time.Now()))

			// Don't wait around for output/etc.
			// Just skip down to ShepSaysJobDone and get out of here fast.

		} else {
			// Normal/Fast c.Wait() exit:
			WPrintf("\n SHEP DONE with WAIT, err: '%s'\n", err)
			if err != nil && err.Error() == "signal: killed" {
				WPrintf("\n SHEP found 'signal:killed', setting j.Cancelled = true\n")
				j.Cancelled = true
			}
			s := string(oe.Bytes())
			strings.Trim(s, "\n")
			slen := len(s)
			out := strings.Split(s, "\n")
			// if file ended in '\n' then we now have an extra empty line to eliminate.
			N := len(out)
			if slen > 0 && s[slen-1] == '\n' && N > 0 && out[N-1] == "" {
				out = out[:N-1]
			}
			j.Out = append(j.Out, out...)

			if err != nil {
				j.Out = append(j.Out, fmt.Sprintf("Shepard finds non-nil err on trying to Wait() on cmd '%s' in dir '%s': %s", cmd, dir, err))
			}
		}
	}()
}
<commit_msg>shepard uses local env, ignores job sub env<commit_after>package main

// copyright(c) 2014, Jason E. Aten
//
// goq : a simple queueing system in go; qsub replacement.
//

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"time"
)

// pass by value to avoid races
func (w *Worker) Shepard(jobPtr *Job) {

	// to avoid data races, make a copy of the job value
	j := *jobPtr

	// reset our input channel
	w.DrainTellShepPidKilled()

	go func() {
		myPid := 0

		defer func() {
			// if myPid set then ShepSaysJobStarted already signalled.
			// otherwise indicate error:
			if myPid == 0 {
				w.ShepSaysJobStarted <- 0
			}
			WPrintf("end of SHEP: just before w.ShepSaysJobDone <- j\n")
			w.ShepSaysJobDone <- &j
			WPrintf("end of SHEP: just after w.ShepSaysJobDone <- j\n")
		}()

		jid := j.Id
		dir := j.Dir
		cmd := j.Cmd
		args := j.Args
		env := CreateShepardedEnv(j.Env)
		if j.Out == nil {
			j.Out = make([]string, 0)
		}
		var origdir string
		var err error
		origdir, err = os.Getwd()
		if err != nil {
			panic(err)
		}

		if dir != "" {
			err = os.Chdir(dir)
			if err != nil {
				j.Out = append(j.Out, fmt.Sprintf("Shepard got error trying to move to submit directory with os.Chdir('%s'): %s", dir, err))
				return
			}
			// go back to our starting dir at the end of shepding this job.
			defer os.Chdir(origdir)
		}

		c := exec.Command(cmd, args...)
		c.Dir = dir
		c.Env = env

		var oe bytes.Buffer
		c.Stdout = &oe
		c.Stderr = &oe

		//	oe, err = c.CombinedOutput()
		err = c.Start()
		if err != nil {
			j.Out = append(j.Out, fmt.Sprintf("Shepard finds non-nil err on trying to Start() cmd '%s' in dir '%s': %s", cmd, dir, err))
			return
		}

		// no error/return should be possible between
		// setting myPid and w.ShepSaysJobStarted <- myPid
		myPid = c.Process.Pid
		j.Pid = int64(myPid)
		VPrintf("\n SHEP Shepard goroutine about to block on w.ShepSaysJobStarted <- j\n")
		w.ShepSaysJobStarted <- myPid
		VPrintf("\n SHEP Shepard goroutine about to block on c.Wait()\n")

		// this c.Wait() can be 15-20 seconds *slooooow*, so also wait on TellShepPidKilled
		//  to speed things up.
		err = nil
		// waitDone is buffered so this next short goro can exit immediately after c.Wait() finishes.
		// i.e. if TellShepPidKilled arrives first, then there will never be a receiver, so
		// without the buffering the channel and goroutine would be blocked, uncollectable, waiting-forever garbage/leak.
		waitDone := make(chan error, 1)
		go func() {
			err = c.Wait()
			waitDone <- err
		}()

		// back in shep goroutine:
		select {
		case err = <-waitDone:
		case killedPid := <-w.TellShepPidKilled:
			if killedPid != myPid {
				panic(fmt.Sprintf("SHEP error: mismatch in myPid(%d) vs killedPid(%d) received on w.TellShepPidKilled", myPid, killedPid))
			}
			j.Cancelled = true
			WPrintf("\n SHEP got notice from w.TellShepPidKilled, setting j.Cancelled = true\n")
		}

		// Now set j.Out based on which of the two cases we just saw:
		//  Either we saw w.TellShepPidKilled, in which case j.Cancelled == true and we want to exit quickly.
		//  Otherwise, we had a normal or fast c.Wait() exit, and we want to gather and send output on j.Out.
		//
		if j.Cancelled {
			j.Out = append(j.Out, fmt.Sprintf("cancelled/killed: job %d / pid %d ; cmd '%s' in dir '%s' on worker '%s' at '%s'", jid, myPid, cmd, dir, j.Workeraddr, time.Now()))

			// Don't wait around for output/etc.
			// Just skip down to ShepSaysJobDone and get out of here fast.

		} else {
			// Normal/Fast c.Wait() exit:
			WPrintf("\n SHEP DONE with WAIT, err: '%s'\n", err)
			if err != nil && err.Error() == "signal: killed" {
				WPrintf("\n SHEP found 'signal:killed', setting j.Cancelled = true\n")
				j.Cancelled = true
			}
			s := string(oe.Bytes())
			strings.Trim(s, "\n")
			slen := len(s)
			out := strings.Split(s, "\n")
			// if file ended in '\n' then we now have an extra empty line to eliminate.
			N := len(out)
			if slen > 0 && s[slen-1] == '\n' && N > 0 && out[N-1] == "" {
				out = out[:N-1]
			}
			j.Out = append(j.Out, out...)

			if err != nil {
				j.Out = append(j.Out, fmt.Sprintf("Shepard finds non-nil err on trying to Wait() on cmd '%s' in dir '%s': %s", cmd, dir, err))
			}
		}
	}()
}

func CreateShepardedEnv(jobenv []string) []string {
	// for now, ignore job sub-env and just use local worker env.
	return os.Environ()
}
<|endoftext|>


<commit_before>// Copyright 2017 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//

package main

import (
	"errors"
	"io"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/moby/moby/pkg/term"
	context "golang.org/x/net/context"

	pb "github.com/kata-containers/agent/protocols/grpc"
)

const sigChanSize = 2048

var sigIgnored = map[syscall.Signal]bool{
	syscall.SIGCHLD:  true,
	syscall.SIGPIPE:  true,
	syscall.SIGWINCH: true,
}

type shim struct {
	containerID string
	execID      string

	ctx   context.Context
	agent *shimAgent
}

func newShim(addr, containerID, execID string) (*shim, error) {
	agent, err := newShimAgent(addr)
	if err != nil {
		return nil, err
	}

	return &shim{containerID: containerID,
		execID: execID,
		ctx:    context.Background(),
		agent:  agent}, nil
}

func (s *shim) proxyStdio(wg *sync.WaitGroup, terminal bool) {
	// don't wait the copying of the stdin, because `io.Copy(inPipe, os.Stdin)`
	// can't terminate when no input. todo: find a better way.
	wg.Add(1)
	if !terminal {
		// In case it's not a terminal, we also need to get the output
		// from stderr.
		wg.Add(1)
	}

	inPipe, outPipe, errPipe := shimStdioPipe(s.ctx, s.agent, s.containerID, s.execID)
	go func() {
		_, err1 := io.Copy(inPipe, os.Stdin)
		_, err2 := s.agent.CloseStdin(s.ctx, &pb.CloseStdinRequest{
			ContainerId: s.containerID,
			ExecId:      s.execID})
		if err1 != nil {
			logger().WithError(err1).Warn("copy stdin failed")
		}
		if err2 != nil {
			logger().WithError(err2).Warn("close stdin failed")
		}
	}()

	go func() {
		_, err := io.Copy(os.Stdout, outPipe)
		logger().WithError(err).Info("copy stdout failed")
		wg.Done()
	}()

	if !terminal {
		go func() {
			_, err := io.Copy(os.Stderr, errPipe)
			logger().WithError(err).Info("copy stderr failed")
			wg.Done()
		}()
	}
}

func (s *shim) forwardAllSignals() chan os.Signal {
	sigc := make(chan os.Signal, sigChanSize)
	// handle all signals for the process.
	signal.Notify(sigc)
	signal.Ignore(syscall.SIGCHLD, syscall.SIGPIPE)

	go func() {
		for sig := range sigc {
			sysSig, ok := sig.(syscall.Signal)
			if !ok {
				err := errors.New("unknown signal")
				logger().WithError(err).WithField("signal", sig.String()).Error("")
				continue
			}

			if fatalSignal(sysSig) {
				logger().WithField("signal", sig).Error("received fatal signal")
				die()
			}

			if sigIgnored[sysSig] {
				//ignore these
				continue
			}
			// forward this signal to container
			_, err := s.agent.SignalProcess(s.ctx, &pb.SignalProcessRequest{
				ContainerId: s.containerID,
				ExecId:      s.execID,
				Signal:      uint32(sysSig)})
			if err != nil {
				logger().WithError(err).WithField("signal", sig.String()).Error("forward signal failed")
			}
		}
	}()
	return sigc
}

func (s *shim) resizeTty(fromTty *os.File) error {
	ws, err := term.GetWinsize(fromTty.Fd())
	if err != nil {
		logger().WithError(err).Info("Error getting window size")
		return nil
	}

	_, err = s.agent.TtyWinResize(s.ctx, &pb.TtyWinResizeRequest{
		ContainerId: s.containerID,
		ExecId:      s.execID,
		Row:         uint32(ws.Height),
		Column:      uint32(ws.Width)})
	if err != nil {
		logger().WithError(err).Error("set window size failed")
	}

	return err
}

func (s *shim) monitorTtySize(tty *os.File) {
	s.resizeTty(tty)
	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, syscall.SIGWINCH)
	go func() {
		for range sigchan {
			s.resizeTty(tty)
		}
	}()
}

func (s *shim) wait() (int32, error) {
	resp, err := s.agent.WaitProcess(s.ctx, &pb.WaitProcessRequest{
		ContainerId: s.containerID,
		ExecId:      s.execID})
	if err != nil {
		return 0, err
	}

	return resp.Status, nil
}
<commit_msg>logging: Log fd on window resize fail<commit_after>// Copyright 2017 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//

package main

import (
	"errors"
	"io"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/moby/moby/pkg/term"
	context "golang.org/x/net/context"

	pb "github.com/kata-containers/agent/protocols/grpc"
)

const sigChanSize = 2048

var sigIgnored = map[syscall.Signal]bool{
	syscall.SIGCHLD:  true,
	syscall.SIGPIPE:  true,
	syscall.SIGWINCH: true,
}

type shim struct {
	containerID string
	execID      string

	ctx   context.Context
	agent *shimAgent
}

func newShim(addr, containerID, execID string) (*shim, error) {
	agent, err := newShimAgent(addr)
	if err != nil {
		return nil, err
	}

	return &shim{containerID: containerID,
		execID: execID,
		ctx:    context.Background(),
		agent:  agent}, nil
}

func (s *shim) proxyStdio(wg *sync.WaitGroup, terminal bool) {
	// don't wait the copying of the stdin, because `io.Copy(inPipe, os.Stdin)`
	// can't terminate when no input. todo: find a better way.
	wg.Add(1)
	if !terminal {
		// In case it's not a terminal, we also need to get the output
		// from stderr.
		wg.Add(1)
	}

	inPipe, outPipe, errPipe := shimStdioPipe(s.ctx, s.agent, s.containerID, s.execID)
	go func() {
		_, err1 := io.Copy(inPipe, os.Stdin)
		_, err2 := s.agent.CloseStdin(s.ctx, &pb.CloseStdinRequest{
			ContainerId: s.containerID,
			ExecId:      s.execID})
		if err1 != nil {
			logger().WithError(err1).Warn("copy stdin failed")
		}
		if err2 != nil {
			logger().WithError(err2).Warn("close stdin failed")
		}
	}()

	go func() {
		_, err := io.Copy(os.Stdout, outPipe)
		logger().WithError(err).Info("copy stdout failed")
		wg.Done()
	}()

	if !terminal {
		go func() {
			_, err := io.Copy(os.Stderr, errPipe)
			logger().WithError(err).Info("copy stderr failed")
			wg.Done()
		}()
	}
}

func (s *shim) forwardAllSignals() chan os.Signal {
	sigc := make(chan os.Signal, sigChanSize)
	// handle all signals for the process.
	signal.Notify(sigc)
	signal.Ignore(syscall.SIGCHLD, syscall.SIGPIPE)

	go func() {
		for sig := range sigc {
			sysSig, ok := sig.(syscall.Signal)
			if !ok {
				err := errors.New("unknown signal")
				logger().WithError(err).WithField("signal", sig.String()).Error("")
				continue
			}

			if fatalSignal(sysSig) {
				logger().WithField("signal", sig).Error("received fatal signal")
				die()
			}

			if sigIgnored[sysSig] {
				//ignore these
				continue
			}
			// forward this signal to container
			_, err := s.agent.SignalProcess(s.ctx, &pb.SignalProcessRequest{
				ContainerId: s.containerID,
				ExecId:      s.execID,
				Signal:      uint32(sysSig)})
			if err != nil {
				logger().WithError(err).WithField("signal", sig.String()).Error("forward signal failed")
			}
		}
	}()
	return sigc
}

func (s *shim) resizeTty(fromTty *os.File) error {
	fd := fromTty.Fd()

	ws, err := term.GetWinsize(fd)
	if err != nil {
		logger().WithError(err).WithField("fd", fd).Info("Error getting window size")
		return nil
	}

	_, err = s.agent.TtyWinResize(s.ctx, &pb.TtyWinResizeRequest{
		ContainerId: s.containerID,
		ExecId:      s.execID,
		Row:         uint32(ws.Height),
		Column:      uint32(ws.Width)})
	if err != nil {
		logger().WithError(err).Error("set window size failed")
	}

	return err
}

func (s *shim) monitorTtySize(tty *os.File) {
	s.resizeTty(tty)
	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, syscall.SIGWINCH)
	go func() {
		for range sigchan {
			s.resizeTty(tty)
		}
	}()
}

func (s *shim) wait() (int32, error) {
	resp, err := s.agent.WaitProcess(s.ctx, &pb.WaitProcessRequest{
		ContainerId: s.containerID,
		ExecId:      s.execID})
	if err != nil {
		return 0, err
	}

	return resp.Status, nil
}
<|endoftext|>


<commit_before>package trap

import (
	"bufio"
	"errors"
	"fmt"
	"go/types"
	"io/ioutil"
	"os"
	"path/filepath"
	"unicode/utf8"

	"github.com/github/codeql-go/extractor/srcarchive"
	"golang.org/x/tools/go/packages"
)

// A Writer provides methods for writing data to a TRAP file
type Writer struct {
	w            *bufio.Writer
	file         *os.File
	Labeler      *Labeler
	path         string
	trapFilePath string
	Package      *packages.Package
}

// NewWriter creates a TRAP file for the given path and returns a writer for
// writing to it
func NewWriter(path string, pkg *packages.Package) (*Writer, error) {
	trapFolder, err := trapFolder()
	if err != nil {
		return nil, err
	}
	trapFilePath := filepath.Join(trapFolder, srcarchive.AppendablePath(path)+".trap")
	trapFileDir := filepath.Dir(trapFilePath)
	err = os.MkdirAll(trapFileDir, 0755)
	if err != nil {
		return nil, err
	}
	tmpFile, err := ioutil.TempFile(trapFileDir, filepath.Base(trapFilePath))
	if err != nil {
		return nil, err
	}
	tw := &Writer{
		bufio.NewWriter(tmpFile),
		tmpFile,
		nil,
		path,
		trapFilePath,
		pkg,
	}
	tw.Labeler = newLabeler(tw)
	return tw, nil
}

func trapFolder() (string, error) {
	trapFolder := os.Getenv("CODEQL_EXTRACTOR_GO_TRAP_DIR")
	if trapFolder == "" {
		trapFolder = os.Getenv("TRAP_FOLDER")
	}
	if trapFolder == "" {
		return "", errors.New("environment variable CODEQL_EXTRACTOR_GO_TRAP_DIR not set")
	}
	err := os.MkdirAll(trapFolder, 0755)
	if err != nil {
		return "", err
	}
	return trapFolder, nil
}

// Close the underlying file writer
func (tw *Writer) Close() error {
	err := tw.w.Flush()
	if err != nil {
		// throw away close error because write errors are likely to be more important
		tw.file.Close()
		return err
	}
	err = tw.file.Close()
	if err != nil {
		return err
	}
	return os.Rename(tw.file.Name(), tw.trapFilePath)
}

// ForEachObject iterates over all objects labeled by this labeler, and invokes
// the provided callback with a writer for the trap file, the object, and its
// label.
func (tw *Writer) ForEachObject(cb func(*Writer, types.Object, Label)) {
	for object, lbl := range tw.Labeler.objectLabels {
		cb(tw, object, lbl)
	}
}

const max_strlen = 1024 * 1024

func capStringLength(s string) string {
	// if the UTF8-encoded string is longer than 1MiB, we truncate it
	if len(s) > max_strlen {
		// to ensure that the truncated string is valid UTF-8, we find the last byte at or
		// before index max_strlen that starts a UTF-8 encoded character, and then cut off
		// right before that byte
		end := max_strlen
		for ; !utf8.RuneStart(s[end]); end-- {
		}
		return s[0:end]
	}
	return s
}

// Emit writes out a tuple of values for the given `table`
func (tw *Writer) Emit(table string, values []interface{}) error {
	fmt.Fprintf(tw.w, "%s(", table)
	for i, value := range values {
		if i > 0 {
			fmt.Fprint(tw.w, ", ")
		}
		switch value := value.(type) {
		case Label:
			fmt.Fprint(tw.w, value.id)
		case string:
			fmt.Fprintf(tw.w, "\"%s\"", escapeString(capStringLength(value)))
		case int:
			fmt.Fprintf(tw.w, "%d", value)
		default:
			return errors.New("Cannot emit value")
		}
	}
	fmt.Fprintf(tw.w, ")\n")
	return nil
}
<commit_msg>Gzip TRAP files.<commit_after>package trap

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"go/types"
	"io/ioutil"
	"os"
	"path/filepath"
	"unicode/utf8"

	"github.com/github/codeql-go/extractor/srcarchive"
	"golang.org/x/tools/go/packages"
)

// A Writer provides methods for writing data to a TRAP file
type Writer struct {
	w            *bufio.Writer
	zip          *gzip.Writer
	file         *os.File
	Labeler      *Labeler
	path         string
	trapFilePath string
	Package      *packages.Package
}

// NewWriter creates a TRAP file for the given path and returns a writer for
// writing to it
func NewWriter(path string, pkg *packages.Package) (*Writer, error) {
	trapFolder, err := trapFolder()
	if err != nil {
		return nil, err
	}
	trapFilePath := filepath.Join(trapFolder, srcarchive.AppendablePath(path)+".trap.gz")
	trapFileDir := filepath.Dir(trapFilePath)
	err = os.MkdirAll(trapFileDir, 0755)
	if err != nil {
		return nil, err
	}
	tmpFile, err := ioutil.TempFile(trapFileDir, filepath.Base(trapFilePath))
	if err != nil {
		return nil, err
	}
	zipWriter := gzip.NewWriter(tmpFile)
	tw := &Writer{
		bufio.NewWriter(zipWriter),
		zipWriter,
		tmpFile,
		nil,
		path,
		trapFilePath,
		pkg,
	}
	tw.Labeler = newLabeler(tw)
	return tw, nil
}

func trapFolder() (string, error) {
	trapFolder := os.Getenv("CODEQL_EXTRACTOR_GO_TRAP_DIR")
	if trapFolder == "" {
		trapFolder = os.Getenv("TRAP_FOLDER")
	}
	if trapFolder == "" {
		return "", errors.New("environment variable CODEQL_EXTRACTOR_GO_TRAP_DIR not set")
	}
	err := os.MkdirAll(trapFolder, 0755)
	if err != nil {
		return "", err
	}
	return trapFolder, nil
}

// Close the underlying file writer
func (tw *Writer) Close() error {
	err := tw.w.Flush()
	if err != nil {
		// throw away close error because write errors are likely to be more important
		tw.file.Close()
		return err
	}
	err = tw.zip.Close()
	if err != nil {
		// return zip-close error, but ignore file-close error
		tw.file.Close()
		return err
	}
	err = tw.file.Close()
	if err != nil {
		return err
	}
	return os.Rename(tw.file.Name(), tw.trapFilePath)
}

// ForEachObject iterates over all objects labeled by this labeler, and invokes
// the provided callback with a writer for the trap file, the object, and its
// label.
func (tw *Writer) ForEachObject(cb func(*Writer, types.Object, Label)) {
	for object, lbl := range tw.Labeler.objectLabels {
		cb(tw, object, lbl)
	}
}

const max_strlen = 1024 * 1024

func capStringLength(s string) string {
	// if the UTF8-encoded string is longer than 1MiB, we truncate it
	if len(s) > max_strlen {
		// to ensure that the truncated string is valid UTF-8, we find the last byte at or
		// before index max_strlen that starts a UTF-8 encoded character, and then cut off
		// right before that byte
		end := max_strlen
		for ; !utf8.RuneStart(s[end]); end-- {
		}
		return s[0:end]
	}
	return s
}

// Emit writes out a tuple of values for the given `table`
func (tw *Writer) Emit(table string, values []interface{}) error {
	fmt.Fprintf(tw.w, "%s(", table)
	for i, value := range values {
		if i > 0 {
			fmt.Fprint(tw.w, ", ")
		}
		switch value := value.(type) {
		case Label:
			fmt.Fprint(tw.w, value.id)
		case string:
			fmt.Fprintf(tw.w, "\"%s\"", escapeString(capStringLength(value)))
		case int:
			fmt.Fprintf(tw.w, "%d", value)
		default:
			return errors.New("Cannot emit value")
		}
	}
	fmt.Fprintf(tw.w, ")\n")
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 VMware, Inc.

// +build freebsd linux

package gosigar

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

var system struct {
	ticks uint64
	btime uint64
}

var Procd string

func getLinuxBootTime() {
	// grab system boot time
	readFile(Procd+"/stat", func(line string) bool {
		if strings.HasPrefix(line, "btime") {
			system.btime, _ = strtoull(line[6:])
			return false // stop reading
		}
		return true
	})
}

func (self *LoadAverage) Get() error {
	line, err := ioutil.ReadFile(Procd + "/loadavg")
	if err != nil {
		return nil
	}

	fields := strings.Fields(string(line))

	self.One, _ = strconv.ParseFloat(fields[0], 64)
	self.Five, _ = strconv.ParseFloat(fields[1], 64)
	self.Fifteen, _ = strconv.ParseFloat(fields[2], 64)

	return nil
}

func (self *Mem) Get() error {

	table, err := parseMeminfo()
	if err != nil {
		return err
	}

	self.Total, _ = table["MemTotal"]
	self.Free, _ = table["MemFree"]
	buffers, _ := table["Buffers"]
	cached, _ := table["Cached"]

	if available, ok := table["MemAvailable"]; ok {
		// MemAvailable is in /proc/meminfo (kernel 3.14+)
		self.ActualFree = available
	} else {
		self.ActualFree = self.Free + buffers + cached
	}

	self.Used = self.Total - self.Free
	self.ActualUsed = self.Total - self.ActualFree

	return nil
}

func (self *Swap) Get() error {

	table, err := parseMeminfo()
	if err != nil {
		return err
	}
	self.Total, _ = table["SwapTotal"]
	self.Free, _ = table["SwapFree"]

	self.Used = self.Total - self.Free
	return nil
}

func (self *Cpu) Get() error {
	return readFile(Procd+"/stat", func(line string) bool {
		if len(line) > 4 && line[0:4] == "cpu " {
			parseCpuStat(self, line)
			return false
		}
		return true

	})
}

func (self *CpuList) Get() error {
	capacity := len(self.List)
	if capacity == 0 {
		capacity = 4
	}
	list := make([]Cpu, 0, capacity)

	err := readFile(Procd+"/stat", func(line string) bool {
		if len(line) > 3 && line[0:3] == "cpu" && line[3] != ' ' {
			cpu := Cpu{}
			parseCpuStat(&cpu, line)
			list = append(list, cpu)
		}
		return true
	})

	self.List = list

	return err
}

func (self *FileSystemList) Get() error {
	capacity := len(self.List)
	if capacity == 0 {
		capacity = 10
	}
	fslist := make([]FileSystem, 0, capacity)

	err := readFile(getMountTableFileName(), func(line string) bool {
		fields := strings.Fields(line)

		fs := FileSystem{}
		fs.DevName = fields[0]
		fs.DirName = fields[1]
		fs.SysTypeName = fields[2]
		fs.Options = fields[3]

		fslist = append(fslist, fs)

		return true
	})

	self.List = fslist

	return err
}

func (self *ProcList) Get() error {
	dir, err := os.Open(Procd)
	if err != nil {
		return err
	}
	defer dir.Close()

	const readAllDirnames = -1 // see os.File.Readdirnames doc

	names, err := dir.Readdirnames(readAllDirnames)
	if err != nil {
		return err
	}

	capacity := len(names)
	list := make([]int, 0, capacity)

	for _, name := range names {
		if name[0] < '0' || name[0] > '9' {
			continue
		}
		pid, err := strconv.Atoi(name)
		if err == nil {
			list = append(list, pid)
		}
	}

	self.List = list

	return nil
}

func (self *ProcState) Get(pid int) error {
	data, err := readProcFile(pid, "stat")
	if err != nil {
		return err
	}

	// Extract the comm value with is surrounded by parentheses.
	lIdx := bytes.Index(data, []byte("("))
	rIdx := bytes.LastIndex(data, []byte(")"))
	if lIdx < 0 || rIdx < 0 || lIdx >= rIdx || rIdx+2 >= len(data) {
		return fmt.Errorf("failed to extract comm for pid %d from '%v'", pid, string(data))
	}
	self.Name = string(data[lIdx+1 : rIdx])

	// Extract the rest of the fields that we are interested in.
	fields := bytes.Fields(data[rIdx+2:])
	if len(fields) <= 36 {
		return fmt.Errorf("expected more stat fields for pid %d from '%v'", pid, string(data))
	}

	interests := bytes.Join([][]byte{
		fields[0],  // state
		fields[1],  // ppid
		fields[2],  // pgrp
		fields[4],  // tty_nr
		fields[15], // priority
		fields[16], // nice
		fields[36], // processor (last processor executed on)
	}, []byte(" "))

	var state string
	_, err = fmt.Fscan(bytes.NewBuffer(interests),
		&state,
		&self.Ppid,
		&self.Pgid,
		&self.Tty,
		&self.Priority,
		&self.Nice,
		&self.Processor,
	)
	if err != nil {
		return fmt.Errorf("failed to parse stat fields for pid %d from '%v': %v", pid, string(data), err)
	}
	self.State = RunState(state[0])

	// Read /proc/[pid]/status to get the uid, then lookup uid to get username.
	status, err := getProcStatus(pid)
	if err != nil {
		return fmt.Errorf("failed to read process status for pid %d: %v", pid, err)
	}
	uids, err := getUIDs(status)
	if err != nil {
		return fmt.Errorf("failed to read process status for pid %d: %v", pid, err)
	}
	user, err := user.LookupId(uids[0])
	if err == nil {
		self.Username = user.Username
	} else {
		self.Username = uids[0]
	}

	return nil
}

func (self *ProcMem) Get(pid int) error {
	contents, err := readProcFile(pid, "statm")
	if err != nil {
		return err
	}

	fields := strings.Fields(string(contents))

	size, _ := strtoull(fields[0])
	self.Size = size << 12

	rss, _ := strtoull(fields[1])
	self.Resident = rss << 12

	share, _ := strtoull(fields[2])
	self.Share = share << 12

	contents, err = readProcFile(pid, "stat")
	if err != nil {
		return err
	}

	fields = strings.Fields(string(contents))

	self.MinorFaults, _ = strtoull(fields[10])
	self.MajorFaults, _ = strtoull(fields[12])
	self.PageFaults = self.MinorFaults + self.MajorFaults

	return nil
}

func (self *ProcTime) Get(pid int) error {
	contents, err := readProcFile(pid, "stat")
	if err != nil {
		return err
	}

	fields := strings.Fields(string(contents))

	user, _ := strtoull(fields[13])
	sys, _ := strtoull(fields[14])
	// convert to millis
	self.User = user * (1000 / system.ticks)
	self.Sys = sys * (1000 / system.ticks)
	self.Total = self.User + self.Sys

	// convert to millis
	self.StartTime, _ = strtoull(fields[21])
	self.StartTime /= system.ticks
	self.StartTime += system.btime
	self.StartTime *= 1000

	return nil
}

func (self *ProcArgs) Get(pid int) error {
	contents, err := readProcFile(pid, "cmdline")
	if err != nil {
		return err
	}

	bbuf := bytes.NewBuffer(contents)

	var args []string

	for {
		arg, err := bbuf.ReadBytes(0)
		if err == io.EOF {
			break
		}
		args = append(args, string(chop(arg)))
	}

	self.List = args

	return nil
}

func (self *ProcEnv) Get(pid int) error {
	contents, err := readProcFile(pid, "environ")
	if err != nil {
		return err
	}

	if self.Vars == nil {
		self.Vars = map[string]string{}
	}

	pairs := bytes.Split(contents, []byte{0})
	for _, kv := range pairs {
		parts := bytes.SplitN(kv, []byte{'='}, 2)
		if len(parts) != 2 {
			continue
		}

		key := string(bytes.TrimSpace(parts[0]))
		if key == "" {
			continue
		}

		self.Vars[key] = string(bytes.TrimSpace(parts[1]))
	}

	return nil
}

func (self *ProcExe) Get(pid int) error {
	fields := map[string]*string{
		"exe":  &self.Name,
		"cwd":  &self.Cwd,
		"root": &self.Root,
	}

	for name, field := range fields {
		val, err := os.Readlink(procFileName(pid, name))

		if err != nil {
			return err
		}

		*field = val
	}

	return nil
}

func parseMeminfo() (map[string]uint64, error) {
	table := map[string]uint64{}

	err := readFile(Procd+"/meminfo", func(line string) bool {
		fields := strings.Split(line, ":")

		if len(fields) != 2 {
			return true // skip on errors
		}

		valueUnit := strings.Fields(fields[1])
		value, err := strtoull(valueUnit[0])
		if err != nil {
			return true // skip on errors
		}

		if len(valueUnit) > 1 && valueUnit[1] == "kB" {
			value *= 1024
		}
		table[fields[0]] = value

		return true
	})
	return table, err
}

func readFile(file string, handler func(string) bool) error {
	contents, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	reader := bufio.NewReader(bytes.NewBuffer(contents))

	for {
		line, _, err := reader.ReadLine()
		if err == io.EOF {
			break
		}
		if !handler(string(line)) {
			break
		}
	}

	return nil
}

func strtoull(val string) (uint64, error) {
	return strconv.ParseUint(val, 10, 64)
}

func procFileName(pid int, name string) string {
	return Procd + "/" + strconv.Itoa(pid) + "/" + name
}

func readProcFile(pid int, name string) ([]byte, error) {
	path := procFileName(pid, name)
	contents, err := ioutil.ReadFile(path)

	if err != nil {
		if perr, ok := err.(*os.PathError); ok {
			if perr.Err == syscall.ENOENT {
				return nil, syscall.ESRCH
			}
		}
	}

	return contents, err
}

// getProcStatus reads /proc/[pid]/status which contains process status
// information in human readable form.
func getProcStatus(pid int) (map[string]string, error) {
	status := make(map[string]string, 42)
	path := filepath.Join(Procd, strconv.Itoa(pid), "status")
	err := readFile(path, func(line string) bool {
		fields := strings.SplitN(line, ":", 2)
		if len(fields) == 2 {
			status[fields[0]] = strings.TrimSpace(fields[1])
		}

		return true
	})
	return status, err
}

// getUIDs reads the "Uid" value from status and splits it into four values --
// real, effective, saved set, and  file system UIDs.
func getUIDs(status map[string]string) ([]string, error) {
	uidLine, ok := status["Uid"]
	if !ok {
		return nil, fmt.Errorf("Uid not found in proc status")
	}

	uidStrs := strings.Fields(uidLine)
	if len(uidStrs) != 4 {
		return nil, fmt.Errorf("Uid line ('%s') did not contain four values", uidLine)
	}

	return uidStrs, nil
}
<commit_msg>Recover panic when reading proc files (#103)<commit_after>// Copyright (c) 2012 VMware, Inc.

// +build freebsd linux

package gosigar

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

var system struct {
	ticks uint64
	btime uint64
}

var Procd string

func getLinuxBootTime() {
	// grab system boot time
	readFile(Procd+"/stat", func(line string) bool {
		if strings.HasPrefix(line, "btime") {
			system.btime, _ = strtoull(line[6:])
			return false // stop reading
		}
		return true
	})
}

func (self *LoadAverage) Get() error {
	line, err := ioutil.ReadFile(Procd + "/loadavg")
	if err != nil {
		return nil
	}

	fields := strings.Fields(string(line))

	self.One, _ = strconv.ParseFloat(fields[0], 64)
	self.Five, _ = strconv.ParseFloat(fields[1], 64)
	self.Fifteen, _ = strconv.ParseFloat(fields[2], 64)

	return nil
}

func (self *Mem) Get() error {

	table, err := parseMeminfo()
	if err != nil {
		return err
	}

	self.Total, _ = table["MemTotal"]
	self.Free, _ = table["MemFree"]
	buffers, _ := table["Buffers"]
	cached, _ := table["Cached"]

	if available, ok := table["MemAvailable"]; ok {
		// MemAvailable is in /proc/meminfo (kernel 3.14+)
		self.ActualFree = available
	} else {
		self.ActualFree = self.Free + buffers + cached
	}

	self.Used = self.Total - self.Free
	self.ActualUsed = self.Total - self.ActualFree

	return nil
}

func (self *Swap) Get() error {

	table, err := parseMeminfo()
	if err != nil {
		return err
	}
	self.Total, _ = table["SwapTotal"]
	self.Free, _ = table["SwapFree"]

	self.Used = self.Total - self.Free
	return nil
}

func (self *Cpu) Get() error {
	return readFile(Procd+"/stat", func(line string) bool {
		if len(line) > 4 && line[0:4] == "cpu " {
			parseCpuStat(self, line)
			return false
		}
		return true

	})
}

func (self *CpuList) Get() error {
	capacity := len(self.List)
	if capacity == 0 {
		capacity = 4
	}
	list := make([]Cpu, 0, capacity)

	err := readFile(Procd+"/stat", func(line string) bool {
		if len(line) > 3 && line[0:3] == "cpu" && line[3] != ' ' {
			cpu := Cpu{}
			parseCpuStat(&cpu, line)
			list = append(list, cpu)
		}
		return true
	})

	self.List = list

	return err
}

func (self *FileSystemList) Get() error {
	capacity := len(self.List)
	if capacity == 0 {
		capacity = 10
	}
	fslist := make([]FileSystem, 0, capacity)

	err := readFile(getMountTableFileName(), func(line string) bool {
		fields := strings.Fields(line)

		fs := FileSystem{}
		fs.DevName = fields[0]
		fs.DirName = fields[1]
		fs.SysTypeName = fields[2]
		fs.Options = fields[3]

		fslist = append(fslist, fs)

		return true
	})

	self.List = fslist

	return err
}

func (self *ProcList) Get() error {
	dir, err := os.Open(Procd)
	if err != nil {
		return err
	}
	defer dir.Close()

	const readAllDirnames = -1 // see os.File.Readdirnames doc

	names, err := dir.Readdirnames(readAllDirnames)
	if err != nil {
		return err
	}

	capacity := len(names)
	list := make([]int, 0, capacity)

	for _, name := range names {
		if name[0] < '0' || name[0] > '9' {
			continue
		}
		pid, err := strconv.Atoi(name)
		if err == nil {
			list = append(list, pid)
		}
	}

	self.List = list

	return nil
}

func (self *ProcState) Get(pid int) error {
	data, err := readProcFile(pid, "stat")
	if err != nil {
		return err
	}

	// Extract the comm value with is surrounded by parentheses.
	lIdx := bytes.Index(data, []byte("("))
	rIdx := bytes.LastIndex(data, []byte(")"))
	if lIdx < 0 || rIdx < 0 || lIdx >= rIdx || rIdx+2 >= len(data) {
		return fmt.Errorf("failed to extract comm for pid %d from '%v'", pid, string(data))
	}
	self.Name = string(data[lIdx+1 : rIdx])

	// Extract the rest of the fields that we are interested in.
	fields := bytes.Fields(data[rIdx+2:])
	if len(fields) <= 36 {
		return fmt.Errorf("expected more stat fields for pid %d from '%v'", pid, string(data))
	}

	interests := bytes.Join([][]byte{
		fields[0],  // state
		fields[1],  // ppid
		fields[2],  // pgrp
		fields[4],  // tty_nr
		fields[15], // priority
		fields[16], // nice
		fields[36], // processor (last processor executed on)
	}, []byte(" "))

	var state string
	_, err = fmt.Fscan(bytes.NewBuffer(interests),
		&state,
		&self.Ppid,
		&self.Pgid,
		&self.Tty,
		&self.Priority,
		&self.Nice,
		&self.Processor,
	)
	if err != nil {
		return fmt.Errorf("failed to parse stat fields for pid %d from '%v': %v", pid, string(data), err)
	}
	self.State = RunState(state[0])

	// Read /proc/[pid]/status to get the uid, then lookup uid to get username.
	status, err := getProcStatus(pid)
	if err != nil {
		return fmt.Errorf("failed to read process status for pid %d: %v", pid, err)
	}
	uids, err := getUIDs(status)
	if err != nil {
		return fmt.Errorf("failed to read process status for pid %d: %v", pid, err)
	}
	user, err := user.LookupId(uids[0])
	if err == nil {
		self.Username = user.Username
	} else {
		self.Username = uids[0]
	}

	return nil
}

func (self *ProcMem) Get(pid int) error {
	contents, err := readProcFile(pid, "statm")
	if err != nil {
		return err
	}

	fields := strings.Fields(string(contents))

	size, _ := strtoull(fields[0])
	self.Size = size << 12

	rss, _ := strtoull(fields[1])
	self.Resident = rss << 12

	share, _ := strtoull(fields[2])
	self.Share = share << 12

	contents, err = readProcFile(pid, "stat")
	if err != nil {
		return err
	}

	fields = strings.Fields(string(contents))

	self.MinorFaults, _ = strtoull(fields[10])
	self.MajorFaults, _ = strtoull(fields[12])
	self.PageFaults = self.MinorFaults + self.MajorFaults

	return nil
}

func (self *ProcTime) Get(pid int) error {
	contents, err := readProcFile(pid, "stat")
	if err != nil {
		return err
	}

	fields := strings.Fields(string(contents))

	user, _ := strtoull(fields[13])
	sys, _ := strtoull(fields[14])
	// convert to millis
	self.User = user * (1000 / system.ticks)
	self.Sys = sys * (1000 / system.ticks)
	self.Total = self.User + self.Sys

	// convert to millis
	self.StartTime, _ = strtoull(fields[21])
	self.StartTime /= system.ticks
	self.StartTime += system.btime
	self.StartTime *= 1000

	return nil
}

func (self *ProcArgs) Get(pid int) error {
	contents, err := readProcFile(pid, "cmdline")
	if err != nil {
		return err
	}

	bbuf := bytes.NewBuffer(contents)

	var args []string

	for {
		arg, err := bbuf.ReadBytes(0)
		if err == io.EOF {
			break
		}
		args = append(args, string(chop(arg)))
	}

	self.List = args

	return nil
}

func (self *ProcEnv) Get(pid int) error {
	contents, err := readProcFile(pid, "environ")
	if err != nil {
		return err
	}

	if self.Vars == nil {
		self.Vars = map[string]string{}
	}

	pairs := bytes.Split(contents, []byte{0})
	for _, kv := range pairs {
		parts := bytes.SplitN(kv, []byte{'='}, 2)
		if len(parts) != 2 {
			continue
		}

		key := string(bytes.TrimSpace(parts[0]))
		if key == "" {
			continue
		}

		self.Vars[key] = string(bytes.TrimSpace(parts[1]))
	}

	return nil
}

func (self *ProcExe) Get(pid int) error {
	fields := map[string]*string{
		"exe":  &self.Name,
		"cwd":  &self.Cwd,
		"root": &self.Root,
	}

	for name, field := range fields {
		val, err := os.Readlink(procFileName(pid, name))

		if err != nil {
			return err
		}

		*field = val
	}

	return nil
}

func parseMeminfo() (map[string]uint64, error) {
	table := map[string]uint64{}

	err := readFile(Procd+"/meminfo", func(line string) bool {
		fields := strings.Split(line, ":")

		if len(fields) != 2 {
			return true // skip on errors
		}

		valueUnit := strings.Fields(fields[1])
		value, err := strtoull(valueUnit[0])
		if err != nil {
			return true // skip on errors
		}

		if len(valueUnit) > 1 && valueUnit[1] == "kB" {
			value *= 1024
		}
		table[fields[0]] = value

		return true
	})
	return table, err
}

func readFile(file string, handler func(string) bool) error {
	contents, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	reader := bufio.NewReader(bytes.NewBuffer(contents))

	for {
		line, _, err := reader.ReadLine()
		if err == io.EOF {
			break
		}
		if !handler(string(line)) {
			break
		}
	}

	return nil
}

func strtoull(val string) (uint64, error) {
	return strconv.ParseUint(val, 10, 64)
}

func procFileName(pid int, name string) string {
	return Procd + "/" + strconv.Itoa(pid) + "/" + name
}

func readProcFile(pid int, name string) (content []byte, err error) {
	path := procFileName(pid, name)

	// Panics have been reported when reading proc files, let's recover and
	// report the path if this happens
	// See https://github.com/elastic/beats/issues/6692
	defer func() {
		if r := recover(); r != nil {
			content = nil
			err = fmt.Errorf("recovered panic when reading proc file '%s': %v", path, r)
		}
	}()
	contents, err := ioutil.ReadFile(path)

	if err != nil {
		if perr, ok := err.(*os.PathError); ok {
			if perr.Err == syscall.ENOENT {
				return nil, syscall.ESRCH
			}
		}
	}

	return contents, err
}

// getProcStatus reads /proc/[pid]/status which contains process status
// information in human readable form.
func getProcStatus(pid int) (map[string]string, error) {
	status := make(map[string]string, 42)
	path := filepath.Join(Procd, strconv.Itoa(pid), "status")
	err := readFile(path, func(line string) bool {
		fields := strings.SplitN(line, ":", 2)
		if len(fields) == 2 {
			status[fields[0]] = strings.TrimSpace(fields[1])
		}

		return true
	})
	return status, err
}

// getUIDs reads the "Uid" value from status and splits it into four values --
// real, effective, saved set, and  file system UIDs.
func getUIDs(status map[string]string) ([]string, error) {
	uidLine, ok := status["Uid"]
	if !ok {
		return nil, fmt.Errorf("Uid not found in proc status")
	}

	uidStrs := strings.Fields(uidLine)
	if len(uidStrs) != 4 {
		return nil, fmt.Errorf("Uid line ('%s') did not contain four values", uidLine)
	}

	return uidStrs, nil
}
<|endoftext|>


<commit_before>package dis

import "github.com/catorpilor/LeetCode/utils"

// DistributeCoins distribute coins
func DistributeCoins(root *utils.TreeNode) int {
	return 0
}
<commit_msg>sovled based on discussion<commit_after>package dis

import "github.com/catorpilor/LeetCode/utils"

// DistributeCoins distribute coins
func DistributeCoins(root *utils.TreeNode) int {
	var res int
	dfs(root, &res)
	return res
}

func dfs(root *utils.TreeNode, res *int) int {
	if root == nil {
		return 0
	}
	// l ,r represents the total coins needs to be transfered for
	// root's substree
	l, r := dfs(root.Left, res), dfs(root.Right, res)
	*res += utils.Abs(l) + utils.Abs(r)
	return root.Val + l + r - 1
}
<|endoftext|>


<commit_before>package keryxlib

// Copyright 2015 MediaMath <http://www.mediamath.com>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

import (
	"github.com/MediaMath/keryxlib/filters"
	"github.com/MediaMath/keryxlib/message"
	"github.com/MediaMath/keryxlib/pg"
	"github.com/MediaMath/keryxlib/streams"
)

//TransactionChannel sets up a keryx stream and schema reader with the provided configuration and returns
//it as a channel
func TransactionChannel(serverVersion string, kc *Config) (<-chan *message.Transaction, error) {

	schemaReader, err := pg.NewSchemaReader(kc.PGConnStrings, "postgres", 255)
	if err != nil {
		return nil, err
	}

	bufferWorkingDirectory, err := kc.GetBufferDirectoryOrTemp()
	if err != nil {
		return nil, err
	}

	f := filters.Exclusive(schemaReader, kc.ExcludeRelations)
	if len(kc.IncludeRelations) > 0 {
		f = filters.Inclusive(schemaReader, kc.IncludeRelations)
	}

	stream := NewKeryxStream(schemaReader, kc.MaxMessagePerTxn)
	return stream.StartKeryxStream(serverVersion, f, kc.DataDir, bufferWorkingDirectory)
}

//FullStream is a facade around the full process of taking WAL entries and publishing them as txn messages.
type FullStream struct {
	walStream       *streams.WalStream
	sr              *pg.SchemaReader
	MaxMessageCount uint
}

//NewKeryxStream takes a schema reader and returns a FullStream
func NewKeryxStream(sr *pg.SchemaReader, maxMessageCount uint) *FullStream {
	return &FullStream{nil, sr, maxMessageCount}
}

//Stop will end the reading on the WAL log and subsequent streams will therefore end.
func (fs *FullStream) Stop() {
	if fs.walStream != nil {
		fs.walStream.Stop()
	}
}

//StartKeryxStream will start all the streams necessary to go from WAL entries to txn messages.
func (fs *FullStream) StartKeryxStream(serverVersion string, filters filters.MessageFilter, dataDir string, bufferWorkingDirectory string) (<-chan *message.Transaction, error) {
	walStream, err := streams.NewWalStream(dataDir)
	if err != nil {
		return nil, err
	}
	fs.walStream = walStream

	wal, err := fs.walStream.Start()
	if err != nil {
		return nil, err
	}

	txnBuffer := &streams.TxnBuffer{Filters: filters, WorkingDirectory: bufferWorkingDirectory, SchemaReader: fs.sr}
	buffered, err := txnBuffer.Start(wal)
	if err != nil {
		fs.Stop()
		return nil, err
	}

	populated := &streams.PopulatedMessageStream{Filters: filters, SchemaReader: fs.sr, MaxMessageCount: fs.MaxMessageCount}
	keryx, err := populated.Start(serverVersion, buffered)
	if err != nil {
		fs.Stop()
		return nil, err
	}

	return keryx, nil
}
<commit_msg>make a stoppable facade entry for keryx transactions<commit_after>package keryxlib

// Copyright 2015 MediaMath <http://www.mediamath.com>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

import (
	"fmt"

	"github.com/MediaMath/keryxlib/filters"
	"github.com/MediaMath/keryxlib/message"
	"github.com/MediaMath/keryxlib/pg"
	"github.com/MediaMath/keryxlib/streams"
)

//TransactionChannel sets up a keryx stream and schema reader with the provided configuration and returns
//it as a channel
func TransactionChannel(serverVersion string, kc *Config) (<-chan *message.Transaction, error) {
	return StartTransactionChannel(serverVersion, kc, nil)
}

//StartTransactionChannel sets up a keryx stream and schema reader with the provided configuration and return
//it as a channel. The channel can be stopped with the provided stopper
func StartTransactionChannel(serverVersion string, kc *Config, stopper *TxnChannelStopper) (<-chan *message.Transaction, error) {
	schemaReader, err := pg.NewSchemaReader(kc.PGConnStrings, "postgres", 255)
	if err != nil {
		return nil, err
	}

	bufferWorkingDirectory, err := kc.GetBufferDirectoryOrTemp()
	if err != nil {
		return nil, err
	}

	f := filters.Exclusive(schemaReader, kc.ExcludeRelations)
	if len(kc.IncludeRelations) > 0 {
		f = filters.Inclusive(schemaReader, kc.IncludeRelations)
	}

	stream := NewKeryxStream(schemaReader, kc.MaxMessagePerTxn)
	if stopper != nil {
		if stopper.done == nil {
			return nil, fmt.Errorf("Please use NewTxnChannelStopper to create your stopper")
		}

		go func() {
			select {
			case <-stopper.done:
				stream.Stop()
			}
		}()
	}
	return stream.StartKeryxStream(serverVersion, f, kc.DataDir, bufferWorkingDirectory)
}

//NewTxnChannelStopper creates a TxnChannelStopper
func NewTxnChannelStopper() *TxnChannelStopper {
	return &TxnChannelStopper{done: make(chan interface{})}
}

//TxnChannelStopper can be used to stop a Transaction channel
type TxnChannelStopper struct {
	done chan interface{}
}

//Stop will initiate a Transaction channel shutdown
func (t *TxnChannelStopper) Stop() {
	close(t.done)
}

//FullStream is a facade around the full process of taking WAL entries and publishing them as txn messages.
type FullStream struct {
	walStream       *streams.WalStream
	sr              *pg.SchemaReader
	MaxMessageCount uint
}

//NewKeryxStream takes a schema reader and returns a FullStream
func NewKeryxStream(sr *pg.SchemaReader, maxMessageCount uint) *FullStream {
	return &FullStream{nil, sr, maxMessageCount}
}

//Stop will end the reading on the WAL log and subsequent streams will therefore end.
func (fs *FullStream) Stop() {
	if fs.walStream != nil {
		fs.walStream.Stop()
	}
}

//StartKeryxStream will start all the streams necessary to go from WAL entries to txn messages.
func (fs *FullStream) StartKeryxStream(serverVersion string, filters filters.MessageFilter, dataDir string, bufferWorkingDirectory string) (<-chan *message.Transaction, error) {
	walStream, err := streams.NewWalStream(dataDir)
	if err != nil {
		return nil, err
	}
	fs.walStream = walStream

	wal, err := fs.walStream.Start()
	if err != nil {
		return nil, err
	}

	txnBuffer := &streams.TxnBuffer{Filters: filters, WorkingDirectory: bufferWorkingDirectory, SchemaReader: fs.sr}
	buffered, err := txnBuffer.Start(wal)
	if err != nil {
		fs.Stop()
		return nil, err
	}

	populated := &streams.PopulatedMessageStream{Filters: filters, SchemaReader: fs.sr, MaxMessageCount: fs.MaxMessageCount}
	keryx, err := populated.Start(serverVersion, buffered)
	if err != nil {
		fs.Stop()
		return nil, err
	}

	return keryx, nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"golang.org/x/oauth2/google"
)

const (
	livenessMetric       = "k8s_deployer"
	gitstoreSubscriberID = "k8s-deployer"
)

func main() {
	configRepo := flag.String("config_repo", "https://skia.googlesource.com/k8s-config.git", "Repo containing Kubernetes configurations.")
	configSubdir := flag.String("config_subdir", "", "Subdirectory within the config repo to apply to this cluster.")
	interval := flag.Duration("interval", 10*time.Minute, "How often to re-apply configurations to the cluster")
	port := flag.String("port", ":8000", "HTTP service port for the web server (e.g., ':8000')")
	promPort := flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")
	prune := flag.Bool("prune", false, "Whether to run 'kubectl apply' with '--prune'")
	kubectl := flag.String("kubectl", "kubectl", "Path to the kubectl executable.")
	k8sServer := flag.String("k8s_server", "", "Address of the Kubernetes server.")

	common.InitWithMust("k8s_deployer", common.PrometheusOpt(promPort))
	defer sklog.Flush()

	if *configRepo == "" {
		sklog.Fatal("config_repo is required.")
	}
	if *configSubdir == "" {
		// Note: this wouldn't be required if we had separate config repos per
		// cluster.
		sklog.Fatal("config_subdir is required.")
	}

	ctx := context.Background()

	// OAuth2.0 TokenSource.
	ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail, auth.ScopeGerrit)
	if err != nil {
		sklog.Fatal(err)
	}
	// Authenticated HTTP client.
	httpClient := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()

	// Gitiles repo.
	repo := gitiles.NewRepo(*configRepo, httpClient)

	// Apply configurations in a loop.  Note that we could respond directly to
	// commits in the repo via GitStore and PubSub, but that would require
	// access to BigTable, and with a relatively small interval we won't notice
	// too much of a delay.
	liveness := metrics2.NewLiveness(livenessMetric)
	go util.RepeatCtx(ctx, *interval, func(ctx context.Context) {
		if err := applyConfigs(ctx, repo, *kubectl, *k8sServer, *configSubdir, *prune); err != nil {
			sklog.Errorf("Failed to apply configs to cluster: %s", err)
		} else {
			liveness.Reset()
		}
	})

	// Run health check server.
	httputils.RunHealthCheckServer(*port)
}

func applyConfigs(ctx context.Context, repo *gitiles.Repo, kubectl, k8sServer, configSubdir string, prune bool) error {
	// Download the configs from Gitiles instead of maintaining a local Git
	// checkout, to avoid dealing with Git, persistent checkouts, etc.

	// Obtain the current set of configurations for the cluster.
	head, err := repo.Details(ctx, "main")
	if err != nil {
		return skerr.Wrapf(err, "failed to get most recent commit")
	}
	files, err := repo.ListFilesRecursiveAtRef(ctx, configSubdir, head.Hash)
	if err != nil {
		return skerr.Wrapf(err, "failed to list configs")
	}

	// Read the config contents in the given directory.
	eg := util.NewNamedErrGroup()
	contents := make(map[string][]byte, len(files))
	contentsMtx := sync.Mutex{}
	sklog.Infof("Downloading config files at %s", head.Hash)
	for _, file := range files {
		file := file // https://golang.org/doc/faq#closures_and_goroutines
		eg.Go(file, func() error {
			fullPath := path.Join(configSubdir, file)
			sklog.Infof("  %s", fullPath)
			fileContents, err := repo.ReadFileAtRef(ctx, fullPath, head.Hash)
			if err != nil {
				return skerr.Wrapf(err, "failed to retrieve contents of %s", fullPath)
			}
			contentsMtx.Lock()
			defer contentsMtx.Unlock()
			contents[file] = fileContents
			return nil
		})
	}
	if err := eg.Wait(); err != nil {
		return skerr.Wrapf(err, "failed to download configs")
	}

	// Write the config contents to a temporary dir.
	tmp, err := ioutil.TempDir("", "")
	if err != nil {
		return skerr.Wrapf(err, "failed to create temp dir")
	}
	defer util.RemoveAll(tmp)

	for path, fileContents := range contents {
		fullPath := filepath.Join(tmp, path)
		dir := filepath.Dir(fullPath)
		if _, err := os.Stat(path); os.IsNotExist(err) {
			if err := os.MkdirAll(dir, os.ModePerm); err != nil {
				return skerr.Wrapf(err, "failed to create %s", dir)
			}
		}
		if err := ioutil.WriteFile(fullPath, fileContents, os.ModePerm); err != nil {
			return skerr.Wrapf(err, "failed to create %s", fullPath)
		}
	}

	// Apply the configs to the cluster.
	// Note: this is a very naive approach.  We rely on the Kubernetes server to
	// determine when changes actually need to be made.  We could instead use
	// the REST API to load the full set of configuration which is actually
	// running on the server, diff that against the checked-in config files,
	// and then explicitly make only the changes we want to make.  That would be
	// a much more complicated and error-prone approach, but it would allow us
	// to partially apply configurations in the case of a failure and to alert
	// on specific components which fail to apply for whatever reason.
	cmd := []string{kubectl, "apply"}
	if k8sServer != "" {
		cmd = append(cmd, "--server", k8sServer)
	}
	if prune {
		cmd = append(cmd, "--prune")
	}
	cmd = append(cmd, "-f", ".")
	output, err := exec.RunCwd(ctx, tmp, cmd...)
	if err != nil {
		return skerr.Wrapf(err, "failed to apply configs: %s", output)
	}
	sklog.Infof("Output from kubectl: %s", output)
	return nil
}
<commit_msg>[k8s-deployer] Use --all with --prune<commit_after>package main

import (
	"context"
	"flag"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"golang.org/x/oauth2/google"
)

const (
	livenessMetric       = "k8s_deployer"
	gitstoreSubscriberID = "k8s-deployer"
)

func main() {
	configRepo := flag.String("config_repo", "https://skia.googlesource.com/k8s-config.git", "Repo containing Kubernetes configurations.")
	configSubdir := flag.String("config_subdir", "", "Subdirectory within the config repo to apply to this cluster.")
	interval := flag.Duration("interval", 10*time.Minute, "How often to re-apply configurations to the cluster")
	port := flag.String("port", ":8000", "HTTP service port for the web server (e.g., ':8000')")
	promPort := flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")
	prune := flag.Bool("prune", false, "Whether to run 'kubectl apply' with '--prune'")
	kubectl := flag.String("kubectl", "kubectl", "Path to the kubectl executable.")
	k8sServer := flag.String("k8s_server", "", "Address of the Kubernetes server.")

	common.InitWithMust("k8s_deployer", common.PrometheusOpt(promPort))
	defer sklog.Flush()

	if *configRepo == "" {
		sklog.Fatal("config_repo is required.")
	}
	if *configSubdir == "" {
		// Note: this wouldn't be required if we had separate config repos per
		// cluster.
		sklog.Fatal("config_subdir is required.")
	}

	ctx := context.Background()

	// OAuth2.0 TokenSource.
	ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail, auth.ScopeGerrit)
	if err != nil {
		sklog.Fatal(err)
	}
	// Authenticated HTTP client.
	httpClient := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()

	// Gitiles repo.
	repo := gitiles.NewRepo(*configRepo, httpClient)

	// Apply configurations in a loop.  Note that we could respond directly to
	// commits in the repo via GitStore and PubSub, but that would require
	// access to BigTable, and with a relatively small interval we won't notice
	// too much of a delay.
	liveness := metrics2.NewLiveness(livenessMetric)
	go util.RepeatCtx(ctx, *interval, func(ctx context.Context) {
		if err := applyConfigs(ctx, repo, *kubectl, *k8sServer, *configSubdir, *prune); err != nil {
			sklog.Errorf("Failed to apply configs to cluster: %s", err)
		} else {
			liveness.Reset()
		}
	})

	// Run health check server.
	httputils.RunHealthCheckServer(*port)
}

func applyConfigs(ctx context.Context, repo *gitiles.Repo, kubectl, k8sServer, configSubdir string, prune bool) error {
	// Download the configs from Gitiles instead of maintaining a local Git
	// checkout, to avoid dealing with Git, persistent checkouts, etc.

	// Obtain the current set of configurations for the cluster.
	head, err := repo.Details(ctx, "main")
	if err != nil {
		return skerr.Wrapf(err, "failed to get most recent commit")
	}
	files, err := repo.ListFilesRecursiveAtRef(ctx, configSubdir, head.Hash)
	if err != nil {
		return skerr.Wrapf(err, "failed to list configs")
	}

	// Read the config contents in the given directory.
	eg := util.NewNamedErrGroup()
	contents := make(map[string][]byte, len(files))
	contentsMtx := sync.Mutex{}
	sklog.Infof("Downloading config files at %s", head.Hash)
	for _, file := range files {
		file := file // https://golang.org/doc/faq#closures_and_goroutines
		eg.Go(file, func() error {
			fullPath := path.Join(configSubdir, file)
			sklog.Infof("  %s", fullPath)
			fileContents, err := repo.ReadFileAtRef(ctx, fullPath, head.Hash)
			if err != nil {
				return skerr.Wrapf(err, "failed to retrieve contents of %s", fullPath)
			}
			contentsMtx.Lock()
			defer contentsMtx.Unlock()
			contents[file] = fileContents
			return nil
		})
	}
	if err := eg.Wait(); err != nil {
		return skerr.Wrapf(err, "failed to download configs")
	}

	// Write the config contents to a temporary dir.
	tmp, err := ioutil.TempDir("", "")
	if err != nil {
		return skerr.Wrapf(err, "failed to create temp dir")
	}
	defer util.RemoveAll(tmp)

	for path, fileContents := range contents {
		fullPath := filepath.Join(tmp, path)
		dir := filepath.Dir(fullPath)
		if _, err := os.Stat(path); os.IsNotExist(err) {
			if err := os.MkdirAll(dir, os.ModePerm); err != nil {
				return skerr.Wrapf(err, "failed to create %s", dir)
			}
		}
		if err := ioutil.WriteFile(fullPath, fileContents, os.ModePerm); err != nil {
			return skerr.Wrapf(err, "failed to create %s", fullPath)
		}
	}

	// Apply the configs to the cluster.
	// Note: this is a very naive approach.  We rely on the Kubernetes server to
	// determine when changes actually need to be made.  We could instead use
	// the REST API to load the full set of configuration which is actually
	// running on the server, diff that against the checked-in config files,
	// and then explicitly make only the changes we want to make.  That would be
	// a much more complicated and error-prone approach, but it would allow us
	// to partially apply configurations in the case of a failure and to alert
	// on specific components which fail to apply for whatever reason.
	cmd := []string{kubectl, "apply"}
	if k8sServer != "" {
		cmd = append(cmd, "--server", k8sServer)
	}
	if prune {
		cmd = append(cmd, "--prune", "--all")
	}
	cmd = append(cmd, "-f", ".")
	output, err := exec.RunCwd(ctx, tmp, cmd...)
	if err != nil {
		return skerr.Wrapf(err, "failed to apply configs: %s", output)
	}
	sklog.Infof("Output from kubectl: %s", output)
	return nil
}
<|endoftext|>


<commit_before>package units

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

var sizeRegex *regexp.Regexp

func init() {
	var err error
	if sizeRegex, err = regexp.Compile("^(\\d+)([kKmMgGtTpP])?[bB]?$"); err != nil {
		panic("Failed to compile the 'size' regular expression")
	}
}

var bytePrefixes = [...]string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}

// HumanSize returns a human-readable approximation of a size
// using SI standard (eg. "44kB", "17MB")
func HumanSize(size int64) string {
	i := 0
	sizef := float64(size)
	for sizef >= 1000.0 {
		sizef = sizef / 1000.0
		i++
	}
	return fmt.Sprintf("%.4g %s", sizef, bytePrefixes[i])
}

// FromHumanSize returns an integer from a human-readable specification of a size
// using SI standard (eg. "44kB", "17MB")
func FromHumanSize(size string) (int64, error) {
	matches := sizeRegex.FindStringSubmatch(size)

	if len(matches) != 3 {
		return -1, fmt.Errorf("Invalid size: '%s'", size)
	}

	theSize, err := strconv.ParseInt(matches[1], 10, 0)
	if err != nil {
		return -1, err
	}

	unit := strings.ToLower(matches[2])

	switch unit {
	case "k":
		theSize *= 1000
	case "m":
		theSize *= 1000 * 1000
	case "g":
		theSize *= 1000 * 1000 * 1000
	case "t":
		theSize *= 1000 * 1000 * 1000 * 1000
	case "p":
		theSize *= 1000 * 1000 * 1000 * 1000 * 1000
	}

	return theSize, nil
}

// Parses a human-readable string representing an amount of RAM
// in bytes, kibibytes, mebibytes, gibibytes, or tebibytes and
// returns the number of bytes, or -1 if the string is unparseable.
// Units are case-insensitive, and the 'b' suffix is optional.
func RAMInBytes(size string) (int64, error) {
	matches := sizeRegex.FindStringSubmatch(size)

	if len(matches) != 3 {
		return -1, fmt.Errorf("Invalid size: '%s'", size)
	}

	memLimit, err := strconv.ParseInt(matches[1], 10, 0)
	if err != nil {
		return -1, err
	}

	unit := strings.ToLower(matches[2])

	switch unit {
	case "k":
		memLimit *= 1024
	case "m":
		memLimit *= 1024 * 1024
	case "g":
		memLimit *= 1024 * 1024 * 1024
	case "t":
		memLimit *= 1024 * 1024 * 1024 * 1024
	case "p":
		memLimit *= 1024 * 1024 * 1024 * 1024 * 1024
	}

	return memLimit, nil
}
<commit_msg>pkg/units: Refactored common code to single func<commit_after>package units

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

const (
	decimalKUnit = 1000
	binaryKUnit  = 1024
)

var sizeRegex *regexp.Regexp

func init() {
	var err error
	if sizeRegex, err = regexp.Compile("^(\\d+)([kKmMgGtTpP])?[bB]?$"); err != nil {
		panic("Failed to compile the 'size' regular expression")
	}
}

var bytePrefixes = [...]string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}

// HumanSize returns a human-readable approximation of a size
// using SI standard (eg. "44kB", "17MB")
func HumanSize(size int64) string {
	i := 0
	sizef := float64(size)
	for sizef >= 1000.0 {
		sizef = sizef / 1000.0
		i++
	}
	return fmt.Sprintf("%.4g %s", sizef, bytePrefixes[i])
}

// FromHumanSize returns an integer from a human-readable specification of a
// size using SI standard (eg. "44kB", "17MB")
func FromHumanSize(size string) (int64, error) {
	return parseSize(size, decimalKUnit)
}

// Parses a human-readable string representing an amount of RAM
// in bytes, kibibytes, mebibytes, gibibytes, or tebibytes and
// returns the number of bytes, or -1 if the string is unparseable.
// Units are case-insensitive, and the 'b' suffix is optional.
func RAMInBytes(size string) (int64, error) {
	return parseSize(size, binaryKUnit)
}

// Parses the human-readable size string into the amount it represents given
// the desired kilo unit [decimalKiloUnit=1000|binaryKiloUnit=1024]
func parseSize(size string, kUnit int64) (int64, error) {
	matches := sizeRegex.FindStringSubmatch(size)

	if len(matches) != 3 {
		return -1, fmt.Errorf("Invalid size: '%s'", size)
	}

	theSize, err := strconv.ParseInt(matches[1], 10, 0)
	if err != nil {
		return -1, err
	}

	unitPrefix := strings.ToLower(matches[2])

	switch unitPrefix {
	case "k":
		theSize *= kUnit
	case "m":
		theSize *= kUnit * kUnit
	case "g":
		theSize *= kUnit * kUnit * kUnit
	case "t":
		theSize *= kUnit * kUnit * kUnit * kUnit
	case "p":
		theSize *= kUnit * kUnit * kUnit * kUnit * kUnit
	}

	return theSize, nil
}
<|endoftext|>


<commit_before>package todoist

import (
	"context"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	linkRegex = regexp.MustCompile(`\[(.*)\]\((.*)\)`)
)

const (
	RFC3339Date = "2006-01-02"
	RFC3339DateTime = "2006-01-02T15:04:05"
)

type Due struct {
	Date        string `json:"date"`
	TimeZone    string `json:"timezone"`
	IsRecurring bool   `json:"is_recurring"`
	String      string `json:"string"`
	Lang        string `json:"en"`
}

type BaseItem struct {
	HaveID
	HaveProjectID
	Content string `json:"content"`
	UserID  int    `json:"user_id"`
}

func (bitem BaseItem) GetContent() string {
	return bitem.Content
}

type CompletedItem struct {
	BaseItem
	CompletedData string      `json:"completed_date"`
	MetaData      interface{} `json:"meta_data"`
	TaskID        int         `json:"task_id"`
}

func (item CompletedItem) DateTime() time.Time {
	t, _ := time.Parse(time.RFC3339, item.CompletedData)
	return t
}

func (item CompletedItem) GetProjectID() int {
	return item.ProjectID
}

func (item CompletedItem) GetLabelIDs() []int {
	return []int{}
}

type CompletedItems []CompletedItem

type Item struct {
	BaseItem
	HaveParentID
	HaveIndent
	AllDay         bool        `json:"all_day"`
	AssignedByUID  int         `json:"assigned_by_uid"`
	Checked        int         `json:"checked"`
	Collapsed      int         `json:"collapsed"`
	DateAdded      string      `json:"date_added"`
	DateLang       string      `json:"date_lang"`
	DateString     string      `json:"date_string"`
	DayOrder       int         `json:"day_order"`
	Due            *Due        `json:"due"`
	HasMoreNotes   bool        `json:"has_more_notes"`
	InHistory      int         `json:"in_history"`
	IsArchived     int         `json:"is_archived"`
	IsDeleted      int         `json:"is_deleted"`
	ItemOrder      int         `json:"item_order"`
	LabelIDs       []int       `json:"labels"`
	Priority       int         `json:"priority"`
	AutoReminder   bool        `json:"auto_reminder"`
	ResponsibleUID interface{} `json:"responsible_uid"`
	SyncID         interface{} `json:"sync_id"`
}

type Items []Item

func (a Items) Len() int           { return len(a) }
func (a Items) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a Items) Less(i, j int) bool { return a[i].ID < a[j].ID }

func (a Items) At(i int) IDCarrier { return a[i] }

func (item Item) DateTime() time.Time {
	var date string

	if item.Due == nil {
		date = ""
	} else {
		date = item.Due.Date
	}

	t, err := time.ParseInLocation(RFC3339DateTime, date, time.Local)
	if err != nil {
		t, _ = time.ParseInLocation(RFC3339Date, date, time.Local)
	}
	return t
}

func (item Item) GetProjectID() int {
	return item.ProjectID
}

func (item Item) GetLabelIDs() []int {
	return item.LabelIDs
}

// interface for Eval actions
type AbstractItem interface {
	DateTime() time.Time
	GetProjectID() int
	GetLabelIDs() []int
}

func GetContentTitle(item ContentCarrier) string {
	return linkRegex.ReplaceAllString(item.GetContent(), "$1")
}

func GetContentURL(item ContentCarrier) string {
	if HasURL(item) {
		return linkRegex.ReplaceAllString(item.GetContent(), "$2")
	}
	return ""
}

func HasURL(item ContentCarrier) bool {
	return linkRegex.MatchString(item.GetContent())
}

func (item Item) AddParam() interface{} {
	param := map[string]interface{}{}
	if item.Content != "" {
		param["content"] = item.Content
	}
	if item.DateString != "" {
		param["date_string"] = item.DateString
	}
	if len(item.LabelIDs) != 0 {
		param["labels"] = item.LabelIDs
	}
	if item.Priority != 0 {
		param["priority"] = item.Priority
	}
	if item.ProjectID != 0 {
		param["project_id"] = item.ProjectID
	}
	param["auto_reminder"] = item.AutoReminder

	return param
}

func (item Item) UpdateParam() interface{} {
	param := map[string]interface{}{}
	if item.ID != 0 {
		param["id"] = item.ID
	}
	if item.Content != "" {
		param["content"] = item.Content
	}
	if item.DateString != "" {
		param["date_string"] = item.DateString
	}
	// TODO: more cool
	if item.DateString == "null" {
		param["date_string"] = ""
	}
	if len(item.LabelIDs) != 0 {
		param["labels"] = item.LabelIDs
	}
	if item.Priority != 0 {
		param["priority"] = item.Priority
	}
	return param
}

func (item Item) MoveParam(to_project Project) interface{} {
	param := map[string]interface{}{
		"project_items": map[string][]int{
			strconv.Itoa(item.ProjectID): []int{item.ID},
		},
		"to_project": to_project.ID,
	}
	return param
}

func (item Item) LabelsString(labels Labels) string {
	label_names := make([]string, 0)
	for _, label_id := range item.LabelIDs {
		label, err := SearchByID(labels, label_id)
		if err != nil {
			return "Error"
		}
		label_names = append(label_names, "@"+label.(Label).Name)
	}
	return strings.Join(label_names, ",")
}

func (c *Client) AddItem(ctx context.Context, item Item) error {
	commands := Commands{
		NewCommand("item_add", item.AddParam()),
	}
	return c.ExecCommands(ctx, commands)
}

func (c *Client) UpdateItem(ctx context.Context, item Item) error {
	commands := Commands{
		NewCommand("item_update", item.UpdateParam()),
	}
	return c.ExecCommands(ctx, commands)
}

func (c *Client) CloseItem(ctx context.Context, ids []int) error {
	var commands Commands
	for _, id := range ids {
		command := NewCommand("item_close", map[string]interface{}{"id": id})
		commands = append(commands, command)
	}
	return c.ExecCommands(ctx, commands)
}

func (c *Client) DeleteItem(ctx context.Context, ids []int) error {
	commands := Commands{
		NewCommand("item_delete", map[string]interface{}{"ids": ids}),
	}
	return c.ExecCommands(ctx, commands)
}

func (c *Client) MoveItem(ctx context.Context, item Item, to_project Project) error {
	commands := Commands{
		NewCommand("item_move", item.MoveParam(to_project)),
	}
	return c.ExecCommands(ctx, commands)
}
<commit_msg>Use id parameter<commit_after>package todoist

import (
	"context"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	linkRegex = regexp.MustCompile(`\[(.*)\]\((.*)\)`)
)

const (
	RFC3339Date = "2006-01-02"
	RFC3339DateTime = "2006-01-02T15:04:05"
)

type Due struct {
	Date        string `json:"date"`
	TimeZone    string `json:"timezone"`
	IsRecurring bool   `json:"is_recurring"`
	String      string `json:"string"`
	Lang        string `json:"en"`
}

type BaseItem struct {
	HaveID
	HaveProjectID
	Content string `json:"content"`
	UserID  int    `json:"user_id"`
}

func (bitem BaseItem) GetContent() string {
	return bitem.Content
}

type CompletedItem struct {
	BaseItem
	CompletedData string      `json:"completed_date"`
	MetaData      interface{} `json:"meta_data"`
	TaskID        int         `json:"task_id"`
}

func (item CompletedItem) DateTime() time.Time {
	t, _ := time.Parse(time.RFC3339, item.CompletedData)
	return t
}

func (item CompletedItem) GetProjectID() int {
	return item.ProjectID
}

func (item CompletedItem) GetLabelIDs() []int {
	return []int{}
}

type CompletedItems []CompletedItem

type Item struct {
	BaseItem
	HaveParentID
	HaveIndent
	AllDay         bool        `json:"all_day"`
	AssignedByUID  int         `json:"assigned_by_uid"`
	Checked        int         `json:"checked"`
	Collapsed      int         `json:"collapsed"`
	DateAdded      string      `json:"date_added"`
	DateLang       string      `json:"date_lang"`
	DateString     string      `json:"date_string"`
	DayOrder       int         `json:"day_order"`
	Due            *Due        `json:"due"`
	HasMoreNotes   bool        `json:"has_more_notes"`
	InHistory      int         `json:"in_history"`
	IsArchived     int         `json:"is_archived"`
	IsDeleted      int         `json:"is_deleted"`
	ItemOrder      int         `json:"item_order"`
	LabelIDs       []int       `json:"labels"`
	Priority       int         `json:"priority"`
	AutoReminder   bool        `json:"auto_reminder"`
	ResponsibleUID interface{} `json:"responsible_uid"`
	SyncID         interface{} `json:"sync_id"`
}

type Items []Item

func (a Items) Len() int           { return len(a) }
func (a Items) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a Items) Less(i, j int) bool { return a[i].ID < a[j].ID }

func (a Items) At(i int) IDCarrier { return a[i] }

func (item Item) DateTime() time.Time {
	var date string

	if item.Due == nil {
		date = ""
	} else {
		date = item.Due.Date
	}

	t, err := time.ParseInLocation(RFC3339DateTime, date, time.Local)
	if err != nil {
		t, _ = time.ParseInLocation(RFC3339Date, date, time.Local)
	}
	return t
}

func (item Item) GetProjectID() int {
	return item.ProjectID
}

func (item Item) GetLabelIDs() []int {
	return item.LabelIDs
}

// interface for Eval actions
type AbstractItem interface {
	DateTime() time.Time
	GetProjectID() int
	GetLabelIDs() []int
}

func GetContentTitle(item ContentCarrier) string {
	return linkRegex.ReplaceAllString(item.GetContent(), "$1")
}

func GetContentURL(item ContentCarrier) string {
	if HasURL(item) {
		return linkRegex.ReplaceAllString(item.GetContent(), "$2")
	}
	return ""
}

func HasURL(item ContentCarrier) bool {
	return linkRegex.MatchString(item.GetContent())
}

func (item Item) AddParam() interface{} {
	param := map[string]interface{}{}
	if item.Content != "" {
		param["content"] = item.Content
	}
	if item.DateString != "" {
		param["date_string"] = item.DateString
	}
	if len(item.LabelIDs) != 0 {
		param["labels"] = item.LabelIDs
	}
	if item.Priority != 0 {
		param["priority"] = item.Priority
	}
	if item.ProjectID != 0 {
		param["project_id"] = item.ProjectID
	}
	param["auto_reminder"] = item.AutoReminder

	return param
}

func (item Item) UpdateParam() interface{} {
	param := map[string]interface{}{}
	if item.ID != 0 {
		param["id"] = item.ID
	}
	if item.Content != "" {
		param["content"] = item.Content
	}
	if item.DateString != "" {
		param["date_string"] = item.DateString
	}
	// TODO: more cool
	if item.DateString == "null" {
		param["date_string"] = ""
	}
	if len(item.LabelIDs) != 0 {
		param["labels"] = item.LabelIDs
	}
	if item.Priority != 0 {
		param["priority"] = item.Priority
	}
	return param
}

func (item Item) MoveParam(to_project Project) interface{} {
	param := map[string]interface{}{
		"project_items": map[string][]int{
			strconv.Itoa(item.ProjectID): []int{item.ID},
		},
		"to_project": to_project.ID,
	}
	return param
}

func (item Item) LabelsString(labels Labels) string {
	label_names := make([]string, 0)
	for _, label_id := range item.LabelIDs {
		label, err := SearchByID(labels, label_id)
		if err != nil {
			return "Error"
		}
		label_names = append(label_names, "@"+label.(Label).Name)
	}
	return strings.Join(label_names, ",")
}

func (c *Client) AddItem(ctx context.Context, item Item) error {
	commands := Commands{
		NewCommand("item_add", item.AddParam()),
	}
	return c.ExecCommands(ctx, commands)
}

func (c *Client) UpdateItem(ctx context.Context, item Item) error {
	commands := Commands{
		NewCommand("item_update", item.UpdateParam()),
	}
	return c.ExecCommands(ctx, commands)
}

func (c *Client) CloseItem(ctx context.Context, ids []int) error {
	var commands Commands
	for _, id := range ids {
		command := NewCommand("item_close", map[string]interface{}{"id": id})
		commands = append(commands, command)
	}
	return c.ExecCommands(ctx, commands)
}

func (c *Client) DeleteItem(ctx context.Context, ids []int) error {
	var commands Commands
	for _, id := range ids {
		command := NewCommand("item_delete", map[string]interface{}{"id": id})
		commands = append(commands, command)
	}
	return c.ExecCommands(ctx, commands)
}

func (c *Client) MoveItem(ctx context.Context, item Item, to_project Project) error {
	commands := Commands{
		NewCommand("item_move", item.MoveParam(to_project)),
	}
	return c.ExecCommands(ctx, commands)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"text/tabwriter"
)

type Commenter struct {
	LineComment string
	StartComment string
	EndComment string
	Nesting bool
}

var (
	noComments = Commenter{`\0`, `\0`, `\0`, false}
	cComments = Commenter{`//`, `/*`, `*/`, false}
)

type Language struct {
	Namer
	Matcher
	LineComment Commenter
}

var (
	blankR = regexp.MustCompile(`^[ \t]*$`)
)

func (l Language) Update(c []byte, s *Stats) {
	s.FileCount++
	bb := bytes.Buffer{}
	for _, b := range c {
		if b == byte('\n') {
			s.TotalLines++
			if blankR.Match(bb.Bytes()) { s.BlankLines++ }
			bb.Reset()
			continue
		}
		bb.WriteByte(b)
	}
}

type Namer string

func (l Namer) Name() string { return string(l) }

type Matcher func(string) bool

func (m Matcher) Match(fname string) bool { return m(fname) }

func mExt(exts ...string) Matcher {
	return func(fname string) bool {
		for _, ext := range exts {
			if ext == path.Ext(fname) {
				return true
			}
		}
		return false
	}
}

func mName(names ...string) Matcher {
	return func(fname string) bool {
		for _, name := range names {
			if name == path.Base(fname) {
				return true
			}
		}
		return false
	}
}

type Stats struct{
	FileCount    int
	TotalLines   int
	CodeLines    int
	BlankLines   int
	CommentLines int
}

var info = map[string]*Stats{}

var languages = []Language{
	Language{"C", mExt(".c", ".h"), cComments},
	Language{"C++", mExt(".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx"), cComments},
	Language{"Go", mExt(".go"), cComments},
	Language{"Haskell", mExt(".hs", ".lhs"), noComments},
	Language{"Perl", mExt(".pl", ".pm"), noComments},
	Language{"Python", mExt(".py"), noComments},
	Language{"Lisp", mExt(".lsp"), noComments},
	Language{"Make", mName("makefile", "Makefile", "MAKEFILE"), noComments},
	Language{"HTML", mExt(".htm", ".html", ".xhtml"), noComments},
}

func handleFile(fname, content string) {
	var l Language
	ok := false
	for _, lang := range languages {
		if lang.Match(fname) {
			ok = true
			l = lang
			break
		}
	}
	if !ok {
		return // ignore this file
	}
	i, ok := info[l.Name()]
	if !ok {
		i = &Stats{}
		info[l.Name()] = i
	}
	c, err := ioutil.ReadFile(fname)
	if err != nil {
		fmt.Fprintf(os.Stderr, "  ! %s\n", fname)
		return
	}
	l.Update(c, i)
}

func printInfo() {
	w := tabwriter.NewWriter(os.Stdout, 2, 8, 2, ' ', tabwriter.AlignRight)
	fmt.Fprintln(w, "Language\tFiles\tCode\tComment\tBlank\tTotal\t")
	for n, i := range info {
		fmt.Fprintf(
			w,
			"%s\t%d\t%d\t%d\t%d\t%d\t\n",
			n,
			i.FileCount,
			i.CodeLines,
			i.CommentLines,
			i.BlankLines,
			i.TotalLines)
	}
	w.Flush()
}

var files []string

func add(n string) {
	fi, err := os.Stat(n)
	if err != nil {
		goto invalid
	}
	if fi.IsDir() {
		fs, err := ioutil.ReadDir(n)
		if err != nil {
			goto invalid
		}
		for _, f := range fs {
			if f.Name()[0] != '.' {
				add(path.Join(n, f.Name()))
			}
		}
		return
	}
	if fi.Mode() & os.ModeType == 0 {
		files = append(files, n)
		return
	}

	println(fi.Mode())

invalid:
	fmt.Fprintf(os.Stderr, "  ! %s\n", n)
}

func main() {
	flag.Parse()

	args := flag.Args()
	if len(args) == 0 {
		args = append(args, `.`)
	}

	for _, n := range args {
		add(n)
	}

	for _, f := range files {
		c, err := ioutil.ReadFile(f)
		if err != nil {
			fmt.Fprintf(os.Stderr, "  ! %s\n", f)
		}
		cs := string(c)
		handleFile(f, cs)
	}
	printInfo()
}
<commit_msg>More optimizations<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"text/tabwriter"
)

type Commenter struct {
	LineComment  string
	StartComment string
	EndComment   string
	Nesting      bool
}

var (
	noComments = Commenter{`\0`, `\0`, `\0`, false}
	cComments  = Commenter{`//`, `/*`, `*/`, false}
)

type Language struct {
	Namer
	Matcher
	Commenter
}

var (
	blankR = regexp.MustCompile(`^[ \t]*$`)
)

func (l Language) Update(c []byte, s *Stats) {
	s.FileCount++
	// line pointers
	lStart := 0

	//lc := []byte(l.LineComment)
	//sc := []byte(l.StartComment)
	//ec := []byte(l.EndComment)

	for i, b := range c {
		if b == byte('\n') {
			s.TotalLines++
			if blankR.Match(c[lStart:i]) {
				s.BlankLines++
			}
			lStart = i+1
			continue
		}
	}
}

type Namer string

func (l Namer) Name() string { return string(l) }

type Matcher func(string) bool

func (m Matcher) Match(fname string) bool { return m(fname) }

func mExt(exts ...string) Matcher {
	return func(fname string) bool {
		for _, ext := range exts {
			if ext == path.Ext(fname) {
				return true
			}
		}
		return false
	}
}

func mName(names ...string) Matcher {
	return func(fname string) bool {
		for _, name := range names {
			if name == path.Base(fname) {
				return true
			}
		}
		return false
	}
}

type Stats struct {
	FileCount    int
	TotalLines   int
	CodeLines    int
	BlankLines   int
	CommentLines int
}

var info = map[string]*Stats{}

var languages = []Language{
	Language{"C", mExt(".c", ".h"), cComments},
	Language{"C++", mExt(".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx"), cComments},
	Language{"Go", mExt(".go"), cComments},
	Language{"Haskell", mExt(".hs", ".lhs"), noComments},
	Language{"Perl", mExt(".pl", ".pm"), noComments},
	Language{"Python", mExt(".py"), noComments},
	Language{"Lisp", mExt(".lsp"), noComments},
	Language{"Make", mName("makefile", "Makefile", "MAKEFILE"), noComments},
	Language{"HTML", mExt(".htm", ".html", ".xhtml"), noComments},
}

func handleFile(fname, content string) {
	var l Language
	ok := false
	for _, lang := range languages {
		if lang.Match(fname) {
			ok = true
			l = lang
			break
		}
	}
	if !ok {
		return // ignore this file
	}
	i, ok := info[l.Name()]
	if !ok {
		i = &Stats{}
		info[l.Name()] = i
	}
	c, err := ioutil.ReadFile(fname)
	if err != nil {
		fmt.Fprintf(os.Stderr, "  ! %s\n", fname)
		return
	}
	l.Update(c, i)
}

func printInfo() {
	w := tabwriter.NewWriter(os.Stdout, 2, 8, 2, ' ', tabwriter.AlignRight)
	fmt.Fprintln(w, "Language\tFiles\tCode\tComment\tBlank\tTotal\t")
	for n, i := range info {
		fmt.Fprintf(
			w,
			"%s\t%d\t%d\t%d\t%d\t%d\t\n",
			n,
			i.FileCount,
			i.CodeLines,
			i.CommentLines,
			i.BlankLines,
			i.TotalLines)
	}
	w.Flush()
}

var files []string

func add(n string) {
	fi, err := os.Stat(n)
	if err != nil {
		goto invalid
	}
	if fi.IsDir() {
		fs, err := ioutil.ReadDir(n)
		if err != nil {
			goto invalid
		}
		for _, f := range fs {
			if f.Name()[0] != '.' {
				add(path.Join(n, f.Name()))
			}
		}
		return
	}
	if fi.Mode()&os.ModeType == 0 {
		files = append(files, n)
		return
	}

	println(fi.Mode())

invalid:
	fmt.Fprintf(os.Stderr, "  ! %s\n", n)
}

func main() {
	flag.Parse()

	args := flag.Args()
	if len(args) == 0 {
		args = append(args, `.`)
	}

	for _, n := range args {
		add(n)
	}

	for _, f := range files {
		c, err := ioutil.ReadFile(f)
		if err != nil {
			fmt.Fprintf(os.Stderr, "  ! %s\n", f)
		}
		cs := string(c)
		handleFile(f, cs)
	}
	printInfo()
}
<|endoftext|>


<commit_before>package main

import (
	twodee "../libs/twodee"
	"github.com/go-gl/gl"
	"runtime"
	"time"
)

func init() {
	// See https://code.google.com/p/go/issues/detail?id=3527
	runtime.LockOSThread()
}

type Application struct {
	layers                *twodee.Layers
	Context               *twodee.Context
	AudioSystem           *AudioSystem
	WinBounds             twodee.Rectangle
	GameEventHandler      *twodee.GameEventHandler
	gameClosingObserverId int
	InitiateCloseGame     bool
}

func NewApplication() (app *Application, err error) {
	var (
		layers            *twodee.Layers
		context           *twodee.Context
		gameLayer         *GameLayer
		hudLayer          *HudLayer
		winbounds         = twodee.Rect(0, 0, 1024, 768)
		gameEventHandler  = twodee.NewGameEventHandler(NumGameEventTypes)
		initiateCloseGame = false
	)
	if context, err = twodee.NewContext(); err != nil {
		return
	}
	context.SetFullscreen(false)
	context.SetCursor(true)
	if err = context.CreateWindow(int(winbounds.Max.X), int(winbounds.Max.Y), "LD30"); err != nil {
		return
	}
	layers = twodee.NewLayers()
	app = &Application{
		layers:            layers,
		Context:           context,
		WinBounds:         winbounds,
		GameEventHandler:  gameEventHandler,
		InitiateCloseGame: initiateCloseGame,
	}
	if gameLayer, err = NewGameLayer(app); err != nil {
		return
	}
	if hudLayer, err = NewHudLayer(app, gameLayer); err != nil {
		return
	}
	if app.AudioSystem, err = NewAudioSystem(app); err != nil {
		return
	}

	layers.Push(gameLayer)
	layers.Push(hudLayer)
	app.gameClosingObserverId = app.GameEventHandler.AddObserver(GameIsClosing, app.CloseGame)
	app.GameEventHandler.Enqueue(twodee.NewBasicGameEvent(PlayBackgroundMusic))
	return
}

func (a *Application) Draw() {
	gl.Clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT)
	a.layers.Render()
}

func (a *Application) Update(elapsed time.Duration) {
	a.layers.Update(elapsed)
}

func (a *Application) Delete() {
	a.GameEventHandler.RemoveObserver(GameIsClosing, a.gameClosingObserverId)
	a.layers.Delete()
	a.AudioSystem.Delete()
	a.Context.Delete()
}

func (a *Application) ProcessEvents() {
	var (
		evt  twodee.Event
		loop = true
	)
	for loop {
		select {
		case evt = <-a.Context.Events.Events:
			a.layers.HandleEvent(evt)
		default:
			// No more events
			loop = false
		}
	}
}

func (a *Application) CloseGame(e twodee.GETyper) {
	a.InitiateCloseGame = true
}

func main() {
	var (
		app *Application
		err error
	)

	if app, err = NewApplication(); err != nil {
		panic(err)
	}
	defer app.Delete()

	var (
		last_render  = time.Now()
		current_time = time.Now()
		updated_to   = current_time
		step         = twodee.Step60Hz
		render_max   = step
	)
	for !app.Context.Window.ShouldClose() && !app.InitiateCloseGame {
		for !updated_to.After(current_time) {
			app.Update(step)
			updated_to = updated_to.Add(step)
		}
		if diff := current_time.Sub(last_render); diff < render_max {
			time.Sleep(render_max - diff)
		}
		app.Draw()
		app.Context.Window.SwapBuffers()
		last_render = current_time
		app.Context.Events.Poll()
		app.GameEventHandler.Poll()
		app.ProcessEvents()
		current_time = time.Now()
	}
}
<commit_msg>Make main game loop smoother.<commit_after>package main

import (
	twodee "../libs/twodee"
	"github.com/go-gl/gl"
	"runtime"
	"time"
)

func init() {
	// See https://code.google.com/p/go/issues/detail?id=3527
	runtime.LockOSThread()
}

type Application struct {
	layers                *twodee.Layers
	Context               *twodee.Context
	AudioSystem           *AudioSystem
	WinBounds             twodee.Rectangle
	GameEventHandler      *twodee.GameEventHandler
	gameClosingObserverId int
	InitiateCloseGame     bool
}

func NewApplication() (app *Application, err error) {
	var (
		layers            *twodee.Layers
		context           *twodee.Context
		gameLayer         *GameLayer
		hudLayer          *HudLayer
		winbounds         = twodee.Rect(0, 0, 1024, 768)
		gameEventHandler  = twodee.NewGameEventHandler(NumGameEventTypes)
		initiateCloseGame = false
	)
	if context, err = twodee.NewContext(); err != nil {
		return
	}
	context.SetFullscreen(false)
	context.SetCursor(true)
	if err = context.CreateWindow(int(winbounds.Max.X), int(winbounds.Max.Y), "LD30"); err != nil {
		return
	}
	layers = twodee.NewLayers()
	app = &Application{
		layers:            layers,
		Context:           context,
		WinBounds:         winbounds,
		GameEventHandler:  gameEventHandler,
		InitiateCloseGame: initiateCloseGame,
	}
	if gameLayer, err = NewGameLayer(app); err != nil {
		return
	}
	if hudLayer, err = NewHudLayer(app, gameLayer); err != nil {
		return
	}
	if app.AudioSystem, err = NewAudioSystem(app); err != nil {
		return
	}

	layers.Push(gameLayer)
	layers.Push(hudLayer)
	app.gameClosingObserverId = app.GameEventHandler.AddObserver(GameIsClosing, app.CloseGame)
	app.GameEventHandler.Enqueue(twodee.NewBasicGameEvent(PlayBackgroundMusic))
	return
}

func (a *Application) Draw() {
	gl.Clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT)
	a.layers.Render()
}

func (a *Application) Update(elapsed time.Duration) {
	a.layers.Update(elapsed)
}

func (a *Application) Delete() {
	a.GameEventHandler.RemoveObserver(GameIsClosing, a.gameClosingObserverId)
	a.layers.Delete()
	a.AudioSystem.Delete()
	a.Context.Delete()
}

func (a *Application) ProcessEvents() {
	var (
		evt  twodee.Event
		loop = true
	)
	for loop {
		select {
		case evt = <-a.Context.Events.Events:
			a.layers.HandleEvent(evt)
		default:
			// No more events
			loop = false
		}
	}
}

func (a *Application) CloseGame(e twodee.GETyper) {
	a.InitiateCloseGame = true
}

func main() {
	var (
		app *Application
		err error
	)

	if app, err = NewApplication(); err != nil {
		panic(err)
	}
	defer app.Delete()

	var (
		last_render  = time.Now()
		current_time = time.Now()
		updated_to   = current_time
		step         = twodee.Step60Hz
		render_max   = step
	)
	for !app.Context.Window.ShouldClose() && !app.InitiateCloseGame {
		for !updated_to.After(current_time) {
			app.Update(step)
			updated_to = updated_to.Add(step)
		}
		if diff := current_time.Sub(last_render); diff < render_max {
			//time.Sleep(render_max - diff)
			time.Sleep(1)
		} else {
			app.Draw()
			app.Context.Window.SwapBuffers()
			last_render = current_time
		}
		app.Context.Events.Poll()
		app.GameEventHandler.Poll()
		app.ProcessEvents()
		current_time = time.Now()
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2020, Cossack Labs Limited
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package keystore

import (
	"path/filepath"

	"github.com/cossacklabs/acra/utils"
	"github.com/cossacklabs/acra/zone"
	"github.com/cossacklabs/themis/gothemis/keys"
)

//
// PublicKeyStore interface (zones)
//

// GetZonePublicKey retrieves public key used to encrypt data in given zone.
func (s *ServerKeyStore) GetZonePublicKey(zoneID []byte) (*keys.PublicKey, error) {
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRing(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return nil, err
	}
	publicKey, err := s.currentPairPublicKey(ring)
	if err != nil {
		log.WithError(err).Debug("failed to get current storage public key for zone")
		return nil, err
	}
	return publicKey, nil
}

//
// PrivateKeyStore interface (zones)
//

// GetZonePrivateKey retrieves private key used to decrypt data in given zone.
func (s *ServerKeyStore) GetZonePrivateKey(zoneID []byte) (*keys.PrivateKey, error) {
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRing(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return nil, err
	}
	privateKey, err := s.currentPairPrivateKey(ring)
	if err != nil {
		log.WithError(err).Debug("failed to get current storage private key for zone")
		return nil, err
	}
	return privateKey, nil
}

// GetZonePrivateKeys retrieves all private key used to decrypt data in given zone.
// The keys are returned from newest to oldest.
func (s *ServerKeyStore) GetZonePrivateKeys(zoneID []byte) ([]*keys.PrivateKey, error) {
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRing(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return nil, err
	}
	privateKeys, err := s.allPairPrivateKeys(ring)
	if err != nil {
		log.WithError(err).Debug("failed to get storage private key for zone")
		return nil, err
	}
	return privateKeys, nil
}

// HasZonePrivateKey returns true if there is a private key used to decrypt data in given zone.
func (s *ServerKeyStore) HasZonePrivateKey(zoneID []byte) bool {
	ring, err := s.OpenKeyRing(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		return false
	}
	present, _ := s.hasCurrentKey(ring)
	return present
}

//
// StorageKeyCreation interface (zones)
//

const zonePrefix = "client"

func (s *ServerKeyStore) zoneStorageKeyPairPath(zoneID []byte) string {
	return filepath.Join(zonePrefix, string(zoneID), storageSuffix)
}

// GenerateZoneKey generates new zone and a storage key for it.
// Returns zone ID followed by public key data.
func (s *ServerKeyStore) GenerateZoneKey() ([]byte, []byte, error) {
	var zoneID []byte
	for {
		zoneID = zone.GenerateZoneID()
		if !s.HasZonePrivateKey(zoneID) {
			break
		}
	}
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRingRW(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return nil, nil, err
	}
	pair, err := s.newCurrentKeyPair(ring)
	if err != nil {
		log.WithError(err).Debug("failed to generate storage key pair for zone")
		return nil, nil, err
	}
	utils.FillSlice(0, pair.Private.Value)
	return zoneID, pair.Public.Value, nil
}

// SaveZoneKeypair overwrites storage keypair used in given zone.
func (s *ServerKeyStore) SaveZoneKeypair(zoneID []byte, keypair *keys.Keypair) error {
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRingRW(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return err
	}
	err = s.addCurrentKeyPair(ring, keypair)
	if err != nil {
		log.WithError(err).Debug("failed to set storage key pair for zone")
		return err
	}
	return nil
}

// RotateZoneKey generates a new storage key pair for given zone.
// Returns new public key data.
func (s *ServerKeyStore) RotateZoneKey(zoneID []byte) ([]byte, error) {
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRingRW(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return nil, err
	}
	pair, err := s.newCurrentKeyPair(ring)
	if err != nil {
		log.WithError(err).Debug("failed to rotate storage key pair for zone")
		return nil, err
	}
	utils.FillSlice(0, pair.Private.Value)
	return pair.Public.Value, nil
}
<commit_msg>Use correct zone prefix for key store v2 (#385)<commit_after>/*
 * Copyright 2020, Cossack Labs Limited
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package keystore

import (
	"path/filepath"

	"github.com/cossacklabs/acra/utils"
	"github.com/cossacklabs/acra/zone"
	"github.com/cossacklabs/themis/gothemis/keys"
)

//
// PublicKeyStore interface (zones)
//

// GetZonePublicKey retrieves public key used to encrypt data in given zone.
func (s *ServerKeyStore) GetZonePublicKey(zoneID []byte) (*keys.PublicKey, error) {
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRing(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return nil, err
	}
	publicKey, err := s.currentPairPublicKey(ring)
	if err != nil {
		log.WithError(err).Debug("failed to get current storage public key for zone")
		return nil, err
	}
	return publicKey, nil
}

//
// PrivateKeyStore interface (zones)
//

// GetZonePrivateKey retrieves private key used to decrypt data in given zone.
func (s *ServerKeyStore) GetZonePrivateKey(zoneID []byte) (*keys.PrivateKey, error) {
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRing(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return nil, err
	}
	privateKey, err := s.currentPairPrivateKey(ring)
	if err != nil {
		log.WithError(err).Debug("failed to get current storage private key for zone")
		return nil, err
	}
	return privateKey, nil
}

// GetZonePrivateKeys retrieves all private key used to decrypt data in given zone.
// The keys are returned from newest to oldest.
func (s *ServerKeyStore) GetZonePrivateKeys(zoneID []byte) ([]*keys.PrivateKey, error) {
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRing(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return nil, err
	}
	privateKeys, err := s.allPairPrivateKeys(ring)
	if err != nil {
		log.WithError(err).Debug("failed to get storage private key for zone")
		return nil, err
	}
	return privateKeys, nil
}

// HasZonePrivateKey returns true if there is a private key used to decrypt data in given zone.
func (s *ServerKeyStore) HasZonePrivateKey(zoneID []byte) bool {
	ring, err := s.OpenKeyRing(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		return false
	}
	present, _ := s.hasCurrentKey(ring)
	return present
}

//
// StorageKeyCreation interface (zones)
//

const zonePrefix = "zone"

func (s *ServerKeyStore) zoneStorageKeyPairPath(zoneID []byte) string {
	return filepath.Join(zonePrefix, string(zoneID), storageSuffix)
}

// GenerateZoneKey generates new zone and a storage key for it.
// Returns zone ID followed by public key data.
func (s *ServerKeyStore) GenerateZoneKey() ([]byte, []byte, error) {
	var zoneID []byte
	for {
		zoneID = zone.GenerateZoneID()
		if !s.HasZonePrivateKey(zoneID) {
			break
		}
	}
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRingRW(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return nil, nil, err
	}
	pair, err := s.newCurrentKeyPair(ring)
	if err != nil {
		log.WithError(err).Debug("failed to generate storage key pair for zone")
		return nil, nil, err
	}
	utils.FillSlice(0, pair.Private.Value)
	return zoneID, pair.Public.Value, nil
}

// SaveZoneKeypair overwrites storage keypair used in given zone.
func (s *ServerKeyStore) SaveZoneKeypair(zoneID []byte, keypair *keys.Keypair) error {
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRingRW(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return err
	}
	err = s.addCurrentKeyPair(ring, keypair)
	if err != nil {
		log.WithError(err).Debug("failed to set storage key pair for zone")
		return err
	}
	return nil
}

// RotateZoneKey generates a new storage key pair for given zone.
// Returns new public key data.
func (s *ServerKeyStore) RotateZoneKey(zoneID []byte) ([]byte, error) {
	log := s.log.WithField("zoneID", zoneID)
	ring, err := s.OpenKeyRingRW(s.zoneStorageKeyPairPath(zoneID))
	if err != nil {
		log.WithError(err).Debug("failed to open storage key ring for zone")
		return nil, err
	}
	pair, err := s.newCurrentKeyPair(ring)
	if err != nil {
		log.WithError(err).Debug("failed to rotate storage key pair for zone")
		return nil, err
	}
	utils.FillSlice(0, pair.Private.Value)
	return pair.Public.Value, nil
}
<|endoftext|>