<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package exec runs external commands. It wraps os.StartProcess to make it
// easier to remap stdin and stdout, connect I/O with pipes, and do other
// adjustments.
package exec

import (
	"bytes"
	"errors"
	"io"
	"os"
	"strconv"
	"syscall"
)

// Error records the name of a binary that failed to be be executed
// and the reason it failed.
type Error struct {
	Name string
	Err  error
}

func (e *Error) Error() string {
	return "exec: " + strconv.Quote(e.Name) + ": " + e.Err.Error()
}

// Cmd represents an external command being prepared or run.
type Cmd struct {
	// Path is the path of the command to run.
	//
	// This is the only field that must be set to a non-zero
	// value.
	Path string

	// Args holds command line arguments, including the command as Args[0].
	// If the Args field is empty or nil, Run uses {Path}.
	//
	// In typical use, both Path and Args are set by calling Command.
	Args []string

	// Env specifies the environment of the process.
	// If Env is nil, Run uses the current process's environment.
	Env []string

	// Dir specifies the working directory of the command.
	// If Dir is the empty string, Run runs the command in the
	// calling process's current directory.
	Dir string

	// Stdin specifies the process's standard input. If Stdin is
	// nil, the process reads from the null device (os.DevNull).
	Stdin io.Reader

	// Stdout and Stderr specify the process's standard output and error.
	//
	// If either is nil, Run connects the corresponding file descriptor
	// to the null device (os.DevNull).
	//
	// If Stdout and Stderr are are the same writer, at most one
	// goroutine at a time will call Write.
	Stdout io.Writer
	Stderr io.Writer

	// ExtraFiles specifies additional open files to be inherited by the
	// new process. It does not include standard input, standard output, or
	// standard error. If non-nil, entry i becomes file descriptor 3+i.
	//
	// BUG: on OS X 10.6, child processes may sometimes inherit extra fds.
	// http://golang.org/issue/2603
	ExtraFiles []*os.File

	// SysProcAttr holds optional, operating system-specific attributes.
	// Run passes it to os.StartProcess as the os.ProcAttr's Sys field.
	SysProcAttr *syscall.SysProcAttr

	// Process is the underlying process, once started.
	Process *os.Process

	err             error // last error (from LookPath, stdin, stdout, stderr)
	finished        bool  // when Wait was called
	childFiles      []*os.File
	closeAfterStart []io.Closer
	closeAfterWait  []io.Closer
	goroutine       []func() error
	errch           chan error // one send per goroutine
}

// Command returns the Cmd struct to execute the named program with
// the given arguments.
//
// It sets Path and Args in the returned structure and zeroes the
// other fields.
//
// If name contains no path separators, Command uses LookPath to
// resolve the path to a complete name if possible. Otherwise it uses
// name directly.
//
// The returned Cmd's Args field is constructed from the command name
// followed by the elements of arg, so arg should not include the
// command name itself. For example, Command("echo", "hello")
func Command(name string, arg ...string) *Cmd {
	aname, err := LookPath(name)
	if err != nil {
		aname = name
	}
	return &Cmd{
		Path: aname,
		Args: append([]string{name}, arg...),
		err:  err,
	}
}

// interfaceEqual protects against panics from doing equality tests on
// two interfaces with non-comparable underlying types
func interfaceEqual(a, b interface{}) bool {
	defer func() {
		recover()
	}()
	return a == b
}

func (c *Cmd) envv() []string {
	if c.Env != nil {
		return c.Env
	}
	return os.Environ()
}

func (c *Cmd) argv() []string {
	if len(c.Args) > 0 {
		return c.Args
	}
	return []string{c.Path}
}

func (c *Cmd) stdin() (f *os.File, err error) {
	if c.Stdin == nil {
		f, err = os.Open(os.DevNull)
		c.closeAfterStart = append(c.closeAfterStart, f)
		return
	}

	if f, ok := c.Stdin.(*os.File); ok {
		return f, nil
	}

	pr, pw, err := os.Pipe()
	if err != nil {
		return
	}

	c.closeAfterStart = append(c.closeAfterStart, pr)
	c.closeAfterWait = append(c.closeAfterWait, pw)
	c.goroutine = append(c.goroutine, func() error {
		_, err := io.Copy(pw, c.Stdin)
		if err1 := pw.Close(); err == nil {
			err = err1
		}
		return err
	})
	return pr, nil
}

func (c *Cmd) stdout() (f *os.File, err error) {
	return c.writerDescriptor(c.Stdout)
}

func (c *Cmd) stderr() (f *os.File, err error) {
	if c.Stderr != nil && interfaceEqual(c.Stderr, c.Stdout) {
		return c.childFiles[1], nil
	}
	return c.writerDescriptor(c.Stderr)
}

func (c *Cmd) writerDescriptor(w io.Writer) (f *os.File, err error) {
	if w == nil {
		f, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0)
		c.closeAfterStart = append(c.closeAfterStart, f)
		return
	}

	if f, ok := w.(*os.File); ok {
		return f, nil
	}

	pr, pw, err := os.Pipe()
	if err != nil {
		return
	}

	c.closeAfterStart = append(c.closeAfterStart, pw)
	c.closeAfterWait = append(c.closeAfterWait, pr)
	c.goroutine = append(c.goroutine, func() error {
		_, err := io.Copy(w, pr)
		return err
	})
	return pw, nil
}

// Run starts the specified command and waits for it to complete.
//
// The returned error is nil if the command runs, has no problems
// copying stdin, stdout, and stderr, and exits with a zero exit
// status.
//
// If the command fails to run or doesn't complete successfully, the
// error is of type *ExitError. Other error types may be
// returned for I/O problems.
func (c *Cmd) Run() error {
	if err := c.Start(); err != nil {
		return err
	}
	return c.Wait()
}

// Start starts the specified command but does not wait for it to complete.
func (c *Cmd) Start() error {
	if c.err != nil {
		return c.err
	}
	if c.Process != nil {
		return errors.New("exec: already started")
	}

	type F func(*Cmd) (*os.File, error)
	for _, setupFd := range []F{(*Cmd).stdin, (*Cmd).stdout, (*Cmd).stderr} {
		fd, err := setupFd(c)
		if err != nil {
			return err
		}
		c.childFiles = append(c.childFiles, fd)
	}
	c.childFiles = append(c.childFiles, c.ExtraFiles...)

	var err error
	c.Process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{
		Dir:   c.Dir,
		Files: c.childFiles,
		Env:   c.envv(),
		Sys:   c.SysProcAttr,
	})
	if err != nil {
		return err
	}

	for _, fd := range c.closeAfterStart {
		fd.Close()
	}

	c.errch = make(chan error, len(c.goroutine))
	for _, fn := range c.goroutine {
		go func(fn func() error) {
			c.errch <- fn()
		}(fn)
	}

	return nil
}

// An ExitError reports an unsuccessful exit by a command.
type ExitError struct {
	*os.Waitmsg
}

func (e *ExitError) Error() string {
	return e.Waitmsg.String()
}

// Wait waits for the command to exit.
// It must have been started by Start.
//
// The returned error is nil if the command runs, has no problems
// copying stdin, stdout, and stderr, and exits with a zero exit
// status.
//
// If the command fails to run or doesn't complete successfully, the
// error is of type *ExitError. Other error types may be
// returned for I/O problems.
func (c *Cmd) Wait() error {
	if c.Process == nil {
		return errors.New("exec: not started")
	}
	if c.finished {
		return errors.New("exec: Wait was already called")
	}
	c.finished = true
	msg, err := c.Process.Wait(0)

	var copyError error
	for _ = range c.goroutine {
		if err := <-c.errch; err != nil && copyError == nil {
			copyError = err
		}
	}

	for _, fd := range c.closeAfterWait {
		fd.Close()
	}

	if err != nil {
		return err
	} else if !msg.Exited() || msg.ExitStatus() != 0 {
		return &ExitError{msg}
	}

	return copyError
}

// Output runs the command and returns its standard output.
func (c *Cmd) Output() ([]byte, error) {
	if c.Stdout != nil {
		return nil, errors.New("exec: Stdout already set")
	}
	var b bytes.Buffer
	c.Stdout = &b
	err := c.Run()
	return b.Bytes(), err
}

// CombinedOutput runs the command and returns its combined standard
// output and standard error.
func (c *Cmd) CombinedOutput() ([]byte, error) {
	if c.Stdout != nil {
		return nil, errors.New("exec: Stdout already set")
	}
	if c.Stderr != nil {
		return nil, errors.New("exec: Stderr already set")
	}
	var b bytes.Buffer
	c.Stdout = &b
	c.Stderr = &b
	err := c.Run()
	return b.Bytes(), err
}

// StdinPipe returns a pipe that will be connected to the command's
// standard input when the command starts.
func (c *Cmd) StdinPipe() (io.WriteCloser, error) {
	if c.Stdin != nil {
		return nil, errors.New("exec: Stdin already set")
	}
	if c.Process != nil {
		return nil, errors.New("exec: StdinPipe after process started")
	}
	pr, pw, err := os.Pipe()
	if err != nil {
		return nil, err
	}
	c.Stdin = pr
	c.closeAfterStart = append(c.closeAfterStart, pr)
	c.closeAfterWait = append(c.closeAfterWait, pw)
	return pw, nil
}

// StdoutPipe returns a pipe that will be connected to the command's
// standard output when the command starts.
// The pipe will be closed automatically after Wait sees the command exit.
func (c *Cmd) StdoutPipe() (io.ReadCloser, error) {
	if c.Stdout != nil {
		return nil, errors.New("exec: Stdout already set")
	}
	if c.Process != nil {
		return nil, errors.New("exec: StdoutPipe after process started")
	}
	pr, pw, err := os.Pipe()
	if err != nil {
		return nil, err
	}
	c.Stdout = pw
	c.closeAfterStart = append(c.closeAfterStart, pw)
	c.closeAfterWait = append(c.closeAfterWait, pr)
	return pr, nil
}

// StderrPipe returns a pipe that will be connected to the command's
// standard error when the command starts.
// The pipe will be closed automatically after Wait sees the command exit.
func (c *Cmd) StderrPipe() (io.ReadCloser, error) {
	if c.Stderr != nil {
		return nil, errors.New("exec: Stderr already set")
	}
	if c.Process != nil {
		return nil, errors.New("exec: StderrPipe after process started")
	}
	pr, pw, err := os.Pipe()
	if err != nil {
		return nil, err
	}
	c.Stderr = pw
	c.closeAfterStart = append(c.closeAfterStart, pw)
	c.closeAfterWait = append(c.closeAfterWait, pr)
	return pr, nil
}
<commit_msg>os/exec: add Cmd.Waitmsg, fix a misleading comment<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package exec runs external commands. It wraps os.StartProcess to make it
// easier to remap stdin and stdout, connect I/O with pipes, and do other
// adjustments.
package exec

import (
	"bytes"
	"errors"
	"io"
	"os"
	"strconv"
	"syscall"
)

// Error records the name of a binary that failed to be be executed
// and the reason it failed.
type Error struct {
	Name string
	Err  error
}

func (e *Error) Error() string {
	return "exec: " + strconv.Quote(e.Name) + ": " + e.Err.Error()
}

// Cmd represents an external command being prepared or run.
type Cmd struct {
	// Path is the path of the command to run.
	//
	// This is the only field that must be set to a non-zero
	// value.
	Path string

	// Args holds command line arguments, including the command as Args[0].
	// If the Args field is empty or nil, Run uses {Path}.
	//
	// In typical use, both Path and Args are set by calling Command.
	Args []string

	// Env specifies the environment of the process.
	// If Env is nil, Run uses the current process's environment.
	Env []string

	// Dir specifies the working directory of the command.
	// If Dir is the empty string, Run runs the command in the
	// calling process's current directory.
	Dir string

	// Stdin specifies the process's standard input. If Stdin is
	// nil, the process reads from the null device (os.DevNull).
	Stdin io.Reader

	// Stdout and Stderr specify the process's standard output and error.
	//
	// If either is nil, Run connects the corresponding file descriptor
	// to the null device (os.DevNull).
	//
	// If Stdout and Stderr are are the same writer, at most one
	// goroutine at a time will call Write.
	Stdout io.Writer
	Stderr io.Writer

	// ExtraFiles specifies additional open files to be inherited by the
	// new process. It does not include standard input, standard output, or
	// standard error. If non-nil, entry i becomes file descriptor 3+i.
	//
	// BUG: on OS X 10.6, child processes may sometimes inherit unwanted fds.
	// http://golang.org/issue/2603
	ExtraFiles []*os.File

	// SysProcAttr holds optional, operating system-specific attributes.
	// Run passes it to os.StartProcess as the os.ProcAttr's Sys field.
	SysProcAttr *syscall.SysProcAttr

	// Process is the underlying process, once started.
	Process *os.Process

	// Waitmsg contains information about an exited process,
	// available after a call to Wait or Run.
	Waitmsg *os.Waitmsg

	err             error // last error (from LookPath, stdin, stdout, stderr)
	finished        bool  // when Wait was called
	childFiles      []*os.File
	closeAfterStart []io.Closer
	closeAfterWait  []io.Closer
	goroutine       []func() error
	errch           chan error // one send per goroutine
}

// Command returns the Cmd struct to execute the named program with
// the given arguments.
//
// It sets Path and Args in the returned structure and zeroes the
// other fields.
//
// If name contains no path separators, Command uses LookPath to
// resolve the path to a complete name if possible. Otherwise it uses
// name directly.
//
// The returned Cmd's Args field is constructed from the command name
// followed by the elements of arg, so arg should not include the
// command name itself. For example, Command("echo", "hello")
func Command(name string, arg ...string) *Cmd {
	aname, err := LookPath(name)
	if err != nil {
		aname = name
	}
	return &Cmd{
		Path: aname,
		Args: append([]string{name}, arg...),
		err:  err,
	}
}

// interfaceEqual protects against panics from doing equality tests on
// two interfaces with non-comparable underlying types
func interfaceEqual(a, b interface{}) bool {
	defer func() {
		recover()
	}()
	return a == b
}

func (c *Cmd) envv() []string {
	if c.Env != nil {
		return c.Env
	}
	return os.Environ()
}

func (c *Cmd) argv() []string {
	if len(c.Args) > 0 {
		return c.Args
	}
	return []string{c.Path}
}

func (c *Cmd) stdin() (f *os.File, err error) {
	if c.Stdin == nil {
		f, err = os.Open(os.DevNull)
		c.closeAfterStart = append(c.closeAfterStart, f)
		return
	}

	if f, ok := c.Stdin.(*os.File); ok {
		return f, nil
	}

	pr, pw, err := os.Pipe()
	if err != nil {
		return
	}

	c.closeAfterStart = append(c.closeAfterStart, pr)
	c.closeAfterWait = append(c.closeAfterWait, pw)
	c.goroutine = append(c.goroutine, func() error {
		_, err := io.Copy(pw, c.Stdin)
		if err1 := pw.Close(); err == nil {
			err = err1
		}
		return err
	})
	return pr, nil
}

func (c *Cmd) stdout() (f *os.File, err error) {
	return c.writerDescriptor(c.Stdout)
}

func (c *Cmd) stderr() (f *os.File, err error) {
	if c.Stderr != nil && interfaceEqual(c.Stderr, c.Stdout) {
		return c.childFiles[1], nil
	}
	return c.writerDescriptor(c.Stderr)
}

func (c *Cmd) writerDescriptor(w io.Writer) (f *os.File, err error) {
	if w == nil {
		f, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0)
		c.closeAfterStart = append(c.closeAfterStart, f)
		return
	}

	if f, ok := w.(*os.File); ok {
		return f, nil
	}

	pr, pw, err := os.Pipe()
	if err != nil {
		return
	}

	c.closeAfterStart = append(c.closeAfterStart, pw)
	c.closeAfterWait = append(c.closeAfterWait, pr)
	c.goroutine = append(c.goroutine, func() error {
		_, err := io.Copy(w, pr)
		return err
	})
	return pw, nil
}

// Run starts the specified command and waits for it to complete.
//
// The returned error is nil if the command runs, has no problems
// copying stdin, stdout, and stderr, and exits with a zero exit
// status.
//
// If the command fails to run or doesn't complete successfully, the
// error is of type *ExitError. Other error types may be
// returned for I/O problems.
func (c *Cmd) Run() error {
	if err := c.Start(); err != nil {
		return err
	}
	return c.Wait()
}

// Start starts the specified command but does not wait for it to complete.
func (c *Cmd) Start() error {
	if c.err != nil {
		return c.err
	}
	if c.Process != nil {
		return errors.New("exec: already started")
	}

	type F func(*Cmd) (*os.File, error)
	for _, setupFd := range []F{(*Cmd).stdin, (*Cmd).stdout, (*Cmd).stderr} {
		fd, err := setupFd(c)
		if err != nil {
			return err
		}
		c.childFiles = append(c.childFiles, fd)
	}
	c.childFiles = append(c.childFiles, c.ExtraFiles...)

	var err error
	c.Process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{
		Dir:   c.Dir,
		Files: c.childFiles,
		Env:   c.envv(),
		Sys:   c.SysProcAttr,
	})
	if err != nil {
		return err
	}

	for _, fd := range c.closeAfterStart {
		fd.Close()
	}

	c.errch = make(chan error, len(c.goroutine))
	for _, fn := range c.goroutine {
		go func(fn func() error) {
			c.errch <- fn()
		}(fn)
	}

	return nil
}

// An ExitError reports an unsuccessful exit by a command.
type ExitError struct {
	*os.Waitmsg
}

func (e *ExitError) Error() string {
	return e.Waitmsg.String()
}

// Wait waits for the command to exit.
// It must have been started by Start.
//
// The returned error is nil if the command runs, has no problems
// copying stdin, stdout, and stderr, and exits with a zero exit
// status.
//
// If the command fails to run or doesn't complete successfully, the
// error is of type *ExitError. Other error types may be
// returned for I/O problems.
func (c *Cmd) Wait() error {
	if c.Process == nil {
		return errors.New("exec: not started")
	}
	if c.finished {
		return errors.New("exec: Wait was already called")
	}
	c.finished = true
	msg, err := c.Process.Wait(0)
	c.Waitmsg = msg

	var copyError error
	for _ = range c.goroutine {
		if err := <-c.errch; err != nil && copyError == nil {
			copyError = err
		}
	}

	for _, fd := range c.closeAfterWait {
		fd.Close()
	}

	if err != nil {
		return err
	} else if !msg.Exited() || msg.ExitStatus() != 0 {
		return &ExitError{msg}
	}

	return copyError
}

// Output runs the command and returns its standard output.
func (c *Cmd) Output() ([]byte, error) {
	if c.Stdout != nil {
		return nil, errors.New("exec: Stdout already set")
	}
	var b bytes.Buffer
	c.Stdout = &b
	err := c.Run()
	return b.Bytes(), err
}

// CombinedOutput runs the command and returns its combined standard
// output and standard error.
func (c *Cmd) CombinedOutput() ([]byte, error) {
	if c.Stdout != nil {
		return nil, errors.New("exec: Stdout already set")
	}
	if c.Stderr != nil {
		return nil, errors.New("exec: Stderr already set")
	}
	var b bytes.Buffer
	c.Stdout = &b
	c.Stderr = &b
	err := c.Run()
	return b.Bytes(), err
}

// StdinPipe returns a pipe that will be connected to the command's
// standard input when the command starts.
func (c *Cmd) StdinPipe() (io.WriteCloser, error) {
	if c.Stdin != nil {
		return nil, errors.New("exec: Stdin already set")
	}
	if c.Process != nil {
		return nil, errors.New("exec: StdinPipe after process started")
	}
	pr, pw, err := os.Pipe()
	if err != nil {
		return nil, err
	}
	c.Stdin = pr
	c.closeAfterStart = append(c.closeAfterStart, pr)
	c.closeAfterWait = append(c.closeAfterWait, pw)
	return pw, nil
}

// StdoutPipe returns a pipe that will be connected to the command's
// standard output when the command starts.
// The pipe will be closed automatically after Wait sees the command exit.
func (c *Cmd) StdoutPipe() (io.ReadCloser, error) {
	if c.Stdout != nil {
		return nil, errors.New("exec: Stdout already set")
	}
	if c.Process != nil {
		return nil, errors.New("exec: StdoutPipe after process started")
	}
	pr, pw, err := os.Pipe()
	if err != nil {
		return nil, err
	}
	c.Stdout = pw
	c.closeAfterStart = append(c.closeAfterStart, pw)
	c.closeAfterWait = append(c.closeAfterWait, pr)
	return pr, nil
}

// StderrPipe returns a pipe that will be connected to the command's
// standard error when the command starts.
// The pipe will be closed automatically after Wait sees the command exit.
func (c *Cmd) StderrPipe() (io.ReadCloser, error) {
	if c.Stderr != nil {
		return nil, errors.New("exec: Stderr already set")
	}
	if c.Process != nil {
		return nil, errors.New("exec: StderrPipe after process started")
	}
	pr, pw, err := os.Pipe()
	if err != nil {
		return nil, err
	}
	c.Stderr = pw
	c.closeAfterStart = append(c.closeAfterStart, pw)
	c.closeAfterWait = append(c.closeAfterWait, pr)
	return pr, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os_test

import (
	. "os"
	"testing"
	"syscall"
)

func TestMkdirAll(t *testing.T) {
	// Create new dir, in _obj so it will get
	// cleaned up by make if not by us.
	path := "_obj/_TestMkdirAll_/dir/./dir2"
	err := MkdirAll(path, 0777)
	if err != nil {
		t.Fatalf("MkdirAll %q: %s", path, err)
	}

	// Already exists, should succeed.
	err = MkdirAll(path, 0777)
	if err != nil {
		t.Fatalf("MkdirAll %q (second time): %s", path, err)
	}

	// Make file.
	fpath := path + "/file"
	_, err = Open(fpath, O_WRONLY|O_CREAT, 0666)
	if err != nil {
		t.Fatalf("create %q: %s", fpath, err)
	}

	// Can't make directory named after file.
	err = MkdirAll(fpath, 0777)
	if err == nil {
		t.Fatalf("MkdirAll %q: no error", fpath)
	}
	perr, ok := err.(*PathError)
	if !ok {
		t.Fatalf("MkdirAll %q returned %T, not *PathError", fpath, err)
	}
	if perr.Path != fpath {
		t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", fpath, perr.Path, fpath)
	}

	// Can't make subdirectory of file.
	ffpath := fpath + "/subdir"
	err = MkdirAll(ffpath, 0777)
	if err == nil {
		t.Fatalf("MkdirAll %q: no error", ffpath)
	}
	perr, ok = err.(*PathError)
	if !ok {
		t.Fatalf("MkdirAll %q returned %T, not *PathError", ffpath, err)
	}
	if perr.Path != fpath {
		t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", ffpath, perr.Path, fpath)
	}

	RemoveAll("_obj/_TestMkdirAll_")
}

func TestRemoveAll(t *testing.T) {
	// Work directory.
	path := "_obj/_TestRemoveAll_"
	fpath := path + "/file"
	dpath := path + "/dir"

	// Make directory with 1 file and remove.
	if err := MkdirAll(path, 0777); err != nil {
		t.Fatalf("MkdirAll %q: %s", path, err)
	}
	fd, err := Open(fpath, O_WRONLY|O_CREAT, 0666)
	if err != nil {
		t.Fatalf("create %q: %s", fpath, err)
	}
	fd.Close()
	if err = RemoveAll(path); err != nil {
		t.Fatalf("RemoveAll %q (first): %s", path, err)
	}
	if _, err := Lstat(path); err == nil {
		t.Fatalf("Lstat %q succeeded after RemoveAll (first)", path)
	}

	// Make directory with file and subdirectory and remove.
	if err = MkdirAll(dpath, 0777); err != nil {
		t.Fatalf("MkdirAll %q: %s", dpath, err)
	}
	fd, err = Open(fpath, O_WRONLY|O_CREAT, 0666)
	if err != nil {
		t.Fatalf("create %q: %s", fpath, err)
	}
	fd.Close()
	fd, err = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666)
	if err != nil {
		t.Fatalf("create %q: %s", fpath, err)
	}
	fd.Close()
	if err = RemoveAll(path); err != nil {
		t.Fatalf("RemoveAll %q (second): %s", path, err)
	}
	if _, err := Lstat(path); err == nil {
		t.Fatalf("Lstat %q succeeded after RemoveAll (second)", path)
	}

	// Determine if we should run the following test.
	testit := true
	if syscall.OS == "windows" {
		// Chmod is not supported under windows.
		testit = false
	} else {
		// Test fails as root.
		testit = Getuid() != 0
	}
	if testit {
		// Make directory with file and subdirectory and trigger error.
		if err = MkdirAll(dpath, 0777); err != nil {
			t.Fatalf("MkdirAll %q: %s", dpath, err)
		}

		for _, s := range []string{fpath, dpath + "/file1", path + "/zzz"} {
			fd, err = Open(s, O_WRONLY|O_CREAT, 0666)
			if err != nil {
				t.Fatalf("create %q: %s", s, err)
			}
			fd.Close()
		}
		if err = Chmod(dpath, 0); err != nil {
			t.Fatalf("Chmod %q 0: %s", dpath, err)
		}
		if err = RemoveAll(path); err == nil {
			_, err := Lstat(path)
			if err == nil {
				t.Errorf("Can lstat %q after supposed RemoveAll", path)
			}
			t.Fatalf("RemoveAll %q succeeded with chmod 0 subdirectory: err %s", path, err)
		}
		perr, ok := err.(*PathError)
		if !ok {
			t.Fatalf("RemoveAll %q returned %T not *PathError", path, err)
		}
		if perr.Path != dpath {
			t.Fatalf("RemoveAll %q failed at %q not %q", path, perr.Path, dpath)
		}
		if err = Chmod(dpath, 0777); err != nil {
			t.Fatalf("Chmod %q 0777: %s", dpath, err)
		}
		for _, s := range []string{fpath, path + "/zzz"} {
			if _, err := Lstat(s); err == nil {
				t.Fatalf("Lstat %q succeeded after partial RemoveAll", s)
			}
		}
	}
	if err = RemoveAll(path); err != nil {
		t.Fatalf("RemoveAll %q after partial RemoveAll: %s", path, err)
	}
	if _, err := Lstat(path); err == nil {
		t.Fatalf("Lstat %q succeeded after RemoveAll (final)", path)
	}
}
<commit_msg>os: fix test of RemoveAll<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os_test

import (
	. "os"
	"testing"
	"syscall"
)

func TestMkdirAll(t *testing.T) {
	// Create new dir, in _obj so it will get
	// cleaned up by make if not by us.
	path := "_obj/_TestMkdirAll_/dir/./dir2"
	err := MkdirAll(path, 0777)
	if err != nil {
		t.Fatalf("MkdirAll %q: %s", path, err)
	}

	// Already exists, should succeed.
	err = MkdirAll(path, 0777)
	if err != nil {
		t.Fatalf("MkdirAll %q (second time): %s", path, err)
	}

	// Make file.
	fpath := path + "/file"
	_, err = Open(fpath, O_WRONLY|O_CREAT, 0666)
	if err != nil {
		t.Fatalf("create %q: %s", fpath, err)
	}

	// Can't make directory named after file.
	err = MkdirAll(fpath, 0777)
	if err == nil {
		t.Fatalf("MkdirAll %q: no error", fpath)
	}
	perr, ok := err.(*PathError)
	if !ok {
		t.Fatalf("MkdirAll %q returned %T, not *PathError", fpath, err)
	}
	if perr.Path != fpath {
		t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", fpath, perr.Path, fpath)
	}

	// Can't make subdirectory of file.
	ffpath := fpath + "/subdir"
	err = MkdirAll(ffpath, 0777)
	if err == nil {
		t.Fatalf("MkdirAll %q: no error", ffpath)
	}
	perr, ok = err.(*PathError)
	if !ok {
		t.Fatalf("MkdirAll %q returned %T, not *PathError", ffpath, err)
	}
	if perr.Path != fpath {
		t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", ffpath, perr.Path, fpath)
	}

	RemoveAll("_obj/_TestMkdirAll_")
}

func TestRemoveAll(t *testing.T) {
	// Work directory.
	path := "_obj/_TestRemoveAll_"
	fpath := path + "/file"
	dpath := path + "/dir"

	// Make directory with 1 file and remove.
	if err := MkdirAll(path, 0777); err != nil {
		t.Fatalf("MkdirAll %q: %s", path, err)
	}
	fd, err := Open(fpath, O_WRONLY|O_CREAT, 0666)
	if err != nil {
		t.Fatalf("create %q: %s", fpath, err)
	}
	fd.Close()
	if err = RemoveAll(path); err != nil {
		t.Fatalf("RemoveAll %q (first): %s", path, err)
	}
	if _, err := Lstat(path); err == nil {
		t.Fatalf("Lstat %q succeeded after RemoveAll (first)", path)
	}

	// Make directory with file and subdirectory and remove.
	if err = MkdirAll(dpath, 0777); err != nil {
		t.Fatalf("MkdirAll %q: %s", dpath, err)
	}
	fd, err = Open(fpath, O_WRONLY|O_CREAT, 0666)
	if err != nil {
		t.Fatalf("create %q: %s", fpath, err)
	}
	fd.Close()
	fd, err = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666)
	if err != nil {
		t.Fatalf("create %q: %s", fpath, err)
	}
	fd.Close()
	if err = RemoveAll(path); err != nil {
		t.Fatalf("RemoveAll %q (second): %s", path, err)
	}
	if _, err := Lstat(path); err == nil {
		t.Fatalf("Lstat %q succeeded after RemoveAll (second)", path)
	}

	// Determine if we should run the following test.
	testit := true
	if syscall.OS == "windows" {
		// Chmod is not supported under windows.
		testit = false
	} else {
		// Test fails as root.
		testit = Getuid() != 0
	}
	if testit {
		// Make directory with file and subdirectory and trigger error.
		if err = MkdirAll(dpath, 0777); err != nil {
			t.Fatalf("MkdirAll %q: %s", dpath, err)
		}

		for _, s := range []string{fpath, dpath + "/file1", path + "/zzz"} {
			fd, err = Open(s, O_WRONLY|O_CREAT, 0666)
			if err != nil {
				t.Fatalf("create %q: %s", s, err)
			}
			fd.Close()
		}
		if err = Chmod(dpath, 0); err != nil {
			t.Fatalf("Chmod %q 0: %s", dpath, err)
		}

		// No error checking here: either RemoveAll
		// will or won't be able to remove dpath;
		// either way we want to see if it removes fpath
		// and path/zzz.  Reasons why RemoveAll might
		// succeed in removing dpath as well include:
		//	* running as root
		//	* running on a file system without permissions (FAT)
		RemoveAll(path)
		Chmod(dpath, 0777)

		for _, s := range []string{fpath, path + "/zzz"} {
			if _, err := Lstat(s); err == nil {
				t.Fatalf("Lstat %q succeeded after partial RemoveAll", s)
			}
		}
	}
	if err = RemoveAll(path); err != nil {
		t.Fatalf("RemoveAll %q after partial RemoveAll: %s", path, err)
	}
	if _, err := Lstat(path); err == nil {
		t.Fatalf("Lstat %q succeeded after RemoveAll (final)", path)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/Shopify/sarama"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/s3"
	"log"
	"sync"
)

var (
	AWSRegion         = "us-west-2"
	envCreds          = credentials.NewEnvCredentials()
	config            = aws.NewConfig().WithCredentials(envCreds).WithRegion(AWSRegion)
	dbSvc             = dynamodb.New(config)
	s3Svc             = s3.New(config)
	retryCount        = 5
	chosenOffset      = sarama.OffsetOldest
	offsetType        = "oldest"
	awsWorkPoolSize   = 5
	S3BucketName      = "eaton-jobdescription-bucket"
	DynamoDBTableName = "Documents"
)

//kafkaServers,kafkaTopic defined in kafka_producer.go

type IndeedKafkaConsumer struct {
	consumer           sarama.Consumer
	partitionConsumers []sarama.PartitionConsumer
}

func NewKafkaConsumer() (*IndeedKafkaConsumer, error) {
	config := sarama.NewConfig()
	config.ClientID = GetLocalAddr()
	config.Consumer.Return.Errors = true
	consumer, err := sarama.NewConsumer(kafkaServers, config)
	if err != nil {
		return nil, err
	}
	partitions, err := consumer.Partitions(kafkaTopic)
	if err != nil {
		return nil, err
	}
	if Debug {
		log.Println("Returned Partitions for topic: ", kafkaTopic, partitions)
	}
	if len(partitions) == 0 {
		return nil, errors.New("no partitions returned to consume!")
	}
	partitionConsumers := make([]sarama.PartitionConsumer, len(partitions), len(partitions))
	switch offsetType {
	case "oldest":
		chosenOffset = sarama.OffsetOldest
		break
	case "newest":
		chosenOffset = sarama.OffsetNewest
		break
	default:
		log.Fatal("unknown offsetType provided: ", offsetType)
	}
	for index, partition := range partitions {
		if Debug {
			log.Println("Creating partition consumer for partition: ", partition, " with offset: ", chosenOffset)
		}
		partitionConsumer, err := consumer.ConsumePartition(kafkaTopic, partition, chosenOffset)
		if Debug {
			log.Println("Created partition consumer: ", consumer)
		}
		if err != nil {
			return nil, err
		}
		if partitionConsumer == nil {
			return nil, errors.New("nil consumer returned!")
		}
		partitionConsumers[index] = partitionConsumer
	}

	return &IndeedKafkaConsumer{
		consumer:           consumer,
		partitionConsumers: partitionConsumers,
	}, nil
}

func (i *IndeedKafkaConsumer) Close() error {
	err := i.consumer.Close()
	if err != nil {
		return err
	}
	for _, p := range i.partitionConsumers {
		err = p.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

type AWSWork struct {
	jobResult JobResult
	msgValue  []byte
}

func (a *AWSWork) DoWork() error {
	var err error
	for i := 1; i < retryCount; i++ {
		_, err = s3Svc.PutObject(&s3.PutObjectInput{
			Bucket:      aws.String(S3BucketName),
			Key:         aws.String(a.jobResult.JobKey),
			Body:        bytes.NewReader(a.msgValue),
			ContentType: aws.String(fmt.Sprintf("application/%s", indeedConstants.IndeedResponseFormat)),
		})
		if err == nil {
			break
		}
	}
	if err != nil {
		return err
	}
	if a.jobResult.FullJobSummary != "" {
		log.Println("JobKey with FullSummary: ", a.jobResult.JobKey)
	}
	putItemInput := &dynamodb.PutItemInput{
		Item: map[string]*dynamodb.AttributeValue{
			//Minimum required fields as defined by EAT-3
			"DocumentID": {
				S: aws.String(a.jobResult.JobKey),
			},
			"Source": {
				S: aws.String("indeed"),
			},
			"Role": {
				S: aws.String("none"),
			},
			"Type": {
				S: aws.String("job description"),
			},
			"FileType": {
				S: aws.String(fmt.Sprintf("https://s3-%s.amazonaws.com/%s/%s", AWSRegion, S3BucketName, a.jobResult.JobKey)),
			},
			//extended metadata for the actual result.
			"CreateDate": {
				S: aws.String(a.jobResult.GetDateString()),
			},
			"JobTitle": {
				S: aws.String(a.jobResult.JobTitle),
			},
			"Company": {
				S: aws.String(a.jobResult.Company),
			},
			"City": {
				S: aws.String(a.jobResult.City),
			},
			"State": {
				S: aws.String(a.jobResult.State),
			},
			"Country": {
				S: aws.String(a.jobResult.Country),
			},
			"FormattedLocation": {
				S: aws.String(a.jobResult.FormattedLocation),
			},
			"ResultSource": {
				S: aws.String(a.jobResult.Source),
			},
			"Snippet": {
				S: aws.String(a.jobResult.Snippet),
			},
			"Latitude": {
				N: aws.String(fmt.Sprintf("%f", a.jobResult.Latitude)),
			},
			"Longitude": {
				N: aws.String(fmt.Sprintf("%f", a.jobResult.Longitude)),
			},
			"Sponsored": {
				BOOL: aws.Bool(a.jobResult.Sponsored),
			},
			"Expired": {
				BOOL: aws.Bool(a.jobResult.Expired),
			},
			"FormattedLocationFull": {
				S: aws.String(a.jobResult.FormattedLocationFull),
			},
			"FormattedRelativeTime": {
				S: aws.String(a.jobResult.FormattedRelativeTime),
			},
			"FullJobSummary": {
				S: aws.String(a.jobResult.FullJobSummary),
			},
		},
		TableName: aws.String(DynamoDBTableName),
	}
	for i := 1; i < retryCount; i++ {
		_, err = dbSvc.PutItem(putItemInput)
		if err == nil {
			break
		}
	}
	if err != nil {
		return err
	}
	log.Println("Successfully stored jobkey ", a.jobResult.JobKey, " in table ", DynamoDBTableName, " and in bucket ", S3BucketName)
	return nil
}

func (i *IndeedKafkaConsumer) ConsumeMessages() <-chan error {
	errChannel := make(chan error)
	go func() {
		defer close(errChannel)
		workChannel := make(chan AWSWork)
		defer close(workChannel)
		for j := 0; j < awsWorkPoolSize; j++ {
			go func() {
				for w := range workChannel {
					err := w.DoWork()
					if err != nil {
						errChannel <- err
					}
				}
			}()
		}
		var wg sync.WaitGroup
		wg.Add(len(i.partitionConsumers) * 2)
		for _, partitionConsumer := range i.partitionConsumers {
			go func(partitionConsumer sarama.PartitionConsumer) {
				for msg := range partitionConsumer.Messages() {
					result := new(JobResult)
					err := xml.Unmarshal(msg.Value, result)
					if err != nil {
						errChannel <- err
						continue
					}
					workChannel <- AWSWork{
						jobResult: *result,
						msgValue:  msg.Value,
					}
				}
				wg.Done()
			}(partitionConsumer)
			go func(partitionConsumer sarama.PartitionConsumer) {
				for err := range partitionConsumer.Errors() {
					errChannel <- err.Err
				}
				wg.Done()
			}(partitionConsumer)
		}
		wg.Wait()
	}()
	return errChannel
}
<commit_msg>EAT-12 - fixed issue with posting to dyanamo with an empty FullJobSummary string.<commit_after>package main

import (
	"bytes"
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/Shopify/sarama"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/s3"
	"log"
	"sync"
)

var (
	AWSRegion         = "us-west-2"
	envCreds          = credentials.NewEnvCredentials()
	config            = aws.NewConfig().WithCredentials(envCreds).WithRegion(AWSRegion)
	dbSvc             = dynamodb.New(config)
	s3Svc             = s3.New(config)
	retryCount        = 5
	chosenOffset      = sarama.OffsetOldest
	offsetType        = "oldest"
	awsWorkPoolSize   = 5
	S3BucketName      = "eaton-jobdescription-bucket"
	DynamoDBTableName = "Documents"
)

//kafkaServers,kafkaTopic defined in kafka_producer.go

type IndeedKafkaConsumer struct {
	consumer           sarama.Consumer
	partitionConsumers []sarama.PartitionConsumer
}

func NewKafkaConsumer() (*IndeedKafkaConsumer, error) {
	config := sarama.NewConfig()
	config.ClientID = GetLocalAddr()
	config.Consumer.Return.Errors = true
	consumer, err := sarama.NewConsumer(kafkaServers, config)
	if err != nil {
		return nil, err
	}
	partitions, err := consumer.Partitions(kafkaTopic)
	if err != nil {
		return nil, err
	}
	if Debug {
		log.Println("Returned Partitions for topic: ", kafkaTopic, partitions)
	}
	if len(partitions) == 0 {
		return nil, errors.New("no partitions returned to consume!")
	}
	partitionConsumers := make([]sarama.PartitionConsumer, len(partitions), len(partitions))
	switch offsetType {
	case "oldest":
		chosenOffset = sarama.OffsetOldest
		break
	case "newest":
		chosenOffset = sarama.OffsetNewest
		break
	default:
		log.Fatal("unknown offsetType provided: ", offsetType)
	}
	for index, partition := range partitions {
		if Debug {
			log.Println("Creating partition consumer for partition: ", partition, " with offset: ", chosenOffset)
		}
		partitionConsumer, err := consumer.ConsumePartition(kafkaTopic, partition, chosenOffset)
		if Debug {
			log.Println("Created partition consumer: ", consumer)
		}
		if err != nil {
			return nil, err
		}
		if partitionConsumer == nil {
			return nil, errors.New("nil consumer returned!")
		}
		partitionConsumers[index] = partitionConsumer
	}

	return &IndeedKafkaConsumer{
		consumer:           consumer,
		partitionConsumers: partitionConsumers,
	}, nil
}

func (i *IndeedKafkaConsumer) Close() error {
	err := i.consumer.Close()
	if err != nil {
		return err
	}
	for _, p := range i.partitionConsumers {
		err = p.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

type AWSWork struct {
	jobResult JobResult
	msgValue  []byte
}

func (a *AWSWork) DoWork() error {
	var err error
	for i := 1; i < retryCount; i++ {
		_, err = s3Svc.PutObject(&s3.PutObjectInput{
			Bucket:      aws.String(S3BucketName),
			Key:         aws.String(a.jobResult.JobKey),
			Body:        bytes.NewReader(a.msgValue),
			ContentType: aws.String(fmt.Sprintf("application/%s", indeedConstants.IndeedResponseFormat)),
		})
		if err == nil {
			break
		}
	}
	if err != nil {
		return err
	}
    attrItem := map[string]*dynamodb.AttributeValue{
			//Minimum required fields as defined by EAT-3
			"DocumentID": {
				S: aws.String(a.jobResult.JobKey),
			},
			"Source": {
				S: aws.String("indeed"),
			},
			"Role": {
				S: aws.String("none"),
			},
			"Type": {
				S: aws.String("job description"),
			},
			"FileType": {
				S: aws.String(fmt.Sprintf("https://s3-%s.amazonaws.com/%s/%s", AWSRegion, S3BucketName, a.jobResult.JobKey)),
			},
			//extended metadata for the actual result.
			"CreateDate": {
				S: aws.String(a.jobResult.GetDateString()),
			},
			"JobTitle": {
				S: aws.String(a.jobResult.JobTitle),
			},
			"Company": {
				S: aws.String(a.jobResult.Company),
			},
			"City": {
				S: aws.String(a.jobResult.City),
			},
			"State": {
				S: aws.String(a.jobResult.State),
			},
			"Country": {
				S: aws.String(a.jobResult.Country),
			},
			"FormattedLocation": {
				S: aws.String(a.jobResult.FormattedLocation),
			},
			"ResultSource": {
				S: aws.String(a.jobResult.Source),
			},
			"Snippet": {
				S: aws.String(a.jobResult.Snippet),
			},
			"Latitude": {
				N: aws.String(fmt.Sprintf("%f", a.jobResult.Latitude)),
			},
			"Longitude": {
				N: aws.String(fmt.Sprintf("%f", a.jobResult.Longitude)),
			},
			"Sponsored": {
				BOOL: aws.Bool(a.jobResult.Sponsored),
			},
			"Expired": {
				BOOL: aws.Bool(a.jobResult.Expired),
			},
			"FormattedLocationFull": {
				S: aws.String(a.jobResult.FormattedLocationFull),
			},
			"FormattedRelativeTime": {
				S: aws.String(a.jobResult.FormattedRelativeTime),
            },
    }
    if a.jobResult.FullJobSummary != "" {
        attrItem["FullJobSummary"] = &dynamodb.AttributeValue{
            S:aws.String(a.jobResult.FullJobSummary),
        }
    }
	putItemInput := &dynamodb.PutItemInput{
		Item: attrItem,
		TableName: aws.String(DynamoDBTableName),
	}
	for i := 1; i < retryCount; i++ {
		_, err = dbSvc.PutItem(putItemInput)
		if err == nil {
			break
		}
	}
	if err != nil {
		return err
	}
	log.Println("Successfully stored jobkey ", a.jobResult.JobKey, " in table ", DynamoDBTableName, " and in bucket ", S3BucketName)
	return nil
}

func (i *IndeedKafkaConsumer) ConsumeMessages() <-chan error {
	errChannel := make(chan error)
	go func() {
		defer close(errChannel)
		workChannel := make(chan AWSWork)
		defer close(workChannel)
		for j := 0; j < awsWorkPoolSize; j++ {
			go func() {
				for w := range workChannel {
					err := w.DoWork()
					if err != nil {
						errChannel <- err
					}
				}
			}()
		}
		var wg sync.WaitGroup
		wg.Add(len(i.partitionConsumers) * 2)
		for _, partitionConsumer := range i.partitionConsumers {
			go func(partitionConsumer sarama.PartitionConsumer) {
				for msg := range partitionConsumer.Messages() {
					result := new(JobResult)
					err := xml.Unmarshal(msg.Value, result)
					if err != nil {
						errChannel <- err
						continue
					}
					workChannel <- AWSWork{
						jobResult: *result,
						msgValue:  msg.Value,
					}
				}
				wg.Done()
			}(partitionConsumer)
			go func(partitionConsumer sarama.PartitionConsumer) {
				for err := range partitionConsumer.Errors() {
					errChannel <- err.Err
				}
				wg.Done()
			}(partitionConsumer)
		}
		wg.Wait()
	}()
	return errChannel
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

func getenv(s *byte) *byte {
	val := gogetenv(gostringnocopy(s))
	if val == "" {
		return nil
	}
	// Strings found in environment are NUL-terminated.
	return &bytes(val)[0]
}

var tracebackbuf [128]byte

func gogetenv(key string) string {
	var file [128]byte
	if len(key) > len(file)-6 {
		return ""
	}

	copy(file[:], "/env/")
	copy(file[5:], key)

	fd := open(&file[0], _OREAD, 0)
	if fd < 0 {
		return ""
	}
	n := seek(fd, 0, 2) - 1
	if n <= 0 {
		close(fd)
		return ""
	}

	p := make([]byte, n)

	r := pread(fd, unsafe.Pointer(&p[0]), int32(n), 0)
	close(fd)
	if r < 0 {
		return ""
	}

	var s string
	sp := (*_string)(unsafe.Pointer(&s))
	sp.str = &p[0]
	sp.len = int(r)
	return s
}
<commit_msg>runtime: handle non-nil-terminated environment strings on Plan 9<commit_after>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

func getenv(s *byte) *byte {
	val := gogetenv(gostringnocopy(s))
	if val == "" {
		return nil
	}
	// Strings found in environment are NUL-terminated.
	return &bytes(val)[0]
}

var tracebackbuf [128]byte

func gogetenv(key string) string {
	var file [128]byte
	if len(key) > len(file)-6 {
		return ""
	}

	copy(file[:], "/env/")
	copy(file[5:], key)

	fd := open(&file[0], _OREAD, 0)
	if fd < 0 {
		return ""
	}
	n := seek(fd, 0, 2)
	if n <= 0 {
		close(fd)
		return ""
	}

	p := make([]byte, n)

	r := pread(fd, unsafe.Pointer(&p[0]), int32(n), 0)
	close(fd)
	if r < 0 {
		return ""
	}

	if p[r-1] == 0 {
		r--
	}

	var s string
	sp := (*_string)(unsafe.Pointer(&s))
	sp.str = &p[0]
	sp.len = int(r)
	return s
}
<|endoftext|>


<commit_before>package sql

import (
	"crypto/rand"
	"crypto/sha256"
	"database/sql"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"io"
	"os"
	"path"
	"sync"

	"github.com/agl/ed25519"
	"github.com/repbin/repbin/hashcash"
	"github.com/repbin/repbin/message"
	"github.com/repbin/repbin/utils/keyproof"
)

// Version of this release
const Version = "0.0.1 very alpha"

var shardRand = make([]byte, 16)

func init() {
	io.ReadFull(rand.Reader, shardRand)
}

var (
	// ErrNoModify is returned if a row was not modified
	ErrNoModify = errors.New("storage: Row not modified")
)

// MessageDB implements a message database
type MessageDB struct {
	db                     *sql.DB
	NumShards              uint64
	shardMutexes           []*sync.Mutex
	dir                    string
	driver                 string
	queries                map[string]string
	signerInsertQ          *sql.Stmt
	signerSelectPublicKeyQ *sql.Stmt
	signerSelectIDQ        *sql.Stmt
	signerUpdateQ          *sql.Stmt
	signerUpdateInsertQ    *sql.Stmt
	signerAddMessageQ      *sql.Stmt
	signerDelMessageQ      *sql.Stmt
	signerPrepareExpireQ   *sql.Stmt
	signerExpireQ          *sql.Stmt
	signerSetExpireQ       *sql.Stmt
	peerInsertQ            *sql.Stmt
	peerUpdateStatQ        *sql.Stmt
	peerUpdateTokenQ       *sql.Stmt
	peerUpdateNotifyQ      *sql.Stmt
	peerSelectQ            *sql.Stmt
	nextMessageCounterQ    *sql.Stmt
	incrMessageCounterQ    *sql.Stmt
	expireMessageCounterQ  *sql.Stmt
	insertMessageCounterQ  *sql.Stmt
	insertMessageQ         *sql.Stmt
	selectMessageQ         *sql.Stmt
	deleteMessageQ         *sql.Stmt
	updateExpireMessageQ   *sql.Stmt
	selectExpireMessageQ   *sql.Stmt
	globalIndexAddQ        *sql.Stmt
	getKeyIndexQ           *sql.Stmt
	getGlobalIndexQ        *sql.Stmt
	messageBlobInsertQ     *sql.Stmt
	messageBlobSelectQ     *sql.Stmt
	messageBlobDeleteQ     *sql.Stmt
	messageExistInsertQ    *sql.Stmt
	messageExistSelectQ    *sql.Stmt
	messageExistExpireQ    *sql.Stmt
}

// New returns a new message database. driver is the database driver to use,
// url the database url. dir is the optional directory in which to store the
// raw message blobs. If dir is empty blobs will be stored in the database (which
// may not be a good idea at all). Shards is the number of lock shards to use
// for sequence generation (memory/lock-probability tradeoff)
func New(driver, url, dir string, shards int) (*MessageDB, error) {
	var db *sql.DB
	var err error
	if driver == "sqlite3" {
		if err := os.MkdirAll(path.Dir(url), 0700); err != nil {
			return nil, err
		}
	}
	db, err = sql.Open(driver, url)
	if err != nil {
		return nil, err
	}
	mdb := &MessageDB{
		NumShards: uint64(shards),
		queries:   queries[driver],
		db:        db,
		dir:       dir,
		driver:    driver,
	}

	if _, err := mdb.db.Exec(mdb.queries["SignerCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["PeerCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["MessageCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["GlobalIndexCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["messageBlobCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["MessageCounterCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["messageExistCreate"]); err != nil {
		return nil, err
	}
	if mdb.signerInsertQ, err = mdb.db.Prepare(mdb.queries["SignerInsert"]); err != nil {
		return nil, err
	}
	if mdb.signerSelectPublicKeyQ, err = mdb.db.Prepare(mdb.queries["SelectSignerPublicKey"]); err != nil {
		return nil, err
	}
	if mdb.signerSelectIDQ, err = mdb.db.Prepare(mdb.queries["SelectSignerID"]); err != nil {
		return nil, err
	}
	if mdb.signerUpdateQ, err = mdb.db.Prepare(mdb.queries["UpdateSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerAddMessageQ, err = mdb.db.Prepare(mdb.queries["AddMessageSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerDelMessageQ, err = mdb.db.Prepare(mdb.queries["DelMessageSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerPrepareExpireQ, err = mdb.db.Prepare(mdb.queries["PrepareExpireSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerExpireQ, err = mdb.db.Prepare(mdb.queries["DeleteExpireSigner"]); err != nil {
		return nil, err
	}

	if mdb.peerInsertQ, err = mdb.db.Prepare(mdb.queries["InsertPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateStatQ, err = mdb.db.Prepare(mdb.queries["UpdateStatPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateNotifyQ, err = mdb.db.Prepare(mdb.queries["UpdateNotifyPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateTokenQ, err = mdb.db.Prepare(mdb.queries["UpdateTokenPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerSelectQ, err = mdb.db.Prepare(mdb.queries["SelectPeer"]); err != nil {
		return nil, err
	}

	if mdb.nextMessageCounterQ, err = mdb.db.Prepare(mdb.queries["NextMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.incrMessageCounterQ, err = mdb.db.Prepare(mdb.queries["IncreaseMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.insertMessageCounterQ, err = mdb.db.Prepare(mdb.queries["InsertMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.expireMessageCounterQ, err = mdb.db.Prepare(mdb.queries["ExpireMessageCounter"]); err != nil {
		return nil, err
	}

	if mdb.insertMessageQ, err = mdb.db.Prepare(mdb.queries["InsertMessage"]); err != nil {
		return nil, err
	}
	if mdb.selectMessageQ, err = mdb.db.Prepare(mdb.queries["SelectMessage"]); err != nil {
		return nil, err
	}
	if mdb.deleteMessageQ, err = mdb.db.Prepare(mdb.queries["DeleteMessage"]); err != nil {
		return nil, err
	}
	if mdb.updateExpireMessageQ, err = mdb.db.Prepare(mdb.queries["UpdateExpireMessage"]); err != nil {
		return nil, err
	}
	if mdb.selectExpireMessageQ, err = mdb.db.Prepare(mdb.queries["SelectExpireMessage"]); err != nil {
		return nil, err
	}

	if mdb.globalIndexAddQ, err = mdb.db.Prepare(mdb.queries["globalIndexAdd"]); err != nil {
		return nil, err
	}
	if mdb.getKeyIndexQ, err = mdb.db.Prepare(mdb.queries["getKeyIndex"]); err != nil {
		return nil, err
	}
	if mdb.getGlobalIndexQ, err = mdb.db.Prepare(mdb.queries["getGlobalIndex"]); err != nil {
		return nil, err
	}

	if mdb.messageBlobInsertQ, err = mdb.db.Prepare(mdb.queries["messageBlobInsert"]); err != nil {
		return nil, err
	}
	if mdb.messageBlobSelectQ, err = mdb.db.Prepare(mdb.queries["messageBlobSelect"]); err != nil {
		return nil, err
	}
	if mdb.messageBlobDeleteQ, err = mdb.db.Prepare(mdb.queries["messageBlobDelete"]); err != nil {
		return nil, err
	}

	if mdb.messageExistInsertQ, err = mdb.db.Prepare(mdb.queries["messageExistInsert"]); err != nil {
		return nil, err
	}
	if mdb.messageExistSelectQ, err = mdb.db.Prepare(mdb.queries["messageExistSelect"]); err != nil {
		return nil, err
	}
	if mdb.messageExistExpireQ, err = mdb.db.Prepare(mdb.queries["messageExistExpire"]); err != nil {
		return nil, err
	}

	if driver == "mysql" {
		if mdb.signerUpdateInsertQ, err = mdb.db.Prepare(mdb.queries["UpdateOrInsertSigner"]); err != nil {
			return nil, err
		}
	}
	if dir != "" {
		if err := os.MkdirAll(dir, 0700); err != nil {
			return nil, err
		}
	}
	mdb.setMutexes()
	return mdb, nil
}

func newMySQLForTest(dir string, shards int) (*MessageDB, error) {
	var url string
	// MySQL in Travis CI doesn't have a password
	if os.Getenv("TRAVIS") == "true" {
		url = "root@/repbin"
	} else {
		url = "root:root@/repbin"
	}
	return New("mysql", url, dir, shards)
}

// LockShard locks shard s
func (db *MessageDB) LockShard(s []byte) {
	db.shardMutexes[db.calcShard(s)].Lock()
}

// UnlockShard unlocks shard s, if locked. Runtime error otherwise
func (db *MessageDB) UnlockShard(s []byte) {
	db.shardMutexes[db.calcShard(s)].Unlock()
}

func (db *MessageDB) setMutexes() {
	db.shardMutexes = make([]*sync.Mutex, db.NumShards)
	for i := range db.shardMutexes {
		db.shardMutexes[i] = new(sync.Mutex)
	}
}

// Close the database
func (db *MessageDB) Close() error {
	return db.db.Close()
}

func toHex(d []byte) string {
	return hex.EncodeToString(d)
}

func fromHex(s string) []byte {
	d, _ := hex.DecodeString(s)
	return d
}

func sliceToSignerPubKey(d []byte) *[message.SignerPubKeySize]byte {
	r := new([message.SignerPubKeySize]byte)
	copy(r[:], d)
	return r
}

func sliceToNonce(d []byte) *[hashcash.NonceSize]byte {
	r := new([hashcash.NonceSize]byte)
	copy(r[:], d)
	return r
}

func sliceToProofTokenSigned(d []byte) *[keyproof.ProofTokenSignedSize]byte {
	r := new([keyproof.ProofTokenSignedSize]byte)
	copy(r[:], d)
	return r
}

func sliceToEDPublicKey(d []byte) *[ed25519.PublicKeySize]byte {
	r := new([ed25519.PublicKeySize]byte)
	copy(r[:], d)
	return r
}

func sliceToCurve25519Key(d []byte) *message.Curve25519Key {
	r := new(message.Curve25519Key)
	copy(r[:], d)
	return r
}

func sliceToMessageID(d []byte) *[message.MessageIDSize]byte {
	r := new([message.MessageIDSize]byte)
	copy(r[:], d)
	return r
}

func boolToInt(b bool) int {
	if b {
		return 1
	}
	return 0
}

func intToBool(i int) bool {
	if i > 0 {
		return true
	}
	return false
}

func updateConvertNilError(res sql.Result, err error) error {
	if err != nil {
		return err
	}
	n, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if n < 1 {
		return ErrNoModify
	}
	return nil
}

func (db *MessageDB) calcShard(d []byte) uint64 {
	h := sha256.Sum256(append(shardRand, d...))
	return binary.BigEndian.Uint64(h[:]) % db.NumShards
}
<commit_msg>moved test function<commit_after>package sql

import (
	"crypto/rand"
	"crypto/sha256"
	"database/sql"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"io"
	"os"
	"path"
	"sync"

	"github.com/agl/ed25519"
	"github.com/repbin/repbin/hashcash"
	"github.com/repbin/repbin/message"
	"github.com/repbin/repbin/utils/keyproof"
)

// Version of this release
const Version = "0.0.1 very alpha"

var shardRand = make([]byte, 16)

func init() {
	io.ReadFull(rand.Reader, shardRand)
}

var (
	// ErrNoModify is returned if a row was not modified
	ErrNoModify = errors.New("storage: Row not modified")
)

// MessageDB implements a message database
type MessageDB struct {
	db                     *sql.DB
	NumShards              uint64
	shardMutexes           []*sync.Mutex
	dir                    string
	driver                 string
	queries                map[string]string
	signerInsertQ          *sql.Stmt
	signerSelectPublicKeyQ *sql.Stmt
	signerSelectIDQ        *sql.Stmt
	signerUpdateQ          *sql.Stmt
	signerUpdateInsertQ    *sql.Stmt
	signerAddMessageQ      *sql.Stmt
	signerDelMessageQ      *sql.Stmt
	signerPrepareExpireQ   *sql.Stmt
	signerExpireQ          *sql.Stmt
	signerSetExpireQ       *sql.Stmt
	peerInsertQ            *sql.Stmt
	peerUpdateStatQ        *sql.Stmt
	peerUpdateTokenQ       *sql.Stmt
	peerUpdateNotifyQ      *sql.Stmt
	peerSelectQ            *sql.Stmt
	nextMessageCounterQ    *sql.Stmt
	incrMessageCounterQ    *sql.Stmt
	expireMessageCounterQ  *sql.Stmt
	insertMessageCounterQ  *sql.Stmt
	insertMessageQ         *sql.Stmt
	selectMessageQ         *sql.Stmt
	deleteMessageQ         *sql.Stmt
	updateExpireMessageQ   *sql.Stmt
	selectExpireMessageQ   *sql.Stmt
	globalIndexAddQ        *sql.Stmt
	getKeyIndexQ           *sql.Stmt
	getGlobalIndexQ        *sql.Stmt
	messageBlobInsertQ     *sql.Stmt
	messageBlobSelectQ     *sql.Stmt
	messageBlobDeleteQ     *sql.Stmt
	messageExistInsertQ    *sql.Stmt
	messageExistSelectQ    *sql.Stmt
	messageExistExpireQ    *sql.Stmt
}

// New returns a new message database. driver is the database driver to use,
// url the database url. dir is the optional directory in which to store the
// raw message blobs. If dir is empty blobs will be stored in the database (which
// may not be a good idea at all). Shards is the number of lock shards to use
// for sequence generation (memory/lock-probability tradeoff)
func New(driver, url, dir string, shards int) (*MessageDB, error) {
	var db *sql.DB
	var err error
	if driver == "sqlite3" {
		os.MkdirAll(path.Dir(url), 0700)
	}
	db, err = sql.Open(driver, url)
	if err != nil {
		return nil, err
	}
	mdb := &MessageDB{
		NumShards: uint64(shards),
		queries:   queries[driver],
		db:        db,
		dir:       dir,
		driver:    driver,
	}

	if _, err := mdb.db.Exec(mdb.queries["SignerCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["PeerCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["MessageCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["GlobalIndexCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["messageBlobCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["MessageCounterCreate"]); err != nil {
		return nil, err
	}
	if _, err := mdb.db.Exec(mdb.queries["messageExistCreate"]); err != nil {
		return nil, err
	}
	if mdb.signerInsertQ, err = mdb.db.Prepare(mdb.queries["SignerInsert"]); err != nil {
		return nil, err
	}
	if mdb.signerSelectPublicKeyQ, err = mdb.db.Prepare(mdb.queries["SelectSignerPublicKey"]); err != nil {
		return nil, err
	}
	if mdb.signerSelectIDQ, err = mdb.db.Prepare(mdb.queries["SelectSignerID"]); err != nil {
		return nil, err
	}
	if mdb.signerUpdateQ, err = mdb.db.Prepare(mdb.queries["UpdateSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerAddMessageQ, err = mdb.db.Prepare(mdb.queries["AddMessageSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerDelMessageQ, err = mdb.db.Prepare(mdb.queries["DelMessageSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerPrepareExpireQ, err = mdb.db.Prepare(mdb.queries["PrepareExpireSigner"]); err != nil {
		return nil, err
	}
	if mdb.signerExpireQ, err = mdb.db.Prepare(mdb.queries["DeleteExpireSigner"]); err != nil {
		return nil, err
	}

	if mdb.peerInsertQ, err = mdb.db.Prepare(mdb.queries["InsertPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateStatQ, err = mdb.db.Prepare(mdb.queries["UpdateStatPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateNotifyQ, err = mdb.db.Prepare(mdb.queries["UpdateNotifyPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerUpdateTokenQ, err = mdb.db.Prepare(mdb.queries["UpdateTokenPeer"]); err != nil {
		return nil, err
	}
	if mdb.peerSelectQ, err = mdb.db.Prepare(mdb.queries["SelectPeer"]); err != nil {
		return nil, err
	}

	if mdb.nextMessageCounterQ, err = mdb.db.Prepare(mdb.queries["NextMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.incrMessageCounterQ, err = mdb.db.Prepare(mdb.queries["IncreaseMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.insertMessageCounterQ, err = mdb.db.Prepare(mdb.queries["InsertMessageCounter"]); err != nil {
		return nil, err
	}
	if mdb.expireMessageCounterQ, err = mdb.db.Prepare(mdb.queries["ExpireMessageCounter"]); err != nil {
		return nil, err
	}

	if mdb.insertMessageQ, err = mdb.db.Prepare(mdb.queries["InsertMessage"]); err != nil {
		return nil, err
	}
	if mdb.selectMessageQ, err = mdb.db.Prepare(mdb.queries["SelectMessage"]); err != nil {
		return nil, err
	}
	if mdb.deleteMessageQ, err = mdb.db.Prepare(mdb.queries["DeleteMessage"]); err != nil {
		return nil, err
	}
	if mdb.updateExpireMessageQ, err = mdb.db.Prepare(mdb.queries["UpdateExpireMessage"]); err != nil {
		return nil, err
	}
	if mdb.selectExpireMessageQ, err = mdb.db.Prepare(mdb.queries["SelectExpireMessage"]); err != nil {
		return nil, err
	}

	if mdb.globalIndexAddQ, err = mdb.db.Prepare(mdb.queries["globalIndexAdd"]); err != nil {
		return nil, err
	}
	if mdb.getKeyIndexQ, err = mdb.db.Prepare(mdb.queries["getKeyIndex"]); err != nil {
		return nil, err
	}
	if mdb.getGlobalIndexQ, err = mdb.db.Prepare(mdb.queries["getGlobalIndex"]); err != nil {
		return nil, err
	}

	if mdb.messageBlobInsertQ, err = mdb.db.Prepare(mdb.queries["messageBlobInsert"]); err != nil {
		return nil, err
	}
	if mdb.messageBlobSelectQ, err = mdb.db.Prepare(mdb.queries["messageBlobSelect"]); err != nil {
		return nil, err
	}
	if mdb.messageBlobDeleteQ, err = mdb.db.Prepare(mdb.queries["messageBlobDelete"]); err != nil {
		return nil, err
	}

	if mdb.messageExistInsertQ, err = mdb.db.Prepare(mdb.queries["messageExistInsert"]); err != nil {
		return nil, err
	}
	if mdb.messageExistSelectQ, err = mdb.db.Prepare(mdb.queries["messageExistSelect"]); err != nil {
		return nil, err
	}
	if mdb.messageExistExpireQ, err = mdb.db.Prepare(mdb.queries["messageExistExpire"]); err != nil {
		return nil, err
	}

	if driver == "mysql" {
		if mdb.signerUpdateInsertQ, err = mdb.db.Prepare(mdb.queries["UpdateOrInsertSigner"]); err != nil {
			return nil, err
		}
	}
	if dir != "" {
		if err := os.MkdirAll(dir, 0700); err != nil {
			return nil, err
		}
	}
	mdb.setMutexes()
	return mdb, nil
}

// LockShard locks shard s
func (db *MessageDB) LockShard(s []byte) {
	db.shardMutexes[db.calcShard(s)].Lock()
}

// UnlockShard unlocks shard s, if locked. Runtime error otherwise
func (db *MessageDB) UnlockShard(s []byte) {
	db.shardMutexes[db.calcShard(s)].Unlock()
}

func (db *MessageDB) setMutexes() {
	db.shardMutexes = make([]*sync.Mutex, db.NumShards)
	for i := range db.shardMutexes {
		db.shardMutexes[i] = new(sync.Mutex)
	}
}

// Close the database
func (db *MessageDB) Close() error {
	return db.db.Close()
}

func toHex(d []byte) string {
	return hex.EncodeToString(d)
}

func fromHex(s string) []byte {
	d, _ := hex.DecodeString(s)
	return d
}

func sliceToSignerPubKey(d []byte) *[message.SignerPubKeySize]byte {
	r := new([message.SignerPubKeySize]byte)
	copy(r[:], d)
	return r
}

func sliceToNonce(d []byte) *[hashcash.NonceSize]byte {
	r := new([hashcash.NonceSize]byte)
	copy(r[:], d)
	return r
}

func sliceToProofTokenSigned(d []byte) *[keyproof.ProofTokenSignedSize]byte {
	r := new([keyproof.ProofTokenSignedSize]byte)
	copy(r[:], d)
	return r
}

func sliceToEDPublicKey(d []byte) *[ed25519.PublicKeySize]byte {
	r := new([ed25519.PublicKeySize]byte)
	copy(r[:], d)
	return r
}

func sliceToCurve25519Key(d []byte) *message.Curve25519Key {
	r := new(message.Curve25519Key)
	copy(r[:], d)
	return r
}

func sliceToMessageID(d []byte) *[message.MessageIDSize]byte {
	r := new([message.MessageIDSize]byte)
	copy(r[:], d)
	return r
}

func boolToInt(b bool) int {
	if b {
		return 1
	}
	return 0
}

func intToBool(i int) bool {
	if i > 0 {
		return true
	}
	return false
}

func updateConvertNilError(res sql.Result, err error) error {
	if err != nil {
		return err
	}
	n, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if n < 1 {
		return ErrNoModify
	}
	return nil
}

func (db *MessageDB) calcShard(d []byte) uint64 {
	h := sha256.Sum256(append(shardRand, d...))
	return binary.BigEndian.Uint64(h[:]) % db.NumShards
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/joeshaw/envdecode"
	"gopkg.in/mailgun/mailgun-go.v1"
)

const (
	mailDomain  = "list.brandur.org"
	fromAddress = "Brandur <brandur@brandur.org>"
)

// Conf contains configuration information for the command. It's extracted from
// environment variables.
type Conf struct {
	// MailgunAPIKey is a key for Mailgun used to send email.
	MailgunAPIKey string `env:"MAILGUN_API_KEY,required"`
}

// Left as a global for now for the sake of convenience, but it's not used in
// very many places and can probably be refactored as a local if desired.
var conf Conf

func renderAndSend(records [][]string, live bool) error {
	mg := mailgun.NewMailgun(mailDomain, conf.MailgunAPIKey, "")

	for _, record := range records {
		// skip empty lines
		if len(record) == 0 {
			continue
		}

		if len(record) < 2 {
			return fmt.Errorf("Record less than 2-width: %v", record)
		}

		name := record[0]
		recipient := record[1]

		subject := "A newsletter"
		body := fmt.Sprintf(`%s,

One thing I've realized over the last few years is that I
don't do a very good job of keeping up with old friends and
family. I don'really use Facebook, and the new age social
media platforms of the Instagram and Snapchat variety are
well beyond me. Also, like any good millenial, I almost
never pick up the phone :)

On a recent trip I was thinking about what to do about it,
and came up with the idea of writing a very occasional
newsletter to people I know. The intent is for each one to
be a short compilation of stories, photographs, and ideas.
It'll remind me to send something to you, and hopefully
remind you to send something back.

This is just a quick note that I'm going to add you to the
receipt list. In case you have a healthy fear of inbox
overload, the bursts will be pretty infrequent; I'll be
competing with total solar eclipses on time scale. If
that's still not good enough, either reply to me here
saying so, or wait until I send it and just click the very
conspicuous "unsubscribe" link and you'll never get one
again (I won't get notified on an unsubscribe, and even if
I did, I wouldn't take it personally).

I hope everything is well!

Brandur`,
			name,
		)

		if live {
			message := mailgun.NewMessage(fromAddress, subject, body, recipient)
			resp, _, err := mg.Send(message)
			if err != nil {
				log.Fatal(err)
			}
			log.Printf(`Sent to: %s (response: "%s")`, recipient, resp)
		} else {
			fmt.Printf("To: %v <%v>\n", name, recipient)
			fmt.Printf("Subject: %v\n\n", subject)
			fmt.Printf("%v\n---\n\n", body)
		}
	}

	return nil
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %v [-live] <recipient_file>\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(0)
	}

	live := flag.Bool("live", false,
		"Send to list (as opposed to dry run)")
	flag.Parse()

	if len(flag.Args()) != 1 {
		flag.Usage()
	}

	err := envdecode.Decode(&conf)
	if err != nil {
		log.Fatal(err)
	}

	f, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	r := csv.NewReader(f)
	r.Comment = '#'

	records, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	err = renderAndSend(records, *live)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Fix name versus greeting<commit_after>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/joeshaw/envdecode"
	"gopkg.in/mailgun/mailgun-go.v1"
)

const (
	mailDomain  = "list.brandur.org"
	fromAddress = "Brandur <brandur@brandur.org>"
)

// Conf contains configuration information for the command. It's extracted from
// environment variables.
type Conf struct {
	// MailgunAPIKey is a key for Mailgun used to send email.
	MailgunAPIKey string `env:"MAILGUN_API_KEY,required"`
}

// Left as a global for now for the sake of convenience, but it's not used in
// very many places and can probably be refactored as a local if desired.
var conf Conf

func renderAndSend(records [][]string, live bool) error {
	mg := mailgun.NewMailgun(mailDomain, conf.MailgunAPIKey, "")

	for _, record := range records {
		// skip empty lines
		if len(record) == 0 {
			continue
		}

		if len(record) < 2 {
			return fmt.Errorf("Record less than 2-width: %v", record)
		}

		greeting := record[0]
		recipient := record[1]

		parts := strings.Split(greeting, " ")
		name := parts[len(parts)-1]

		subject := "A newsletter"
		body := fmt.Sprintf(`%s,

One thing I've realized over the last few years is that I
don't do a very good job of keeping up with old friends and
family. I don'really use Facebook, and the new age social
media platforms of the Instagram and Snapchat variety are
well beyond me. Also, like any good millenial, I almost
never pick up the phone :)

On a recent trip I was thinking about what to do about it,
and came up with the idea of writing a very occasional
newsletter to people I know. The intent is for each one to
be a short compilation of stories, photographs, and ideas.
It'll remind me to send something to you, and hopefully
remind you to send something back.

This is just a quick note that I'm going to add you to the
receipt list. In case you have a healthy fear of inbox
overload, the bursts will be pretty infrequent; I'll be
competing with total solar eclipses on time scale. If
that's still not good enough, either reply to me here
saying so, or wait until I send it and just click the very
conspicuous "unsubscribe" link and you'll never get one
again (I won't get notified on an unsubscribe, and even if
I did, I wouldn't take it personally).

I hope everything is well!

Brandur`,
			greeting,
		)

		if live {
			message := mailgun.NewMessage(fromAddress, subject, body, recipient)
			resp, _, err := mg.Send(message)
			if err != nil {
				log.Fatal(err)
			}
			log.Printf(`Sent to: %s (response: "%s")`, recipient, resp)
		} else {
			fmt.Printf("To: %v <%v>\n", name, recipient)
			fmt.Printf("Subject: %v\n\n", subject)
			fmt.Printf("%v\n---\n\n", body)
		}
	}

	return nil
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %v [-live] <recipient_file>\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(0)
	}

	live := flag.Bool("live", false,
		"Send to list (as opposed to dry run)")
	flag.Parse()

	if len(flag.Args()) != 1 {
		flag.Usage()
	}

	err := envdecode.Decode(&conf)
	if err != nil {
		log.Fatal(err)
	}

	f, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	r := csv.NewReader(f)
	r.Comment = '#'

	records, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	err = renderAndSend(records, *live)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tls

import (
	"bytes"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

// Tests that the linker is able to remove references to the Client or Server if unused.
func TestLinkerGC(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode")
	}
	t.Parallel()
	goBin := testenv.GoToolPath(t)
	testenv.MustHaveGoBuild(t)

	tests := []struct {
		name    string
		program string
		want    []string
		bad     []string
	}{
		{
			name: "empty_import",
			program: `package main
import _ "crypto/tls"
func main() {}
`,
			bad: []string{
				"tls.(*Conn)",
				"type.crypto/tls.clientHandshakeState",
				"type.crypto/tls.serverHandshakeState",
			},
		},
		{
			name: "only_conn",
			program: `package main
import "crypto/tls"
var c = new(tls.Conn)
func main() {}
`,
			want: []string{"tls.(*Conn)"},
			bad: []string{
				"type.crypto/tls.clientHandshakeState",
				"type.crypto/tls.serverHandshakeState",
			},
		},
		{
			name: "client_and_server",
			program: `package main
import "crypto/tls"
func main() {
  tls.Dial("", "", nil)
  tls.Server(nil, nil)
}
`,
			want: []string{
				"crypto/tls.(*Conn).clientHandshake",
				"crypto/tls.(*Conn).serverHandshake",
			},
		},
		{
			name: "only_client",
			program: `package main
import "crypto/tls"
func main() { tls.Dial("", "", nil) }
`,
			want: []string{
				"crypto/tls.(*Conn).clientHandshake",
			},
			bad: []string{
				"crypto/tls.(*Conn).serverHandshake",
			},
		},
		// TODO: add only_server like func main() { tls.Server(nil, nil) }
		// That currently brings in the client via Conn.handleRenegotiation.

	}
	tmpDir := t.TempDir()
	goFile := filepath.Join(tmpDir, "x.go")
	exeFile := filepath.Join(tmpDir, "x.exe")
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if err := ioutil.WriteFile(goFile, []byte(tt.program), 0644); err != nil {
				t.Fatal(err)
			}
			os.Remove(exeFile)
			cmd := exec.Command(goBin, "build", "-o", "x.exe", "x.go")
			cmd.Dir = tmpDir
			if out, err := cmd.CombinedOutput(); err != nil {
				t.Fatalf("compile: %v, %s", err, out)
			}

			cmd = exec.Command(goBin, "tool", "nm", "x.exe")
			cmd.Dir = tmpDir
			nm, err := cmd.CombinedOutput()
			if err != nil {
				t.Fatalf("nm: %v, %s", err, nm)
			}
			for _, sym := range tt.want {
				if !bytes.Contains(nm, []byte(sym)) {
					t.Errorf("expected symbol %q not found", sym)
				}
			}
			for _, sym := range tt.bad {
				if bytes.Contains(nm, []byte(sym)) {
					t.Errorf("unexpected symbol %q found", sym)
				}
			}
		})
	}
}
<commit_msg>crypto/tls: fix TestLinkerGC test<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tls

import (
	"bytes"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

// Tests that the linker is able to remove references to the Client or Server if unused.
func TestLinkerGC(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode")
	}
	t.Parallel()
	goBin := testenv.GoToolPath(t)
	testenv.MustHaveGoBuild(t)

	tests := []struct {
		name    string
		program string
		want    []string
		bad     []string
	}{
		{
			name: "empty_import",
			program: `package main
import _ "crypto/tls"
func main() {}
`,
			bad: []string{
				"tls.(*Conn)",
				"type.crypto/tls.clientHandshakeState",
				"type.crypto/tls.serverHandshakeState",
			},
		},
		{
			name: "client_and_server",
			program: `package main
import "crypto/tls"
func main() {
  tls.Dial("", "", nil)
  tls.Server(nil, nil)
}
`,
			want: []string{
				"crypto/tls.(*Conn).clientHandshake",
				"crypto/tls.(*Conn).serverHandshake",
			},
		},
		{
			name: "only_client",
			program: `package main
import "crypto/tls"
func main() { tls.Dial("", "", nil) }
`,
			want: []string{
				"crypto/tls.(*Conn).clientHandshake",
			},
			bad: []string{
				"crypto/tls.(*Conn).serverHandshake",
			},
		},
		// TODO: add only_server like func main() { tls.Server(nil, nil) }
		// That currently brings in the client via Conn.handleRenegotiation.

	}
	tmpDir := t.TempDir()
	goFile := filepath.Join(tmpDir, "x.go")
	exeFile := filepath.Join(tmpDir, "x.exe")
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if err := ioutil.WriteFile(goFile, []byte(tt.program), 0644); err != nil {
				t.Fatal(err)
			}
			os.Remove(exeFile)
			cmd := exec.Command(goBin, "build", "-o", "x.exe", "x.go")
			cmd.Dir = tmpDir
			if out, err := cmd.CombinedOutput(); err != nil {
				t.Fatalf("compile: %v, %s", err, out)
			}

			cmd = exec.Command(goBin, "tool", "nm", "x.exe")
			cmd.Dir = tmpDir
			nm, err := cmd.CombinedOutput()
			if err != nil {
				t.Fatalf("nm: %v, %s", err, nm)
			}
			for _, sym := range tt.want {
				if !bytes.Contains(nm, []byte(sym)) {
					t.Errorf("expected symbol %q not found", sym)
				}
			}
			for _, sym := range tt.bad {
				if bytes.Contains(nm, []byte(sym)) {
					t.Errorf("unexpected symbol %q found", sym)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package httputilmore

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"os"
	"regexp"
	"strings"

	"github.com/grokify/gotilla/strconv/strconvutil"
)

const (
	HeaderAuthorization                = "Authorization"
	HeaderContentDisposition           = "Content-Disposition"
	HeaderContentLength                = "Content-Length"
	HeaderContentTransferEncoding      = "Content-Transfer-Encoding"
	HeaderContentType                  = "Content-Type"
	HeaderLocation                     = "Location"
	HeaderUserAgent                    = "User-Agent"
	HeaderXContentTypeOptions          = "X-Content-Type-Options"
	ContentTypeAppJsonUtf8             = "application/json; charset=utf-8"
	ContentTypeAppFormUrlEncoded       = "application/x-www-form-urlencoded"
	ContentTypeAppXml                  = "application/xml"
	ContentTypeAppXmlUtf8              = "application/xml; charset=utf-8"
	ContentTypeTextCalendarUtf8Request = `text/calendar; charset="utf-8"; method=REQUEST`
	ContentTypeTextHtmlUtf8            = "text/html; charset=utf-8"
	ContentTypeTextPlainUsAscii        = "text/plain; charset=us-ascii"
	ContentTypeTextPlainUtf8           = "text/plain; charset=utf-8"
	ContentTypeTextXmlUtf8             = "text/xml; charset=utf-8"
	SchemeHTTPS                        = "https"
)

// GetWriteFile performs a HTTP GET request and saves the response body
// to the file path specified
func GetWriteFile(url string, filename string, perm os.FileMode) ([]byte, error) {
	_, bytes, err := GetResponseAndBytes(url)
	if err != nil {
		return bytes, err
	}
	err = ioutil.WriteFile(filename, bytes, perm)
	return bytes, err
}

// PostJsonSimple performs a HTTP POST request converting a body interface{} to
// JSON and adding the appropriate JSON Content-Type header.
func PostJsonSimple(requrl string, body interface{}) (*http.Response, error) {
	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return &http.Response{}, err
	}

	req, err := http.NewRequest(http.MethodPost, requrl, bytes.NewBuffer(bodyBytes))
	if err != nil {
		return &http.Response{}, err
	}
	req.Header.Set(HeaderContentType, ContentTypeAppJsonUtf8)

	client := &http.Client{}
	return client.Do(req)
}

// ResponseBody returns the body as a byte array
func ResponseBody(res *http.Response) ([]byte, error) {
	defer res.Body.Close()
	contents, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return []byte{}, err
	}
	return contents, nil
}

// ResponseBodyJSONMapIndent returns the body as a generic JSON dictionary
func ResponseBodyJSONMapIndent(res *http.Response, prefix string, indent string) ([]byte, error) {
	body, err := ResponseBody(res)
	if err != nil {
		return body, err
	}
	any := map[string]interface{}{}
	json.Unmarshal(body, &any)
	return json.MarshalIndent(any, prefix, indent)
}

// GetResponseAndBytes retreives a URL and returns the response body
// as a byte array in addition to the *http.Response.
func GetResponseAndBytes(url string) (*http.Response, []byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return resp, []byte{}, err
	}
	bytes, err := ResponseBody(resp)
	return resp, bytes, err
}

// UnmarshalResponseJSON unmarshal a `*http.Response` JSON body into
// a data pointer.
func UnmarshalResponseJSON(resp *http.Response, data interface{}) error {
	//bytes, err := ResponseBody(resp)
	bytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	return json.Unmarshal(bytes, data)
}

// PrintRequestOut prints a http.Request using `httputil.DumpRequestOut`.
func PrintRequestOut(req *http.Request, includeBody bool) error {
	reqBytes, err := httputil.DumpRequestOut(req, includeBody)
	if err != nil {
		return err
	}
	fmt.Println(string(reqBytes))
	return nil
}

// PrintResponse prints a http.Response using `httputil.DumpResponse`.
func PrintResponse(resp *http.Response, includeBody bool) error {
	respBytes, err := httputil.DumpResponse(resp, includeBody)
	if err != nil {
		return err
	}
	fmt.Println(string(respBytes))
	return nil
}

// ParseHeader converts a raw strign to a header struct.
func ParseHeader(s string) http.Header {
	h := http.Header{}
	lines := strings.Split(s, "\n")
	rx := regexp.MustCompile(`^([^\s+]+):\s*(.*)$`)
	for _, line := range lines {
		m := rx.FindStringSubmatch(line)
		if len(m) == 3 {
			key := strings.TrimSpace(m[1])
			val := strings.TrimSpace(m[2])
			if len(key) > 0 {
				h.Add(key, val)
			}
		}
	}
	return h
}

// MergeHeader merges two http.Header adding the values of the second
// to the first.
func MergeHeader(base, more http.Header, overwrite bool) http.Header {
	if base == nil {
		base = http.Header{}
	}
	if more == nil {
		return base
	}
	for k, vals := range more {
		if overwrite {
			base.Del(k)
		}

		for _, v := range vals {
			v = strings.TrimSpace(v)
			if len(v) > 0 {
				base.Add(k, v)
			}
		}
	}
	return base
}

func ParseMultipartFormDataBoundaryFromHeader(contentType string) string {
	rx := regexp.MustCompile(`^multipart/form-data.+boundary="?([^;"]+)`)
	m := rx.FindStringSubmatch(contentType)
	if len(m) > 0 {
		return m[1]
	}
	return ""
}

// RateLimitInfo is a structure for holding parsed rate limit info.
// It uses headers from the GitHub, RingCentral and Twitter APIs.
type RateLimitInfo struct {
	StatusCode          int
	RetryAfter          int
	XRateLimitLimit     int
	XRateLimitRemaining int
	XRateLimitReset     int
	XRateLimitWindow    int
}

// NewResponseRateLimitInfo returns a RateLimitInfo from a http.Response.
func NewResponseRateLimitInfo(resp *http.Response, useXrlHyphen bool) RateLimitInfo {
	rlstat := RateLimitInfo{
		StatusCode: resp.StatusCode,
		RetryAfter: strconvutil.AtoiWithDefault(resp.Header.Get("Retry-After"), 0)}

	if useXrlHyphen {
		rlstat.XRateLimitLimit = strconvutil.AtoiWithDefault(resp.Header.Get("X-Rate-Limit-Limit"), 0)
		rlstat.XRateLimitRemaining = strconvutil.AtoiWithDefault(resp.Header.Get("X-Rate-Limit-Remaining"), 0)
		rlstat.XRateLimitReset = strconvutil.AtoiWithDefault(resp.Header.Get("X-Rate-Limit-Reset"), 0)
		rlstat.XRateLimitWindow = strconvutil.AtoiWithDefault(resp.Header.Get("X-Rate-Limit-Window"), 0)
	} else {
		rlstat.XRateLimitLimit = strconvutil.AtoiWithDefault(resp.Header.Get("X-RateLimit-Limit"), 0)
		rlstat.XRateLimitRemaining = strconvutil.AtoiWithDefault(resp.Header.Get("X-RateLimit-Remaining"), 0)
		rlstat.XRateLimitReset = strconvutil.AtoiWithDefault(resp.Header.Get("X-RateLimit-Reset"), 0)
		rlstat.XRateLimitWindow = strconvutil.AtoiWithDefault(resp.Header.Get("X-RateLimit-Window"), 0)
	}
	return rlstat
}

type FnLogRateLimitInfo func(RateLimitInfo)
<commit_msg>add httputilmore.HeaderAccept<commit_after>package httputilmore

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"os"
	"regexp"
	"strings"

	"github.com/grokify/gotilla/strconv/strconvutil"
)

const (
	HeaderAccept                       = "Accept"
	HeaderAuthorization                = "Authorization"
	HeaderContentDisposition           = "Content-Disposition"
	HeaderContentLength                = "Content-Length"
	HeaderContentTransferEncoding      = "Content-Transfer-Encoding"
	HeaderContentType                  = "Content-Type"
	HeaderLocation                     = "Location"
	HeaderUserAgent                    = "User-Agent"
	HeaderXContentTypeOptions          = "X-Content-Type-Options"
	ContentTypeAppJsonUtf8             = "application/json; charset=utf-8"
	ContentTypeAppFormUrlEncoded       = "application/x-www-form-urlencoded"
	ContentTypeAppXml                  = "application/xml"
	ContentTypeAppXmlUtf8              = "application/xml; charset=utf-8"
	ContentTypeTextCalendarUtf8Request = `text/calendar; charset="utf-8"; method=REQUEST`
	ContentTypeTextHtmlUtf8            = "text/html; charset=utf-8"
	ContentTypeTextPlainUsAscii        = "text/plain; charset=us-ascii"
	ContentTypeTextPlainUtf8           = "text/plain; charset=utf-8"
	ContentTypeTextXmlUtf8             = "text/xml; charset=utf-8"
	SchemeHTTPS                        = "https"
)

// GetWriteFile performs a HTTP GET request and saves the response body
// to the file path specified
func GetWriteFile(url string, filename string, perm os.FileMode) ([]byte, error) {
	_, bytes, err := GetResponseAndBytes(url)
	if err != nil {
		return bytes, err
	}
	err = ioutil.WriteFile(filename, bytes, perm)
	return bytes, err
}

// PostJsonSimple performs a HTTP POST request converting a body interface{} to
// JSON and adding the appropriate JSON Content-Type header.
func PostJsonSimple(requrl string, body interface{}) (*http.Response, error) {
	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return &http.Response{}, err
	}

	req, err := http.NewRequest(http.MethodPost, requrl, bytes.NewBuffer(bodyBytes))
	if err != nil {
		return &http.Response{}, err
	}
	req.Header.Set(HeaderContentType, ContentTypeAppJsonUtf8)

	client := &http.Client{}
	return client.Do(req)
}

// ResponseBody returns the body as a byte array
func ResponseBody(res *http.Response) ([]byte, error) {
	defer res.Body.Close()
	contents, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return []byte{}, err
	}
	return contents, nil
}

// ResponseBodyJSONMapIndent returns the body as a generic JSON dictionary
func ResponseBodyJSONMapIndent(res *http.Response, prefix string, indent string) ([]byte, error) {
	body, err := ResponseBody(res)
	if err != nil {
		return body, err
	}
	any := map[string]interface{}{}
	json.Unmarshal(body, &any)
	return json.MarshalIndent(any, prefix, indent)
}

// GetResponseAndBytes retreives a URL and returns the response body
// as a byte array in addition to the *http.Response.
func GetResponseAndBytes(url string) (*http.Response, []byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return resp, []byte{}, err
	}
	bytes, err := ResponseBody(resp)
	return resp, bytes, err
}

// UnmarshalResponseJSON unmarshal a `*http.Response` JSON body into
// a data pointer.
func UnmarshalResponseJSON(resp *http.Response, data interface{}) error {
	//bytes, err := ResponseBody(resp)
	bytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	return json.Unmarshal(bytes, data)
}

// PrintRequestOut prints a http.Request using `httputil.DumpRequestOut`.
func PrintRequestOut(req *http.Request, includeBody bool) error {
	reqBytes, err := httputil.DumpRequestOut(req, includeBody)
	if err != nil {
		return err
	}
	fmt.Println(string(reqBytes))
	return nil
}

// PrintResponse prints a http.Response using `httputil.DumpResponse`.
func PrintResponse(resp *http.Response, includeBody bool) error {
	respBytes, err := httputil.DumpResponse(resp, includeBody)
	if err != nil {
		return err
	}
	fmt.Println(string(respBytes))
	return nil
}

// ParseHeader converts a raw strign to a header struct.
func ParseHeader(s string) http.Header {
	h := http.Header{}
	lines := strings.Split(s, "\n")
	rx := regexp.MustCompile(`^([^\s+]+):\s*(.*)$`)
	for _, line := range lines {
		m := rx.FindStringSubmatch(line)
		if len(m) == 3 {
			key := strings.TrimSpace(m[1])
			val := strings.TrimSpace(m[2])
			if len(key) > 0 {
				h.Add(key, val)
			}
		}
	}
	return h
}

// MergeHeader merges two http.Header adding the values of the second
// to the first.
func MergeHeader(base, more http.Header, overwrite bool) http.Header {
	if base == nil {
		base = http.Header{}
	}
	if more == nil {
		return base
	}
	for k, vals := range more {
		if overwrite {
			base.Del(k)
		}

		for _, v := range vals {
			v = strings.TrimSpace(v)
			if len(v) > 0 {
				base.Add(k, v)
			}
		}
	}
	return base
}

func ParseMultipartFormDataBoundaryFromHeader(contentType string) string {
	rx := regexp.MustCompile(`^multipart/form-data.+boundary="?([^;"]+)`)
	m := rx.FindStringSubmatch(contentType)
	if len(m) > 0 {
		return m[1]
	}
	return ""
}

// RateLimitInfo is a structure for holding parsed rate limit info.
// It uses headers from the GitHub, RingCentral and Twitter APIs.
type RateLimitInfo struct {
	StatusCode          int
	RetryAfter          int
	XRateLimitLimit     int
	XRateLimitRemaining int
	XRateLimitReset     int
	XRateLimitWindow    int
}

// NewResponseRateLimitInfo returns a RateLimitInfo from a http.Response.
func NewResponseRateLimitInfo(resp *http.Response, useXrlHyphen bool) RateLimitInfo {
	rlstat := RateLimitInfo{
		StatusCode: resp.StatusCode,
		RetryAfter: strconvutil.AtoiWithDefault(resp.Header.Get("Retry-After"), 0)}

	if useXrlHyphen {
		rlstat.XRateLimitLimit = strconvutil.AtoiWithDefault(resp.Header.Get("X-Rate-Limit-Limit"), 0)
		rlstat.XRateLimitRemaining = strconvutil.AtoiWithDefault(resp.Header.Get("X-Rate-Limit-Remaining"), 0)
		rlstat.XRateLimitReset = strconvutil.AtoiWithDefault(resp.Header.Get("X-Rate-Limit-Reset"), 0)
		rlstat.XRateLimitWindow = strconvutil.AtoiWithDefault(resp.Header.Get("X-Rate-Limit-Window"), 0)
	} else {
		rlstat.XRateLimitLimit = strconvutil.AtoiWithDefault(resp.Header.Get("X-RateLimit-Limit"), 0)
		rlstat.XRateLimitRemaining = strconvutil.AtoiWithDefault(resp.Header.Get("X-RateLimit-Remaining"), 0)
		rlstat.XRateLimitReset = strconvutil.AtoiWithDefault(resp.Header.Get("X-RateLimit-Reset"), 0)
		rlstat.XRateLimitWindow = strconvutil.AtoiWithDefault(resp.Header.Get("X-RateLimit-Window"), 0)
	}
	return rlstat
}

type FnLogRateLimitInfo func(RateLimitInfo)
<|endoftext|>


<commit_before>// Package ratelimiter provides possibility to check
// rate limit usage by given resource with given allowed rate
// and time interval. It uses redis as backend so can be used
// to check ratelimit for distributed instances of your app.
package ratelimiter

import (
	"fmt"
	"time"

	"github.com/garyburd/redigo/redis"
)

const expirationWindow = 60

type LimitCtx struct {
	// ported from here: http://flask.pocoo.org/snippets/70/
	ExpireAt  int64
	Key       string
	Limit     int
	Per       time.Duration
	Current   int
	RedisPool *redis.Pool
}

// Returns how many times resource can be used
// before reaching limit
func (self *LimitCtx) Remaining() int {
	return self.Limit - self.Current
}

// Returns whether limit has been reached or not
func (self *LimitCtx) Reached() bool {
	return self.Current > self.Limit
}

// Increments rate limit counter
func (self *LimitCtx) Incr() error {

	c := self.RedisPool.Get()
	defer c.Close()
	key := fmt.Sprintf("rl:%v:%v", self.Key, self.ExpireAt)
	c.Send("MULTI")
	c.Send("INCR", key)
	c.Send("EXPIREAT", key, self.ExpireAt+expirationWindow)
	r, err := redis.Ints(c.Do("EXEC"))
	self.Current = r[0]
	return err
}

// Initializes new LimiterCtx instance which then can be used
// to increment and check ratelimit usage
func BuildLimiter(redisPool *redis.Pool, key string, limit int, per time.Duration) *LimitCtx {
	perSeconds := int64(per.Seconds())
	now := float64(time.Now().Unix())
	expireAt := int64(now/perSeconds)*perSeconds + perSeconds
	return &LimitCtx{
		Key:       key,
		Limit:     limit,
		Per:       per,
		RedisPool: redisPool,
		ExpireAt:  expireAt,
	}
}

// Shorthand function to increment resource usage
// and to get LimiterCtx back. Wrapper around BuildLimiter and LimiterCtx.Incr
func Incr(redisPool *redis.Pool, name string, limit int, period time.Duration) (*LimitCtx, error) {
	limitCtx := BuildLimiter(redisPool, name, limit, period)
	err := limitCtx.Incr()
	return limitCtx, err
}
<commit_msg>fix expire window<commit_after>// Package ratelimiter provides possibility to check
// rate limit usage by given resource with given allowed rate
// and time interval. It uses redis as backend so can be used
// to check ratelimit for distributed instances of your app.
package ratelimiter

import (
	"fmt"
	"math"
	"time"

	"github.com/garyburd/redigo/redis"
)

const expirationWindow = 60

type LimitCtx struct {
	// ported from here: http://flask.pocoo.org/snippets/70/
	ExpireAt  int64
	Key       string
	Limit     int
	Per       time.Duration
	Current   int
	RedisPool *redis.Pool
}

// Returns how many times resource can be used
// before reaching limit
func (self *LimitCtx) Remaining() int {
	return self.Limit - self.Current
}

// Returns whether limit has been reached or not
func (self *LimitCtx) Reached() bool {
	return self.Current > self.Limit
}

// Increments rate limit counter
func (self *LimitCtx) Incr() error {

	c := self.RedisPool.Get()
	defer c.Close()
	key := fmt.Sprintf("rl:%v:%v", self.Key, self.ExpireAt)
	c.Send("MULTI")
	c.Send("INCR", key)
	c.Send("EXPIREAT", key, self.ExpireAt+expirationWindow)
	r, err := redis.Ints(c.Do("EXEC"))
	self.Current = r[0]
	return err
}

// Initializes new LimiterCtx instance which then can be used
// to increment and check ratelimit usage
func BuildLimiter(redisPool *redis.Pool, key string, limit int, per time.Duration) *LimitCtx {
	perSeconds := per.Seconds()
	now := float64(time.Now().Unix())
	expireAt := math.Floor(now/perSeconds)*perSeconds + perSeconds
	return &LimitCtx{
		Key:       key,
		Limit:     limit,
		Per:       per,
		RedisPool: redisPool,
		ExpireAt:  int64(expireAt),
	}
}

// Shorthand function to increment resource usage
// and to get LimiterCtx back. Wrapper around BuildLimiter and LimiterCtx.Incr
func Incr(redisPool *redis.Pool, name string, limit int, period time.Duration) (*LimitCtx, error) {
	limitCtx := BuildLimiter(redisPool, name, limit, period)
	err := limitCtx.Incr()
	return limitCtx, err
}
<|endoftext|>