<commit_before>package libcontainerd

import (
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"syscall"
	"time"

	"github.com/Microsoft/hcsshim"
	"github.com/Sirupsen/logrus"
	"github.com/opencontainers/runtime-spec/specs-go"
)

type container struct {
	containerCommon

	// Platform specific fields are below here. There are none presently on Windows.
	options []CreateOption

	// The ociSpec is required, as client.Create() needs a spec,
	// but can be called from the RestartManager context which does not
	// otherwise have access to the Spec
	ociSpec specs.Spec

	manualStopRequested bool
	hcsContainer        hcsshim.Container
}

func (ctr *container) newProcess(friendlyName string) *process {
	return &process{
		processCommon: processCommon{
			containerID:  ctr.containerID,
			friendlyName: friendlyName,
			client:       ctr.client,
		},
	}
}

// start starts a created container.
// Caller needs to lock container ID before calling this method.
func (ctr *container) start(attachStdio StdioCallback) error {
	var err error
	isServicing := false

	for _, option := range ctr.options {
		if s, ok := option.(*ServicingOption); ok && s.IsServicing {
			isServicing = true
		}
	}

	// Start the container.  If this is a servicing container, this call will block
	// until the container is done with the servicing execution.
	logrus.Debugln("libcontainerd: starting container ", ctr.containerID)
	if err = ctr.hcsContainer.Start(); err != nil {
		logrus.Errorf("libcontainerd: failed to start container: %s", err)
		if err := ctr.terminate(); err != nil {
			logrus.Errorf("libcontainerd: failed to cleanup after a failed Start. %s", err)
		} else {
			logrus.Debugln("libcontainerd: cleaned up after failed Start by calling Terminate")
		}
		return err
	}

	// Note we always tell HCS to
	// create stdout as it's required regardless of '-i' or '-t' options, so that
	// docker can always grab the output through logs. We also tell HCS to always
	// create stdin, even if it's not used - it will be closed shortly. Stderr
	// is only created if it we're not -t.
	createProcessParms := &hcsshim.ProcessConfig{
		EmulateConsole:   ctr.ociSpec.Process.Terminal,
		WorkingDirectory: ctr.ociSpec.Process.Cwd,
		CreateStdInPipe:  !isServicing,
		CreateStdOutPipe: !isServicing,
		CreateStdErrPipe: !ctr.ociSpec.Process.Terminal && !isServicing,
	}
	createProcessParms.ConsoleSize[0] = uint(ctr.ociSpec.Process.ConsoleSize.Height)
	createProcessParms.ConsoleSize[1] = uint(ctr.ociSpec.Process.ConsoleSize.Width)

	// Configure the environment for the process
	createProcessParms.Environment = setupEnvironmentVariables(ctr.ociSpec.Process.Env)
	createProcessParms.CommandLine = strings.Join(ctr.ociSpec.Process.Args, " ")
	createProcessParms.User = ctr.ociSpec.Process.User.Username

	// Start the command running in the container.
	newProcess, err := ctr.hcsContainer.CreateProcess(createProcessParms)
	if err != nil {
		logrus.Errorf("libcontainerd: CreateProcess() failed %s", err)
		if err := ctr.terminate(); err != nil {
			logrus.Errorf("libcontainerd: failed to cleanup after a failed CreateProcess. %s", err)
		} else {
			logrus.Debugln("libcontainerd: cleaned up after failed CreateProcess by calling Terminate")
		}
		return err
	}

	pid := newProcess.Pid()

	// Save the hcs Process and PID
	ctr.process.friendlyName = InitFriendlyName
	ctr.process.hcsProcess = newProcess

	// If this is a servicing container, wait on the process synchronously here and
	// if it succeeds, wait for it cleanly shutdown and merge into the parent container.
	if isServicing {
		exitCode := ctr.waitProcessExitCode(&ctr.process)

		if exitCode != 0 {
			if err := ctr.terminate(); err != nil {
				logrus.Warnf("libcontainerd: terminating servicing container %s failed: %s", ctr.containerID, err)
			}
			return fmt.Errorf("libcontainerd: servicing container %s returned non-zero exit code %d", ctr.containerID, exitCode)
		}

		return ctr.hcsContainer.WaitTimeout(time.Minute * 5)
	}

	var stdout, stderr io.ReadCloser
	var stdin io.WriteCloser
	stdin, stdout, stderr, err = newProcess.Stdio()
	if err != nil {
		logrus.Errorf("libcontainerd: failed to get stdio pipes: %s", err)
		if err := ctr.terminate(); err != nil {
			logrus.Errorf("libcontainerd: failed to cleanup after a failed Stdio. %s", err)
		}
		return err
	}

	iopipe := &IOPipe{Terminal: ctr.ociSpec.Process.Terminal}

	iopipe.Stdin = createStdInCloser(stdin, newProcess)

	// Convert io.ReadClosers to io.Readers
	if stdout != nil {
		iopipe.Stdout = ioutil.NopCloser(&autoClosingReader{ReadCloser: stdout})
	}
	if stderr != nil {
		iopipe.Stderr = ioutil.NopCloser(&autoClosingReader{ReadCloser: stderr})
	}

	// Save the PID
	logrus.Debugf("libcontainerd: process started - PID %d", pid)
	ctr.systemPid = uint32(pid)

	// Spin up a go routine waiting for exit to handle cleanup
	go ctr.waitExit(&ctr.process, true)

	ctr.client.appendContainer(ctr)

	if err := attachStdio(*iopipe); err != nil {
		// OK to return the error here, as waitExit will handle tear-down in HCS
		return err
	}

	// Tell the docker engine that the container has started.
	si := StateInfo{
		CommonStateInfo: CommonStateInfo{
			State: StateStart,
			Pid:   ctr.systemPid, // Not sure this is needed? Double-check monitor.go in daemon BUGBUG @jhowardmsft
		}}
	logrus.Debugf("libcontainerd: start() completed OK, %+v", si)
	return ctr.client.backend.StateChanged(ctr.containerID, si)

}

// waitProcessExitCode will wait for the given process to exit and return its error code.
func (ctr *container) waitProcessExitCode(process *process) int {
	// Block indefinitely for the process to exit.
	err := process.hcsProcess.Wait()
	if err != nil {
		if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE {
			logrus.Warnf("libcontainerd: Wait() failed (container may have been killed): %s", err)
		}
		// Fall through here, do not return. This ensures we attempt to continue the
		// shutdown in HCS and tell the docker engine that the process/container
		// has exited to avoid a container being dropped on the floor.
	}

	exitCode, err := process.hcsProcess.ExitCode()
	if err != nil {
		if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE {
			logrus.Warnf("libcontainerd: unable to get exit code from container %s", ctr.containerID)
		}
		// Since we got an error retrieving the exit code, make sure that the code we return
		// doesn't incorrectly indicate success.
		exitCode = -1

		// Fall through here, do not return. This ensures we attempt to continue the
		// shutdown in HCS and tell the docker engine that the process/container
		// has exited to avoid a container being dropped on the floor.
	}

	return exitCode
}

// waitExit runs as a goroutine waiting for the process to exit. It's
// equivalent to (in the linux containerd world) where events come in for
// state change notifications from containerd.
func (ctr *container) waitExit(process *process, isFirstProcessToStart bool) error {
	logrus.Debugln("libcontainerd: waitExit() on pid", process.systemPid)

	exitCode := ctr.waitProcessExitCode(process)
	// Lock the container while shutting down
	ctr.client.lock(ctr.containerID)

	// Assume the container has exited
	si := StateInfo{
		CommonStateInfo: CommonStateInfo{
			State:     StateExit,
			ExitCode:  uint32(exitCode),
			Pid:       process.systemPid,
			ProcessID: process.friendlyName,
		},
		UpdatePending: false,
	}

	// But it could have been an exec'd process which exited
	if !isFirstProcessToStart {
		si.State = StateExitProcess
		ctr.cleanProcess(process.friendlyName)
	} else {
		updatePending, err := ctr.hcsContainer.HasPendingUpdates()
		if err != nil {
			logrus.Warnf("libcontainerd: HasPendingUpdates() failed (container may have been killed): %s", err)
		} else {
			si.UpdatePending = updatePending
		}

		logrus.Debugf("libcontainerd: shutting down container %s", ctr.containerID)
		if err := ctr.shutdown(); err != nil {
			logrus.Debugf("libcontainerd: failed to shutdown container %s", ctr.containerID)
		} else {
			logrus.Debugf("libcontainerd: completed shutting down container %s", ctr.containerID)
		}
		if err := ctr.hcsContainer.Close(); err != nil {
			logrus.Error(err)
		}

		// Remove process from list if we have exited
		if si.State == StateExit {
			ctr.client.deleteContainer(ctr.containerID)
		}
	}

	if err := process.hcsProcess.Close(); err != nil {
		logrus.Errorf("libcontainerd: hcsProcess.Close(): %v", err)
	}

	// Unlock here before we call back into the daemon to update state
	ctr.client.unlock(ctr.containerID)

	// Call into the backend to notify it of the state change.
	logrus.Debugf("libcontainerd: waitExit() calling backend.StateChanged %+v", si)
	if err := ctr.client.backend.StateChanged(ctr.containerID, si); err != nil {
		logrus.Error(err)
	}

	logrus.Debugf("libcontainerd: waitExit() completed OK, %+v", si)

	return nil
}

// cleanProcess removes process from the map.
// Caller needs to lock container ID before calling this method.
func (ctr *container) cleanProcess(id string) {
	delete(ctr.processes, id)
}

// shutdown shuts down the container in HCS
// Caller needs to lock container ID before calling this method.
func (ctr *container) shutdown() error {
	const shutdownTimeout = time.Minute * 5
	err := ctr.hcsContainer.Shutdown()
	if hcsshim.IsPending(err) {
		// Explicit timeout to avoid a (remote) possibility that shutdown hangs indefinitely.
		err = ctr.hcsContainer.WaitTimeout(shutdownTimeout)
	} else if hcsshim.IsAlreadyStopped(err) {
		err = nil
	}

	if err != nil {
		logrus.Debugf("libcontainerd: error shutting down container %s %v calling terminate", ctr.containerID, err)
		if err := ctr.terminate(); err != nil {
			return err
		}
		return err
	}

	return nil
}

// terminate terminates the container in HCS
// Caller needs to lock container ID before calling this method.
func (ctr *container) terminate() error {
	const terminateTimeout = time.Minute * 5
	err := ctr.hcsContainer.Terminate()

	if hcsshim.IsPending(err) {
		err = ctr.hcsContainer.WaitTimeout(terminateTimeout)
	} else if hcsshim.IsAlreadyStopped(err) {
		err = nil
	}

	if err != nil {
		logrus.Debugf("libcontainerd: error terminating container %s %v", ctr.containerID, err)
		return err
	}

	return nil
}
<commit_msg>Stop holding client container lock during shutdown<commit_after>package libcontainerd

import (
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"syscall"
	"time"

	"github.com/Microsoft/hcsshim"
	"github.com/Sirupsen/logrus"
	"github.com/opencontainers/runtime-spec/specs-go"
)

type container struct {
	containerCommon

	// Platform specific fields are below here. There are none presently on Windows.
	options []CreateOption

	// The ociSpec is required, as client.Create() needs a spec,
	// but can be called from the RestartManager context which does not
	// otherwise have access to the Spec
	ociSpec specs.Spec

	manualStopRequested bool
	hcsContainer        hcsshim.Container
}

func (ctr *container) newProcess(friendlyName string) *process {
	return &process{
		processCommon: processCommon{
			containerID:  ctr.containerID,
			friendlyName: friendlyName,
			client:       ctr.client,
		},
	}
}

// start starts a created container.
// Caller needs to lock container ID before calling this method.
func (ctr *container) start(attachStdio StdioCallback) error {
	var err error
	isServicing := false

	for _, option := range ctr.options {
		if s, ok := option.(*ServicingOption); ok && s.IsServicing {
			isServicing = true
		}
	}

	// Start the container.  If this is a servicing container, this call will block
	// until the container is done with the servicing execution.
	logrus.Debugln("libcontainerd: starting container ", ctr.containerID)
	if err = ctr.hcsContainer.Start(); err != nil {
		logrus.Errorf("libcontainerd: failed to start container: %s", err)
		if err := ctr.terminate(); err != nil {
			logrus.Errorf("libcontainerd: failed to cleanup after a failed Start. %s", err)
		} else {
			logrus.Debugln("libcontainerd: cleaned up after failed Start by calling Terminate")
		}
		return err
	}

	// Note we always tell HCS to
	// create stdout as it's required regardless of '-i' or '-t' options, so that
	// docker can always grab the output through logs. We also tell HCS to always
	// create stdin, even if it's not used - it will be closed shortly. Stderr
	// is only created if it we're not -t.
	createProcessParms := &hcsshim.ProcessConfig{
		EmulateConsole:   ctr.ociSpec.Process.Terminal,
		WorkingDirectory: ctr.ociSpec.Process.Cwd,
		CreateStdInPipe:  !isServicing,
		CreateStdOutPipe: !isServicing,
		CreateStdErrPipe: !ctr.ociSpec.Process.Terminal && !isServicing,
	}
	createProcessParms.ConsoleSize[0] = uint(ctr.ociSpec.Process.ConsoleSize.Height)
	createProcessParms.ConsoleSize[1] = uint(ctr.ociSpec.Process.ConsoleSize.Width)

	// Configure the environment for the process
	createProcessParms.Environment = setupEnvironmentVariables(ctr.ociSpec.Process.Env)
	createProcessParms.CommandLine = strings.Join(ctr.ociSpec.Process.Args, " ")
	createProcessParms.User = ctr.ociSpec.Process.User.Username

	// Start the command running in the container.
	newProcess, err := ctr.hcsContainer.CreateProcess(createProcessParms)
	if err != nil {
		logrus.Errorf("libcontainerd: CreateProcess() failed %s", err)
		if err := ctr.terminate(); err != nil {
			logrus.Errorf("libcontainerd: failed to cleanup after a failed CreateProcess. %s", err)
		} else {
			logrus.Debugln("libcontainerd: cleaned up after failed CreateProcess by calling Terminate")
		}
		return err
	}

	pid := newProcess.Pid()

	// Save the hcs Process and PID
	ctr.process.friendlyName = InitFriendlyName
	ctr.process.hcsProcess = newProcess

	// If this is a servicing container, wait on the process synchronously here and
	// if it succeeds, wait for it cleanly shutdown and merge into the parent container.
	if isServicing {
		exitCode := ctr.waitProcessExitCode(&ctr.process)

		if exitCode != 0 {
			if err := ctr.terminate(); err != nil {
				logrus.Warnf("libcontainerd: terminating servicing container %s failed: %s", ctr.containerID, err)
			}
			return fmt.Errorf("libcontainerd: servicing container %s returned non-zero exit code %d", ctr.containerID, exitCode)
		}

		return ctr.hcsContainer.WaitTimeout(time.Minute * 5)
	}

	var stdout, stderr io.ReadCloser
	var stdin io.WriteCloser
	stdin, stdout, stderr, err = newProcess.Stdio()
	if err != nil {
		logrus.Errorf("libcontainerd: failed to get stdio pipes: %s", err)
		if err := ctr.terminate(); err != nil {
			logrus.Errorf("libcontainerd: failed to cleanup after a failed Stdio. %s", err)
		}
		return err
	}

	iopipe := &IOPipe{Terminal: ctr.ociSpec.Process.Terminal}

	iopipe.Stdin = createStdInCloser(stdin, newProcess)

	// Convert io.ReadClosers to io.Readers
	if stdout != nil {
		iopipe.Stdout = ioutil.NopCloser(&autoClosingReader{ReadCloser: stdout})
	}
	if stderr != nil {
		iopipe.Stderr = ioutil.NopCloser(&autoClosingReader{ReadCloser: stderr})
	}

	// Save the PID
	logrus.Debugf("libcontainerd: process started - PID %d", pid)
	ctr.systemPid = uint32(pid)

	// Spin up a go routine waiting for exit to handle cleanup
	go ctr.waitExit(&ctr.process, true)

	ctr.client.appendContainer(ctr)

	if err := attachStdio(*iopipe); err != nil {
		// OK to return the error here, as waitExit will handle tear-down in HCS
		return err
	}

	// Tell the docker engine that the container has started.
	si := StateInfo{
		CommonStateInfo: CommonStateInfo{
			State: StateStart,
			Pid:   ctr.systemPid, // Not sure this is needed? Double-check monitor.go in daemon BUGBUG @jhowardmsft
		}}
	logrus.Debugf("libcontainerd: start() completed OK, %+v", si)
	return ctr.client.backend.StateChanged(ctr.containerID, si)

}

// waitProcessExitCode will wait for the given process to exit and return its error code.
func (ctr *container) waitProcessExitCode(process *process) int {
	// Block indefinitely for the process to exit.
	err := process.hcsProcess.Wait()
	if err != nil {
		if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE {
			logrus.Warnf("libcontainerd: Wait() failed (container may have been killed): %s", err)
		}
		// Fall through here, do not return. This ensures we attempt to continue the
		// shutdown in HCS and tell the docker engine that the process/container
		// has exited to avoid a container being dropped on the floor.
	}

	exitCode, err := process.hcsProcess.ExitCode()
	if err != nil {
		if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE {
			logrus.Warnf("libcontainerd: unable to get exit code from container %s", ctr.containerID)
		}
		// Since we got an error retrieving the exit code, make sure that the code we return
		// doesn't incorrectly indicate success.
		exitCode = -1

		// Fall through here, do not return. This ensures we attempt to continue the
		// shutdown in HCS and tell the docker engine that the process/container
		// has exited to avoid a container being dropped on the floor.
	}

	return exitCode
}

// waitExit runs as a goroutine waiting for the process to exit. It's
// equivalent to (in the linux containerd world) where events come in for
// state change notifications from containerd.
func (ctr *container) waitExit(process *process, isFirstProcessToStart bool) error {
	logrus.Debugln("libcontainerd: waitExit() on pid", process.systemPid)

	exitCode := ctr.waitProcessExitCode(process)
	// Lock the container while removing the process/container from the list
	ctr.client.lock(ctr.containerID)

	if !isFirstProcessToStart {
		ctr.cleanProcess(process.friendlyName)
	} else {
		ctr.client.deleteContainer(ctr.containerID)
	}

	// Unlock here so other threads are unblocked
	ctr.client.unlock(ctr.containerID)

	// Assume the container has exited
	si := StateInfo{
		CommonStateInfo: CommonStateInfo{
			State:     StateExit,
			ExitCode:  uint32(exitCode),
			Pid:       process.systemPid,
			ProcessID: process.friendlyName,
		},
		UpdatePending: false,
	}

	// But it could have been an exec'd process which exited
	if !isFirstProcessToStart {
		si.State = StateExitProcess
	} else {
		updatePending, err := ctr.hcsContainer.HasPendingUpdates()
		if err != nil {
			logrus.Warnf("libcontainerd: HasPendingUpdates() failed (container may have been killed): %s", err)
		} else {
			si.UpdatePending = updatePending
		}

		logrus.Debugf("libcontainerd: shutting down container %s", ctr.containerID)
		if err := ctr.shutdown(); err != nil {
			logrus.Debugf("libcontainerd: failed to shutdown container %s", ctr.containerID)
		} else {
			logrus.Debugf("libcontainerd: completed shutting down container %s", ctr.containerID)
		}
		if err := ctr.hcsContainer.Close(); err != nil {
			logrus.Error(err)
		}
	}

	if err := process.hcsProcess.Close(); err != nil {
		logrus.Errorf("libcontainerd: hcsProcess.Close(): %v", err)
	}

	// Call into the backend to notify it of the state change.
	logrus.Debugf("libcontainerd: waitExit() calling backend.StateChanged %+v", si)
	if err := ctr.client.backend.StateChanged(ctr.containerID, si); err != nil {
		logrus.Error(err)
	}

	logrus.Debugf("libcontainerd: waitExit() completed OK, %+v", si)

	return nil
}

// cleanProcess removes process from the map.
// Caller needs to lock container ID before calling this method.
func (ctr *container) cleanProcess(id string) {
	delete(ctr.processes, id)
}

// shutdown shuts down the container in HCS
// Caller needs to lock container ID before calling this method.
func (ctr *container) shutdown() error {
	const shutdownTimeout = time.Minute * 5
	err := ctr.hcsContainer.Shutdown()
	if hcsshim.IsPending(err) {
		// Explicit timeout to avoid a (remote) possibility that shutdown hangs indefinitely.
		err = ctr.hcsContainer.WaitTimeout(shutdownTimeout)
	} else if hcsshim.IsAlreadyStopped(err) {
		err = nil
	}

	if err != nil {
		logrus.Debugf("libcontainerd: error shutting down container %s %v calling terminate", ctr.containerID, err)
		if err := ctr.terminate(); err != nil {
			return err
		}
		return err
	}

	return nil
}

// terminate terminates the container in HCS
// Caller needs to lock container ID before calling this method.
func (ctr *container) terminate() error {
	const terminateTimeout = time.Minute * 5
	err := ctr.hcsContainer.Terminate()

	if hcsshim.IsPending(err) {
		err = ctr.hcsContainer.WaitTimeout(terminateTimeout)
	} else if hcsshim.IsAlreadyStopped(err) {
		err = nil
	}

	if err != nil {
		logrus.Debugf("libcontainerd: error terminating container %s %v", ctr.containerID, err)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 Xing Xing <mikespook@gmail.com> All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

/*
This module is Gearman API for golang.
The protocol was implemented by native way.
*/

package gearman

import (
	"github.com/mikespook/gearman-go/client"
	"github.com/mikespook/gearman-go/worker"
	"strings"
	"sync"
	"testing"
	"time"
)

const (
	STR      = "The gearman-go is a pure go implemented library."
	GEARMAND = "127.0.0.1:4730"
)

func ToUpper(job worker.Job) ([]byte, error) {
	data := job.Data()
	data = []byte(strings.ToUpper(string(data)))
	return data, nil
}

func Sleep(job worker.Job) ([]byte, error) {
	time.Sleep(time.Second * 5)
	return nil, nil
}

func TestJobs(t *testing.T) {
	w := worker.New(worker.Unlimited)
	if err := w.AddServer("tcp4", GEARMAND); err != nil {
		t.Error(err)
		return
	}
	defer w.Close()
	t.Log("Servers added...")
	if err := w.AddFunc("ToUpper", ToUpper, 0); err != nil {
		t.Error(err)
		return
	}
	if err := w.AddFunc("Sleep", Sleep, 0); err != nil {
		t.Error(err)
		return
	}
	t.Log("Functions added...")
	w.ErrorHandler = func(e error) {
		t.Error(e)
	}
	go w.Work()
	t.Log("Worker is running...")

	c, err := client.New("tcp4", GEARMAND)
	if err != nil {
		t.Error(err)
		return
	}
	defer c.Close()

	c.ErrorHandler = func(e error) {
		t.Log(e)
	}

	{
		var w sync.WaitGroup
		jobHandler := func(job *client.Response) {
			upper := strings.ToUpper(STR)
			if string(job.Data) != upper {
				t.Errorf("%s expected, got %s", upper, job.Data)
			}
			w.Done()
		}

		w.Add(1)
		handle, err := c.Do("ToUpper", []byte(STR), client.JOB_NORMAL, jobHandler)
		if err != nil {
			t.Error(err)
			return
		}
		w.Wait()
		status, err := c.Status(handle)
		if err != nil {
			t.Error(err)
			return
		}
		if status.Known {
			t.Errorf("%s shouldn't be known", status.Handle)
			return
		}

		if status.Running {
			t.Errorf("%s shouldn't be running", status.Handle)
		}
	}
	{
		handle, err := c.DoBg("Sleep", nil, client.JOB_NORMAL)
		if err != nil {
			t.Error(err)
			return
		}
		time.Sleep(time.Second)
		status, err := c.Status(handle)
		if err != nil {
			t.Error(err)
			return
		}

		if !status.Known {
			t.Errorf("%s should be known", status.Handle)
			return
		}

		if !status.Running {
			t.Errorf("%s should be running", status.Handle)
		}
	}
	{
		status, err := c.Status("not exists handle")
		if err != nil {
			t.Error(err)
			return
		}

		if status.Known {
			t.Errorf("%s shouldn't be known", status.Handle)
			return
		}

		if status.Running {
			t.Errorf("%s shouldn't be running", status.Handle)
		}
	}
}
<commit_msg>fixed global test-case<commit_after>// Copyright 2011 Xing Xing <mikespook@gmail.com> All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

/*
This module is Gearman API for golang.
The protocol was implemented by native way.
*/

package gearman

import (
	"github.com/mikespook/gearman-go/client"
	"github.com/mikespook/gearman-go/worker"
	"strings"
	"sync"
	"testing"
	"time"
)

const (
	STR      = "The gearman-go is a pure go implemented library."
	GEARMAND = "127.0.0.1:4730"
)

func ToUpper(job worker.Job) ([]byte, error) {
	data := job.Data()
	data = []byte(strings.ToUpper(string(data)))
	return data, nil
}

func Sleep(job worker.Job) ([]byte, error) {
	time.Sleep(time.Second * 5)
	return nil, nil
}

func TestJobs(t *testing.T) {
	w := worker.New(worker.Unlimited)
	if err := w.AddServer("tcp4", GEARMAND); err != nil {
		t.Error(err)
		return
	}
	defer w.Close()
	t.Log("Servers added...")
	if err := w.AddFunc("ToUpper", ToUpper, 0); err != nil {
		t.Error(err)
		return
	}
	if err := w.AddFunc("Sleep", Sleep, 0); err != nil {
		t.Error(err)
		return
	}
	t.Log("Functions added...")
	w.ErrorHandler = func(e error) {
		t.Error(e)
	}
	if err := w.Ready(); err != nil {
		t.Error(err)
		return
	}
	go w.Work()
	t.Log("Worker is running...")

	c, err := client.New("tcp4", GEARMAND)
	if err != nil {
		t.Error(err)
		return
	}
	defer c.Close()

	c.ErrorHandler = func(e error) {
		t.Log(e)
	}

	{
		var w sync.WaitGroup
		jobHandler := func(job *client.Response) {
			upper := strings.ToUpper(STR)
			if string(job.Data) != upper {
				t.Errorf("%s expected, got %s", upper, job.Data)
			}
			w.Done()
		}

		w.Add(1)
		handle, err := c.Do("ToUpper", []byte(STR), client.JOB_NORMAL, jobHandler)
		if err != nil {
			t.Error(err)
			return
		}
		w.Wait()
		status, err := c.Status(handle)
		if err != nil {
			t.Error(err)
			return
		}
		if status.Known {
			t.Errorf("%s shouldn't be known", status.Handle)
			return
		}

		if status.Running {
			t.Errorf("%s shouldn't be running", status.Handle)
		}
	}
	{
		handle, err := c.DoBg("Sleep", nil, client.JOB_NORMAL)
		if err != nil {
			t.Error(err)
			return
		}
		time.Sleep(time.Second)
		status, err := c.Status(handle)
		if err != nil {
			t.Error(err)
			return
		}

		if !status.Known {
			t.Errorf("%s should be known", status.Handle)
			return
		}

		if !status.Running {
			t.Errorf("%s should be running", status.Handle)
		}
	}
	{
		status, err := c.Status("not exists handle")
		if err != nil {
			t.Error(err)
			return
		}

		if status.Known {
			t.Errorf("%s shouldn't be known", status.Handle)
			return
		}

		if status.Running {
			t.Errorf("%s shouldn't be running", status.Handle)
		}
	}
}
<|endoftext|>


<commit_before>package models

import "time"

// Publication publication struct
type Publication struct {
	Context   []string `json:"@context,omitempty"`
	Metadata  Metadata `json:"metadata"`
	Links     []Link   `json:"links"`
	Spine     []Link   `json:"spine"`
	Resources []Link   `json:"resources,omitempty"`
}

// Metadata metadata struct
type Metadata struct {
	RDFType         string        `json:"@type,omitempty"`
	Title           string        `json:"title"`
	Author          []Contributor `json:"author,omitempty"`
	Translator      []Contributor `json:"translator,omitempty"`
	Editor          []Contributor `json:"editor,omitempty"`
	Artist          []Contributor `json:"artist,omitempty"`
	Illustrator     []Contributor `json:"illustrator,omitempty"`
	Letterer        []Contributor `json:"letterer,omitempty"`
	Penciler        []Contributor `json:"penciler,omitempty"`
	Colorist        []Contributor `json:"colorist,omitempty"`
	Inker           []Contributor `json:"inker,omitempty"`
	Narrator        []Contributor `json:"narrator,omitempty"`
	Contributor     []Contributor `json:"contributor,omitempty"`
	Publisher       []Contributor `json:"publisher,omitempty"`
	Imprint         []Contributor `json:"imprint,omitempty"`
	Identifier      string        `json:"identifier"`
	Language        []string      `json:"language,omitempty"`
	Modified        *time.Time    `json:"modified,omitempty"`
	PublicationDate *time.Time    `json:"published,omitempty"`
	Description     string        `json:"description,omitempty"`
	Direction       string        `json:"direction,omitempty"`
	Rendition       Rendition     `json:"rendition,omitempty"`
	Source          string        `json:"source,omitempty"`
	EpubType        []string      `json:"epub-type,omitempty"`
	Right           string        `json:"right,omitempty"`
	Subject         []Subject     `json:"subject,omitempty"`
}

// Link link struct
type Link struct {
	Href       string         `json:"href"`
	TypeLink   string         `json:"type"`
	Rel        []string       `json:"rel,omitempty"`
	Height     int            `json:"height,omitempty"`
	Width      int            `json:"width,omitempty"`
	Title      string         `json:"title,omitempty"`
	Properties []string       `json:"properties,omitempty"`
	Duration   *time.Duration `json:"duration,omitempty"`
	Templated  bool           `json:"templated,omitempty"`
}

// Contributor shared Contributor struct
type Contributor struct {
	Name       string `json:"name"`
	SortAs     string `json:"sort_as,omitempty"`
	Identifier string `json:"identifier,omitempty"`
	Role       string `json:"role,omitempty"`
}

// Rendition rendition
type Rendition struct {
	Flow        string `json:"flow,omitempty"`
	Layout      string `json:"layout,omitempty"`
	Orientation string `json:"orientation,omitempty"`
	Spread      string `json:"spread,omitempty"`
}

// Subject subject
type Subject struct {
	Name   string `json:"name"`
	SortAs string `json:"sort_as,omitempty"`
	Scheme string `json:"scheme,omitempty"`
	Code   string `json:"code,omitempty"`
}

// BelongsTo belongs to
type BelongsTo struct {
	Series     []Collection `json:"series,omitempty"`
	Collection []Collection `json:"collection,omitempty"`
}

// Collection shared Collection struct
type Collection struct {
	Name       string  `json:"name"`
	SortAs     string  `json:"sort_as,omitempty"`
	Identifier string  `json:"identifier,omitempty"`
	Position   float32 `json:"position,omitempty"`
}

func (publication *Publication) linkCover() {
	// returns the link object for the cover
}

func (publication *Publication) linkNavDoc() {

}
<commit_msg>add comments to publication struct<commit_after>package models

import "time"

// Publication publication struct
type Publication struct {
	Context   []string `json:"@context,omitempty"`
	Metadata  Metadata `json:"metadata"`
	Links     []Link   `json:"links"`
	Spine     []Link   `json:"spine"`
	Resources []Link   `json:"resources,omitempty"`

	TOC       []Link `json:"toc,omitempty"`
	PageList  []Link `json:"page-list,omitempty"`
	Landmarks []Link `json:"landmarks,omitempty"`
	LOI       []Link `json:"loi,omitempty"`
	LOA       []Link `json:"loa,omitempty"`
	LOV       []Link `json:"lov,omitempty"`
	LOT       []Link `json:"lot,omitempty"`
}

// Metadata metadata struct
type Metadata struct {
	RDFType         string        `json:"@type,omitempty"`
	Title           string        `json:"title"`
	Author          []Contributor `json:"author,omitempty"`
	Translator      []Contributor `json:"translator,omitempty"`
	Editor          []Contributor `json:"editor,omitempty"`
	Artist          []Contributor `json:"artist,omitempty"`
	Illustrator     []Contributor `json:"illustrator,omitempty"`
	Letterer        []Contributor `json:"letterer,omitempty"`
	Penciler        []Contributor `json:"penciler,omitempty"`
	Colorist        []Contributor `json:"colorist,omitempty"`
	Inker           []Contributor `json:"inker,omitempty"`
	Narrator        []Contributor `json:"narrator,omitempty"`
	Contributor     []Contributor `json:"contributor,omitempty"`
	Publisher       []Contributor `json:"publisher,omitempty"`
	Imprint         []Contributor `json:"imprint,omitempty"`
	Identifier      string        `json:"identifier"`
	Language        []string      `json:"language,omitempty"`
	Modified        *time.Time    `json:"modified,omitempty"`
	PublicationDate *time.Time    `json:"published,omitempty"`
	Description     string        `json:"description,omitempty"`
	Direction       string        `json:"direction,omitempty"`
	Rendition       Rendition     `json:"rendition,omitempty"`
	Source          string        `json:"source,omitempty"`
	EpubType        []string      `json:"epub-type,omitempty"`
	Right           string        `json:"right,omitempty"`
	Subject         []Subject     `json:"subject,omitempty"`
}

// Link link struct
type Link struct {
	Href       string         `json:"href"`
	TypeLink   string         `json:"type"`
	Rel        []string       `json:"rel,omitempty"`
	Height     int            `json:"height,omitempty"`
	Width      int            `json:"width,omitempty"`
	Title      string         `json:"title,omitempty"`
	Properties []string       `json:"properties,omitempty"`
	Duration   *time.Duration `json:"duration,omitempty"`
	Templated  bool           `json:"templated,omitempty"`
}

// Contributor shared Contributor struct
type Contributor struct {
	Name       string `json:"name"`
	SortAs     string `json:"sort_as,omitempty"`
	Identifier string `json:"identifier,omitempty"`
	Role       string `json:"role,omitempty"`
}

// Rendition rendition
type Rendition struct {
	Flow        string `json:"flow,omitempty"`
	Layout      string `json:"layout,omitempty"`
	Orientation string `json:"orientation,omitempty"`
	Spread      string `json:"spread,omitempty"`
}

// Subject subject
type Subject struct {
	Name   string `json:"name"`
	SortAs string `json:"sort_as,omitempty"`
	Scheme string `json:"scheme,omitempty"`
	Code   string `json:"code,omitempty"`
}

// BelongsTo belongs to
type BelongsTo struct {
	Series     []Collection `json:"series,omitempty"`
	Collection []Collection `json:"collection,omitempty"`
}

// Collection shared Collection struct
type Collection struct {
	Name       string  `json:"name"`
	SortAs     string  `json:"sort_as,omitempty"`
	Identifier string  `json:"identifier,omitempty"`
	Position   float32 `json:"position,omitempty"`
}

func (publication *Publication) linkCover() {
	// returns the link object for the cover
}

func (publication *Publication) linkNavDoc() {

}
<|endoftext|>


<commit_before>// Copyright © 2014-2015, Civis Analytics

package gelf

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"time"
	"unicode/utf8"
)

// Message meets the Graylog2 Extended Log Format.
// http://graylog2.org/gelf#specs
type Message struct {
	Version          string                 `json:"version"`
	Host             string                 `json:"host"`
	ShortMessage     string                 `json:"short_message"`
	FullMessage      string                 `json:"full_message,omitempty"`
	Timestamp        int64                  `json:"timestamp"`
	Level            Level                  `json:"level"`
	AdditionalFields string                 `json:",omitempty"`
	additional       map[string]interface{} `json:"a,omitempty"`
}

var reservedFields = []string{"version", "host", "short_message", "full_message", "timestamp", "level", "_id"}

// NewMessage returns a new Graylog2 Extended Log Format message.
func NewMessage(l Level, short string, full string) (*Message, error) {
	a := make(map[string]interface{})

	host, err := os.Hostname()
	if err != nil {
		return nil, err
	}

	return &Message{
		Version:      GELFVersion,
		Host:         host,
		ShortMessage: short,
		FullMessage:  full,
		Timestamp:    time.Now().UnixNano(),
		Level:        l,
		additional:   a,
	}, nil
}

func typeOf(v interface{}) string {
	return fmt.Sprintf("%T", v)
}

// Add will add additional fields to a message in the form of a key and value
// pair. Values can be of string or int type.
func (m *Message) Add(key string, value interface{}) error {
	// Verify additional fields against reserved field names.
	// If field is not reserved, add to message.
	for _, rf := range reservedFields {
		if key == rf {
			return fmt.Errorf("Invalid field[%s]", key)
		}
	}

	// Verify value is a string or int.
	if typeOf(value) != "string" && typeOf(value) != "int64" && typeOf(value) != "int" {
		return fmt.Errorf("Invalid field type[%s]", typeOf(value))
	}

	// Verify underscore prefix
	r, _ := utf8.DecodeRuneInString(key)
	if string(r) == "_" {
		m.additional[key] = value
	} else {
		m.additional["_"+key] = value
	}

	return nil
}

// String is a convience method that meets the fmt.String interface providing an
// easy way to print the string JSON representation of a message.
func (m *Message) String() string {
	if len(m.additional) == 0 {
		baseMessageFields, _ := json.Marshal(m)
		return string(baseMessageFields)
	}

	// Maps do not marshal to JSON as top-level objects.
	// To work around we marshal the map of additional fields, modify the string
	// and append to the outbound JSON encoded struct.
	additionalFields, _ := json.Marshal(m.additional)
	filteredFields := strings.Replace(string(additionalFields[1:]), "\\\"", "\"", -1)

	baseMessageFields, _ := json.Marshal(m)
	trimBaseMessageFields := strings.TrimRight(string(baseMessageFields), "}")

	return trimBaseMessageFields + "," + filteredFields
}
<commit_msg>Verify JavaScript types<commit_after>// Copyright © 2014-2015, Civis Analytics

package gelf

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"time"
	"unicode/utf8"
)

// Message meets the Graylog2 Extended Log Format.
// http://graylog2.org/gelf#specs
type Message struct {
	Version          string                 `json:"version"`
	Host             string                 `json:"host"`
	ShortMessage     string                 `json:"short_message"`
	FullMessage      string                 `json:"full_message,omitempty"`
	Timestamp        int64                  `json:"timestamp"`
	Level            Level                  `json:"level"`
	AdditionalFields string                 `json:",omitempty"`
	additional       map[string]interface{} `json:"a,omitempty"`
}

var reservedFields = []string{"version", "host", "short_message", "full_message", "timestamp", "level", "_id"}

// NewMessage returns a new Graylog2 Extended Log Format message.
func NewMessage(l Level, short string, full string) (*Message, error) {
	a := make(map[string]interface{})

	host, err := os.Hostname()
	if err != nil {
		return nil, err
	}

	return &Message{
		Version:      GELFVersion,
		Host:         host,
		ShortMessage: short,
		FullMessage:  full,
		Timestamp:    time.Now().UnixNano(),
		Level:        l,
		additional:   a,
	}, nil
}

func typeOf(v interface{}) string {
	return fmt.Sprintf("%T", v)
}

// Add will add additional fields to a message in the form of a key and value
// pair. Values can be of JavaScript string or number type.
func (m *Message) Add(key string, value interface{}) error {
	// Verify additional fields against reserved field names.
	// If field is not reserved, add to message.
	for _, rf := range reservedFields {
		if key == rf {
			return fmt.Errorf("Invalid field[%s]", key)
		}
	}

	// Verify value is a JavaScript string or number.
	if typeOf(value) != "string" && typeOf(value) != "float64" && typeOf(value) != "int" {
		return fmt.Errorf("Invalid field type[%s]", typeOf(value))
	}

	// Verify underscore prefix
	r, _ := utf8.DecodeRuneInString(key)
	if string(r) == "_" {
		m.additional[key] = value
	} else {
		m.additional["_"+key] = value
	}

	return nil
}

// String is a convience method that meets the fmt.String interface providing an
// easy way to print the string JSON representation of a message.
func (m *Message) String() string {
	if len(m.additional) == 0 {
		baseMessageFields, _ := json.Marshal(m)
		return string(baseMessageFields)
	}

	// Maps do not marshal to JSON as top-level objects.
	// To work around we marshal the map of additional fields, modify the string
	// and append to the outbound JSON encoded struct.
	additionalFields, _ := json.Marshal(m.additional)
	filteredFields := strings.Replace(string(additionalFields[1:]), "\\\"", "\"", -1)

	baseMessageFields, _ := json.Marshal(m)
	trimBaseMessageFields := strings.TrimRight(string(baseMessageFields), "}")

	return trimBaseMessageFields + "," + filteredFields
}
<|endoftext|>


<commit_before>package twitterbot

import (
	"fmt"
	"time"

	"github.com/ChimeraCoder/anaconda"
)

type TBot struct {
	api  *anaconda.TwitterApi
	keys *Keys
}

func New(config string) (*TBot, error) {
	keys, err := ReadConfig(config)
	if err != nil {
		return nil, err
	}

	anaconda.SetConsumerKey(keys.consumerPublic)
	anaconda.SetConsumerSecret(keys.consumerSecret)
	api := anaconda.NewTwitterApi(keys.accessPublic, keys.accessSecret)

	return &TBot{api, keys}, nil
}

type TweetCreator interface {
	NextTweet() string
}

func (t *TBot) Run(creator TweetCreator) {
	var previousTweet string

	for {
		tweet := creator.NextTweet()
		if previousTweet == "" || previousTweet != tweet {
			fmt.Println("[" + time.Now().Format(time.RFC850) + "] Posting " + tweet)
			t.api.PostTweet(tweet, nil)
			previousTweet = tweet
		}
		fmt.Println("[" + time.Now().Format(time.RFC850) + "] Sleeping...")
		time.Sleep(10 * time.Minute)
	}
}
<commit_msg>Remove `keys` from the TBot struct.<commit_after>package twitterbot

import (
	"fmt"
	"time"

	"github.com/ChimeraCoder/anaconda"
)

type TBot struct {
	api  *anaconda.TwitterApi
}

func New(config string) (*TBot, error) {
	keys, err := ReadConfig(config)
	if err != nil {
		return nil, err
	}

	anaconda.SetConsumerKey(keys.consumerPublic)
	anaconda.SetConsumerSecret(keys.consumerSecret)
	api := anaconda.NewTwitterApi(keys.accessPublic, keys.accessSecret)

	return &TBot{api}, nil
}

type TweetCreator interface {
	NextTweet() string
}

func (t *TBot) Run(creator TweetCreator) {
	var previousTweet string

	for {
		tweet := creator.NextTweet()
		if previousTweet == "" || previousTweet != tweet {
			fmt.Println("[" + time.Now().Format(time.RFC850) + "] Posting " + tweet)
			t.api.PostTweet(tweet, nil)
			previousTweet = tweet
		}
		fmt.Println("[" + time.Now().Format(time.RFC850) + "] Sleeping...")
		time.Sleep(10 * time.Minute)
	}
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package beam

import (
	"bytes"
	"fmt"
	"reflect"

	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
)

// Create inserts a fixed non-empty set of values into the pipeline. The values must
// be of the same type 'A' and the returned PCollection is of type A.
//
// The returned PCollections can be used as any other PCollections. The values
// are JSON-coded. Each runner may place limits on the sizes of the values and
// Create should generally only be used for small collections.
func Create(s Scope, values ...interface{}) PCollection {
	return Must(TryCreate(s, values...))
}

// CreateList inserts a fixed set of values into the pipeline from a slice or
// array. Unlike Create this supports the creation of an empty PCollection.
func CreateList(s Scope, list interface{}) PCollection {
	val := reflect.ValueOf(list)
	if val.Kind() != reflect.Slice && val.Kind() != reflect.Array {
		panic(fmt.Sprintf("Input %v must be a slice or array", list))
	}
	var ret []interface{}
	for i := 0; i < val.Len(); i++ {
		ret = append(ret, val.Index(i).Interface())
	}
	var t reflect.Type
	if len(ret) == 0 {
		t = reflect.TypeOf(list).Elem()
	} else {
		t = reflect.ValueOf(ret[0]).Type()
	}
	return Must(TryCreateList(s, ret, t))
}

func addCreateCtx(err error, s Scope) error {
	return errors.WithContextf(err, "inserting Create in scope %s", s)
}

// TryCreate inserts a fixed non-empty set of values into the pipeline. The
// values must be of the same type.
func TryCreate(s Scope, values ...interface{}) (PCollection, error) {
	if len(values) == 0 {
		return PCollection{}, addCreateCtx(errors.New("create has no values"), s)
	}

	t := reflect.ValueOf(values[0]).Type()
	return TryCreateList(s, values, t)
}

// TryCreateList inserts a fixed set of values into the pipeline from a slice or
// array. The values must be of the same type. Unlike TryCreate this supports
// the creation of an empty PCollection.
func TryCreateList(s Scope, values []interface{}, t reflect.Type) (PCollection, error) {
	fn := &createFn{Type: EncodedType{T: t}}
	enc := NewElementEncoder(t)

	for i, value := range values {
		if other := reflect.ValueOf(value).Type(); other != t {
			err := errors.Errorf("value %v at index %v has type %v, want %v", value, i, other, t)
			return PCollection{}, addCreateCtx(err, s)
		}
		var buf bytes.Buffer
		if err := enc.Encode(value, &buf); err != nil {
			return PCollection{}, addCreateCtx(errors.Wrapf(err, "marshalling of %v failed", value), s)
		}
		fn.Values = append(fn.Values, buf.Bytes())
	}

	imp := Impulse(s)

	ret, err := TryParDo(s, fn, imp, TypeDefinition{Var: TType, T: t})
	if err != nil || len(ret) != 1 {
		panic(addCreateCtx(errors.WithContext(err, "internal error"), s))
	}
	return ret[0], nil
}

// TODO(herohde) 6/26/2017: make 'create' a SDF once supported. See BEAM-2421.

type createFn struct {
	Values [][]byte    `json:"values"`
	Type   EncodedType `json:"type"`
}

func (c *createFn) ProcessElement(_ []byte, emit func(T)) error {
	dec := NewElementDecoder(c.Type.T)
	for _, val := range c.Values {
		element, err := dec.Decode(bytes.NewBuffer(val))
		if err != nil {
			return err
		}
		emit(element)
	}
	return nil
}
<commit_msg>Refactor CreateList<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package beam

import (
	"bytes"
	"reflect"

	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
)

// Create inserts a fixed non-empty set of values into the pipeline. The values must
// be of the same type 'A' and the returned PCollection is of type A.
//
// The returned PCollections can be used as any other PCollections. The values
// are JSON-coded. Each runner may place limits on the sizes of the values and
// Create should generally only be used for small collections.
func Create(s Scope, values ...interface{}) PCollection {
	return Must(TryCreate(s, values...))
}

// CreateList inserts a fixed set of values into the pipeline from a slice or
// array. Unlike Create this supports the creation of an empty PCollection.
func CreateList(s Scope, list interface{}) PCollection {
	return Must(TryCreateList(s, list))
}

// TryCreate inserts a fixed non-empty set of values into the pipeline. The
// values must be of the same type.
func TryCreate(s Scope, values ...interface{}) (PCollection, error) {
	if len(values) == 0 {
		err := errors.New("create has no values")
		return PCollection{}, addCreateCtx(err, s)
	}

	t := reflect.ValueOf(values[0]).Type()
	return createList(s, values, t)
}

// TryCreateList inserts a fixed set of values into the pipeline from a slice or
// array. The values must be of the same type. Unlike TryCreate this supports
// the creation of an empty PCollection.
func TryCreateList(s Scope, list interface{}) (PCollection, error) {
	val := reflect.ValueOf(list)
	if val.Kind() != reflect.Slice && val.Kind() != reflect.Array {
		err := errors.Errorf("input %v must be a slice or array", list)
		return PCollection{}, addCreateCtx(err, s)
	}

	var ret []interface{}
	for i := 0; i < val.Len(); i++ {
		ret = append(ret, val.Index(i).Interface())
	}

	var t reflect.Type
	if len(ret) == 0 {
		t = reflect.TypeOf(list).Elem()
	} else {
		t = reflect.ValueOf(ret[0]).Type()
	}
	return createList(s, ret, t)
}

func addCreateCtx(err error, s Scope) error {
	return errors.WithContextf(err, "inserting Create in scope %s", s)
}

func createList(s Scope, values []interface{}, t reflect.Type) (PCollection, error) {
	fn := &createFn{Type: EncodedType{T: t}}
	enc := NewElementEncoder(t)

	for i, value := range values {
		if other := reflect.ValueOf(value).Type(); other != t {
			err := errors.Errorf("value %v at index %v has type %v, want %v", value, i, other, t)
			return PCollection{}, addCreateCtx(err, s)
		}
		var buf bytes.Buffer
		if err := enc.Encode(value, &buf); err != nil {
			err = errors.Wrapf(err, "marshalling of %v failed", value)
			return PCollection{}, addCreateCtx(err, s)
		}
		fn.Values = append(fn.Values, buf.Bytes())
	}

	imp := Impulse(s)

	ret, err := TryParDo(s, fn, imp, TypeDefinition{Var: TType, T: t})
	if err != nil || len(ret) != 1 {
		panic(addCreateCtx(errors.WithContext(err, "internal error"), s))
	}
	return ret[0], nil
}

// TODO(herohde) 6/26/2017: make 'create' a SDF once supported. See BEAM-2421.

type createFn struct {
	Values [][]byte    `json:"values"`
	Type   EncodedType `json:"type"`
}

func (c *createFn) ProcessElement(_ []byte, emit func(T)) error {
	dec := NewElementDecoder(c.Type.T)
	for _, val := range c.Values {
		element, err := dec.Decode(bytes.NewBuffer(val))
		if err != nil {
			return err
		}
		emit(element)
	}
	return nil
}
<|endoftext|>


<commit_before>package git

import (
	"github.com/bmizerany/assert"
	"strings"
	"testing"
)

func TestGitDir(t *testing.T) {
	gitDir, _ := Dir()
	assert.T(t, strings.Contains(gitDir, ".git"))
}

func TestGitPullReqMsgFile(t *testing.T) {
	gitPullReqMsgFile, _ := PullReqMsgFile()
	assert.T(t, strings.Contains(gitPullReqMsgFile, "PULLREQ_EDITMSG"))
}

func TestGitEditor(t *testing.T) {
	gitEditor, err := Editor()
	if err == nil {
		assert.NotEqual(t, "", gitEditor)
	}
}

func TestGitRemote(t *testing.T) {
	gitRemote, _ := OriginRemote()
	assert.Equal(t, "origin", gitRemote.Name)
	assert.T(t, strings.Contains(gitRemote.URL, "gh"))
}

func TestGitHead(t *testing.T) {
	gitHead, _ := Head()
	assert.NotEqual(t, "", gitHead)
}

func TestGitLog(t *testing.T) {
	logs, _ := Log("master", "HEAD")
	assert.T(t, len(logs) >= 0)
}

func TestGitRef(t *testing.T) {
	gitRef, err := Ref("master")
	assert.Equal(t, nil, err)
	assert.NotEqual(t, "", gitRef)
}

func TestGitRefList(t *testing.T) {
	refList, err := RefList("e357a98a1a580b09d4f1d9bf613a6a51e131ef6e", "49e984e2fe86f68c386aeb133b390d39e4264ec1")
	assert.Equal(t, nil, err)
	assert.Equal(t, 1, len(refList))

	assert.Equal(t, "49e984e2fe86f68c386aeb133b390d39e4264ec1", refList[0])
}

func TestGitShow(t *testing.T) {
	output, err := Show("ce20e63ad00751bfed5d08072b11cf1b43af1995")
	assert.Equal(t, nil, err)
	assert.Equal(t, "Add Git.RefList", output)
}
<commit_msg>Fix test failure<commit_after>package git

import (
	"github.com/bmizerany/assert"
	"strings"
	"testing"
)

func TestGitDir(t *testing.T) {
	gitDir, _ := Dir()
	assert.T(t, strings.Contains(gitDir, ".git"))
}

func TestGitPullReqMsgFile(t *testing.T) {
	gitPullReqMsgFile, _ := PullReqMsgFile()
	assert.T(t, strings.Contains(gitPullReqMsgFile, "PULLREQ_EDITMSG"))
}

func TestGitEditor(t *testing.T) {
	gitEditor, err := Editor()
	if err == nil {
		assert.NotEqual(t, "", gitEditor)
	}
}

func TestGitRemote(t *testing.T) {
	gitRemote, _ := OriginRemote()
	assert.Equal(t, "origin", gitRemote.Name)
	assert.T(t, strings.Contains(gitRemote.URL, "gh"))
}

func TestGitHead(t *testing.T) {
	gitHead, _ := Head()
	assert.NotEqual(t, "", gitHead)
}

func TestGitLog(t *testing.T) {
	log, err := Log("e357a98a1a580b09d4f1d9bf613a6a51e131ef6e", "49e984e2fe86f68c386aeb133b390d39e4264ec1")
	assert.Equal(t, nil, err)
	assert.NotEqual(t, "", log)
}

func TestGitRef(t *testing.T) {
	gitRef, err := Ref("1c1077c052d32a83aa13a8afaa4a9630d2f28ef6")
	assert.Equal(t, nil, err)
	assert.Equal(t, "1c1077c052d32a83aa13a8afaa4a9630d2f28ef6", gitRef)
}

func TestGitRefList(t *testing.T) {
	refList, err := RefList("e357a98a1a580b09d4f1d9bf613a6a51e131ef6e", "49e984e2fe86f68c386aeb133b390d39e4264ec1")
	assert.Equal(t, nil, err)
	assert.Equal(t, 1, len(refList))

	assert.Equal(t, "49e984e2fe86f68c386aeb133b390d39e4264ec1", refList[0])
}

func TestGitShow(t *testing.T) {
	output, err := Show("ce20e63ad00751bfed5d08072b11cf1b43af1995")
	assert.Equal(t, nil, err)
	assert.Equal(t, "Add Git.RefList", output)
}
<|endoftext|>


<commit_before>package fs

import (
	"testing"

	"github.com/robertkrimen/otto"
)

func TestFS(t *testing.T) {
	vm := otto.New()
	vm.Set("error", func(call otto.FunctionCall) otto.Value {
		name, _ := call.Argument(0).ToString()
		t.Error(name)
		return otto.UndefinedValue()
	})
	vm.Set("FS", NewFS(vm))
	var fsTest = `
try{
// Open a new file
name="sample.txt";
FS.writeFile(name,"");
var f=FS.open("sample.txt");
var msg="hello";
f.write(msg);
f.flush();
}catch(e){
	error(e);
}
`
	_, err := vm.Eval(fsTest)
	if err != nil {
		t.Error(err)
	}
}
<commit_msg>Update test for fs module<commit_after>package fs

import (
	"testing"

	"github.com/robertkrimen/otto"
)

func TestFS(t *testing.T) {
	vm := otto.New()
	vm.Set("error", func(call otto.FunctionCall) otto.Value {
		name, _ := call.Argument(0).ToString()
		t.Error(name)
		return otto.UndefinedValue()
	})
	vm.Set("FS", NewFS(vm))
	var fsTest = `
try{
// Open a new file
name="sample.txt";
FS.writeFile(name,"");
var f=FS.open("sample.txt");
var msg="hello";
f.write(msg);
f.flush();
FS.remove(name);
}catch(e){
	error(e);
}
`
	_, err := vm.Eval(fsTest)
	if err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package gohost

import (
	"errors"
	"math"
	"net/http"

	"github.com/grpc-ecosystem/go-grpc-middleware"
	"google.golang.org/grpc"

	// hosts pprof endpoint
	_ "net/http/pprof"
)

const (
	// DefaultMaxSendMsgSize is the default max send message size, per gRPC
	DefaultMaxSendMsgSize = 1024 * 1024 * 4

	// DefaultMaxRecvMsgSize is the default max receive message size, per gRPC
	DefaultMaxRecvMsgSize = 1024 * 1024 * 4
)

// Hoster is used to serve gRPC and HTTP endpoints.
type Hoster struct {
	// Service contains the actual implementation of the service calls. Additionally implement the HTTPService interface if an HTTP endpoint is desired.
	Service GRPCService

	// GRPCAddr is the endpoint (host and port) on which to host the gRPC service.
	GRPCAddr string

	// HTTPAddr is the endpoint (host and port) on which to host the HTTP service. May be left blank if not using HTTP.
	HTTPAddr string

	// PPROFAddr is the endpoint (host and port) on which to host the /debug/pprof endpoint for profiling. May be left blank if not using pprof.
	PPROFAddr string

	// CertFile is the certificate file for use with TLS. May be left blank if using insecure mode.
	CertFile string

	// KeyFile is the private key file for use with TLS. May be left blank if using insecure mode.
	KeyFile string

	// InsecureSkipVerify will cause verification of the host name during a TLS handshake to be skipped if set to true.
	InsecureSkipVerify bool

	// EnableCORS will enable all cross-origin resource sharing if set to true.
	EnableCORS bool

	// MaxSendMsgSize will change the size of the message that can be sent from the service.
	MaxSendMsgSize int

	// MaxRecvMsgSize will change the size of the message that can be received by the service.
	MaxRecvMsgSize int

	// UnaryInterceptors is an array of unary interceptors to be used by the service. They will be executed in order, from first to last.
	UnaryInterceptors []grpc.UnaryServerInterceptor

	// StreamInterceptors is an array of stream interceptors to be used by the service. They will be executed in order, from first to last.
	StreamInterceptors []grpc.StreamServerInterceptor

	// Logger is the logging method to be used for info and error logging by the hoster.
	Logger func(format string, v ...interface{})
}

// NewHoster creates a new hoster instance with defaults set. This is the minimum required to host a server.
func NewHoster(service GRPCService, grpcAddr string) *Hoster {
	return &Hoster{
		Service:        service,
		GRPCAddr:       grpcAddr,
		MaxSendMsgSize: DefaultMaxSendMsgSize,
		MaxRecvMsgSize: DefaultMaxRecvMsgSize,
	}
}

// ListenAndServe creates and starts the server.
func (h *Hoster) ListenAndServe() error {
	// validate parameters
	if h.Service == nil {
		return errors.New("gRPC service implementation must be provided")
	}
	if h.GRPCAddr == "" {
		return errors.New("gRPC address must be provided")
	}

	// check if pprof endpoint is enabled
	if h.PPROFAddr != "" {
		h.log("Starting pprof endpoint: %v", h.PPROFAddr)
		go func() {
			h.log("Error serving pprof endpoint: %v", http.ListenAndServe(h.PPROFAddr, nil))
		}()
	}

	// check if HTTP endpoint is enabled
	if h.HTTPAddr != "" {
		// ensure interface is implemented
		httpService, ok := h.Service.(HTTPService)
		if !ok {
			return errors.New("service does not implement HTTP interface")
		}

		// configure dial options
		dialOpts := []grpc.DialOption{
			grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(math.MaxInt32), grpc.MaxCallRecvMsgSize(math.MaxInt32)),
		}

		// start the HTTP endpoint
		if h.IsTLSEnabled() {
			h.log("Starting HTTP endpoint with TLS enabled: %v", h.HTTPAddr)
			go func() {
				h.log("Error serving HTTP endpoint: %v", ServeHTTPWithTLS(httpService, h.HTTPAddr, h.GRPCAddr, h.EnableCORS, dialOpts, h.CertFile, h.KeyFile, h.InsecureSkipVerify))
			}()
		} else {
			h.log("Starting insecure HTTP endpoint: %v", h.HTTPAddr)
			go func() {
				h.log("Error serving HTTP endpoint: %v", ServeHTTP(httpService, h.HTTPAddr, h.GRPCAddr, h.EnableCORS, dialOpts))
			}()
		}
	}

	// configure server options
	serverOpts := []grpc.ServerOption{
		grpc.MaxSendMsgSize(h.MaxSendMsgSize),
		grpc.MaxRecvMsgSize(h.MaxRecvMsgSize),
	}

	// add interceptors
	if len(h.UnaryInterceptors) > 0 {
		unaryInterceptorChain := grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(h.UnaryInterceptors...))
		serverOpts = append(serverOpts, unaryInterceptorChain)
	}
	if len(h.StreamInterceptors) > 0 {
		streamInterceptorChain := grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(h.StreamInterceptors...))
		serverOpts = append(serverOpts, streamInterceptorChain)
	}

	// start the gRPC endpoint
	if h.IsTLSEnabled() {
		h.log("Starting gRPC endpoint with TLS enabled: %v", h.GRPCAddr)
		return ServeGRPCWithTLS(h.Service, h.GRPCAddr, serverOpts, h.CertFile, h.KeyFile)
	}

	h.log("Starting insecure gRPC endpoint: %v", h.GRPCAddr)
	return ServeGRPC(h.Service, h.GRPCAddr, serverOpts)
}

// IsTLSEnabled will return true if TLS properties are set and ready to use.
func (h *Hoster) IsTLSEnabled() bool {
	return h.CertFile != "" && h.KeyFile != ""
}

// log will safely call the log function provided.
func (h *Hoster) log(format string, v ...interface{}) {
	if h.Logger != nil {
		h.Logger(format, v...)
	}
}
<commit_msg>Moved endpoint code into separate private methods for cleanliness.<commit_after>package gohost

import (
	"errors"
	"math"
	"net/http"

	"github.com/grpc-ecosystem/go-grpc-middleware"
	"google.golang.org/grpc"

	// hosts pprof endpoint
	_ "net/http/pprof"
)

const (
	// DefaultMaxSendMsgSize is the default max send message size, per gRPC
	DefaultMaxSendMsgSize = 1024 * 1024 * 4

	// DefaultMaxRecvMsgSize is the default max receive message size, per gRPC
	DefaultMaxRecvMsgSize = 1024 * 1024 * 4
)

// Hoster is used to serve gRPC and HTTP endpoints.
type Hoster struct {
	// Service contains the actual implementation of the service calls. Additionally implement the HTTPService interface if an HTTP endpoint is desired.
	Service GRPCService

	// GRPCAddr is the endpoint (host and port) on which to host the gRPC service.
	GRPCAddr string

	// HTTPAddr is the endpoint (host and port) on which to host the HTTP service. May be left blank if not using HTTP.
	HTTPAddr string

	// PPROFAddr is the endpoint (host and port) on which to host the /debug/pprof endpoint for profiling. May be left blank if not using pprof.
	PPROFAddr string

	// CertFile is the certificate file for use with TLS. May be left blank if using insecure mode.
	CertFile string

	// KeyFile is the private key file for use with TLS. May be left blank if using insecure mode.
	KeyFile string

	// InsecureSkipVerify will cause verification of the host name during a TLS handshake to be skipped if set to true.
	InsecureSkipVerify bool

	// EnableCORS will enable all cross-origin resource sharing if set to true.
	EnableCORS bool

	// MaxSendMsgSize will change the size of the message that can be sent from the service.
	MaxSendMsgSize int

	// MaxRecvMsgSize will change the size of the message that can be received by the service.
	MaxRecvMsgSize int

	// UnaryInterceptors is an array of unary interceptors to be used by the service. They will be executed in order, from first to last.
	UnaryInterceptors []grpc.UnaryServerInterceptor

	// StreamInterceptors is an array of stream interceptors to be used by the service. They will be executed in order, from first to last.
	StreamInterceptors []grpc.StreamServerInterceptor

	// Logger is the logging method to be used for info and error logging by the hoster.
	Logger func(format string, v ...interface{})
}

// NewHoster creates a new hoster instance with defaults set. This is the minimum required to host a server.
func NewHoster(service GRPCService, grpcAddr string) *Hoster {
	return &Hoster{
		Service:        service,
		GRPCAddr:       grpcAddr,
		MaxSendMsgSize: DefaultMaxSendMsgSize,
		MaxRecvMsgSize: DefaultMaxRecvMsgSize,
	}
}

// ListenAndServe creates and starts the server.
func (h *Hoster) ListenAndServe() error {
	// validate parameters
	if h.Service == nil {
		return errors.New("gRPC service implementation must be provided")
	}
	if h.GRPCAddr == "" {
		return errors.New("gRPC address must be provided")
	}

	// serve pprof endpoint
	h.servePPROF()

	// serve HTTP endpoint
	err := h.serveHTTP()
	if err != nil {
		return err
	}

	// serve gRPC endpoint
	return h.serveGRPC()
}

// IsTLSEnabled will return true if TLS properties are set and ready to use.
func (h *Hoster) IsTLSEnabled() bool {
	return h.CertFile != "" && h.KeyFile != ""
}

// serveGRPC will start the gRPC endpoint.
func (h *Hoster) serveGRPC() error {
	// configure server options
	serverOpts := []grpc.ServerOption{
		grpc.MaxSendMsgSize(h.MaxSendMsgSize),
		grpc.MaxRecvMsgSize(h.MaxRecvMsgSize),
	}

	// add interceptors
	if len(h.UnaryInterceptors) > 0 {
		unaryInterceptorChain := grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(h.UnaryInterceptors...))
		serverOpts = append(serverOpts, unaryInterceptorChain)
	}
	if len(h.StreamInterceptors) > 0 {
		streamInterceptorChain := grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(h.StreamInterceptors...))
		serverOpts = append(serverOpts, streamInterceptorChain)
	}

	// start the gRPC endpoint
	if h.IsTLSEnabled() {
		h.log("Starting gRPC endpoint with TLS enabled: %v", h.GRPCAddr)
		return ServeGRPCWithTLS(h.Service, h.GRPCAddr, serverOpts, h.CertFile, h.KeyFile)
	}

	h.log("Starting insecure gRPC endpoint: %v", h.GRPCAddr)
	return ServeGRPC(h.Service, h.GRPCAddr, serverOpts)
}

// serveHTTP will start the HTTP endpoint.
func (h *Hoster) serveHTTP() error {
	// check if HTTP endpoint is enabled
	if h.HTTPAddr != "" {
		// ensure interface is implemented
		httpService, ok := h.Service.(HTTPService)
		if !ok {
			return errors.New("service does not implement HTTP interface")
		}

		// configure dial options
		dialOpts := []grpc.DialOption{
			grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(math.MaxInt32), grpc.MaxCallRecvMsgSize(math.MaxInt32)),
		}

		// start the HTTP endpoint
		if h.IsTLSEnabled() {
			h.log("Starting HTTP endpoint with TLS enabled: %v", h.HTTPAddr)
			go func() {
				h.log("Error serving HTTP endpoint: %v", ServeHTTPWithTLS(httpService, h.HTTPAddr, h.GRPCAddr, h.EnableCORS, dialOpts, h.CertFile, h.KeyFile, h.InsecureSkipVerify))
			}()
		} else {
			h.log("Starting insecure HTTP endpoint: %v", h.HTTPAddr)
			go func() {
				h.log("Error serving HTTP endpoint: %v", ServeHTTP(httpService, h.HTTPAddr, h.GRPCAddr, h.EnableCORS, dialOpts))
			}()
		}
	}

	return nil
}

// servePPROF will start the pprof endpoint.
func (h *Hoster) servePPROF() {
	// check if pprof endpoint is enabled
	if h.PPROFAddr != "" {
		h.log("Starting pprof endpoint: %v", h.PPROFAddr)
		go func() {
			h.log("Error serving pprof endpoint: %v", http.ListenAndServe(h.PPROFAddr, nil))
		}()
	}
}

// log will safely call the log function provided.
func (h *Hoster) log(format string, v ...interface{}) {
	if h.Logger != nil {
		h.Logger(format, v...)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017-present The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package releaser implements a set of utilities and a wrapper around Goreleaser
// to help automate the Hugo release process.
package releaser

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"text/template"
	"time"
)

const (
	issueLinkTemplate                        = "[#%d](https://github.com/gohugoio/hugo/issues/%d)"
	linkTemplate                             = "[%s](%s)"
	releaseNotesMarkdownTemplatePatchRelease = `
{{ if eq (len .All) 1 }}
This is a bug-fix release with one important fix.
{{ else }}
This is a bug-fix release with a couple of important fixes.
{{ end }}
{{ range .All }}
{{- if .GitHubCommit -}}
* {{ .Subject }} {{ . | commitURL }} {{ . | authorURL }} {{ range .Issues }}{{ . | issue }}{{ end }}
{{ else -}}
* {{ .Subject }} {{ range .Issues }}{{ . | issue }}{{ end }}
{{ end -}}
{{- end }}

`
	releaseNotesMarkdownTemplate = `
{{- $contribsPerAuthor := .All.ContribCountPerAuthor -}}
{{- $docsContribsPerAuthor := .Docs.ContribCountPerAuthor -}}

This release represents **{{ len .All }} contributions by {{ len $contribsPerAuthor }} contributors** to the main Hugo code base.

{{- if  gt (len $contribsPerAuthor) 3 -}}
{{- $u1 := index $contribsPerAuthor 0 -}}
{{- $u2 := index $contribsPerAuthor 1 -}}
{{- $u3 := index $contribsPerAuthor 2 -}}
{{- $u4 := index $contribsPerAuthor 3 -}}
{{- $u1.AuthorLink }} leads the Hugo development with a significant amount of contributions, but also a big shoutout to {{ $u2.AuthorLink }}, {{ $u3.AuthorLink }}, and {{ $u4.AuthorLink }} for their ongoing contributions.
And a big thanks to [@digitalcraftsman](https://github.com/digitalcraftsman) and [@onedrawingperday](https://github.com/onedrawingperday) for their relentless work on keeping the themes site in pristine condition and to [@davidsneighbour](https://github.com/davidsneighbour), [@coliff](https://github.com/coliff) and [@kaushalmodi](https://github.com/kaushalmodi) for all the great work on the documentation site.
{{ end }}
Many have also been busy writing and fixing the documentation in [hugoDocs](https://github.com/gohugoio/hugoDocs),
which has received **{{ len .Docs }} contributions by {{ len $docsContribsPerAuthor }} contributors**.
{{- if  gt (len $docsContribsPerAuthor) 3 -}}
{{- $u1 := index $docsContribsPerAuthor 0 -}}
{{- $u2 := index $docsContribsPerAuthor 1 -}}
{{- $u3 := index $docsContribsPerAuthor 2 -}}
{{- $u4 := index $docsContribsPerAuthor 3 }} A special thanks to {{ $u1.AuthorLink }}, {{ $u2.AuthorLink }}, {{ $u3.AuthorLink }}, and {{ $u4.AuthorLink }} for their work on the documentation site.
{{ end }}

Hugo now has:

{{ with .Repo -}}
* {{ .Stars }}+ [stars](https://github.com/gohugoio/hugo/stargazers)
* {{ len .Contributors }}+ [contributors](https://github.com/gohugoio/hugo/graphs/contributors)
{{- end -}}
{{ with .ThemeCount }}
* {{ . }}+ [themes](http://themes.gohugo.io/)
{{ end }}
{{ with .Notes }}
## Notes
{{ template "change-section" . }}
{{- end -}}
## Enhancements
{{ template "change-headers"  .Enhancements -}}
## Fixes
{{ template "change-headers"  .Fixes -}}

{{ define "change-headers" }}
{{ $tmplChanges := index . "templateChanges" -}}
{{- $outChanges := index . "outChanges" -}}
{{- $coreChanges := index . "coreChanges" -}}
{{- $otherChanges := index . "otherChanges" -}}
{{- with $tmplChanges -}}
### Templates
{{ template "change-section" . }}
{{- end -}}
{{- with $outChanges -}}
### Output
{{ template "change-section"  . }}
{{- end -}}
{{- with $coreChanges -}}
### Core
{{ template "change-section" . }}
{{- end -}}
{{- with $otherChanges -}}
### Other
{{ template "change-section"  . }}
{{- end -}}
{{ end }}

{{ define "change-section" }}
{{ range . }}
{{- if .GitHubCommit -}}
* {{ .Subject }} {{ . | commitURL }} {{ . | authorURL }} {{ range .Issues }}{{ . | issue }}{{ end }}
{{ else -}}
* {{ .Subject }} {{ range .Issues }}{{ . | issue }}{{ end }}
{{ end -}}
{{- end }}
{{ end }}
`
)

var templateFuncs = template.FuncMap{
	"isPatch": func(c changeLog) bool {
		return !strings.HasSuffix(c.Version, "0")
	},
	"issue": func(id int) string {
		return fmt.Sprintf(issueLinkTemplate, id, id)
	},
	"commitURL": func(info gitInfo) string {
		if info.GitHubCommit.HTMLURL == "" {
			return ""
		}
		return fmt.Sprintf(linkTemplate, info.Hash, info.GitHubCommit.HTMLURL)
	},
	"authorURL": func(info gitInfo) string {
		if info.GitHubCommit.Author.Login == "" {
			return ""
		}
		return fmt.Sprintf(linkTemplate, "@"+info.GitHubCommit.Author.Login, info.GitHubCommit.Author.HTMLURL)
	},
}

func writeReleaseNotes(version string, infosMain, infosDocs gitInfos, to io.Writer) error {
	client := newGitHubAPI("hugo")
	changes := gitInfosToChangeLog(infosMain, infosDocs)
	changes.Version = version
	repo, err := client.fetchRepo()
	if err == nil {
		changes.Repo = &repo
	}
	themeCount, err := fetchThemeCount()
	if err == nil {
		changes.ThemeCount = themeCount
	}

	mtempl := releaseNotesMarkdownTemplate

	if !strings.HasSuffix(version, "0") {
		mtempl = releaseNotesMarkdownTemplatePatchRelease
	}

	tmpl, err := template.New("").Funcs(templateFuncs).Parse(mtempl)
	if err != nil {
		return err
	}

	err = tmpl.Execute(to, changes)
	if err != nil {
		return err
	}

	return nil

}

func fetchThemeCount() (int, error) {
	resp, err := http.Get("https://raw.githubusercontent.com/gohugoio/hugoThemes/master/.gitmodules")
	if err != nil {
		return 0, err
	}
	defer resp.Body.Close()

	b, _ := ioutil.ReadAll(resp.Body)
	return bytes.Count(b, []byte("submodule")), nil
}

func writeReleaseNotesToTmpFile(version string, infosMain, infosDocs gitInfos) (string, error) {
	f, err := ioutil.TempFile("", "hugorelease")
	if err != nil {
		return "", err
	}

	defer f.Close()

	if err := writeReleaseNotes(version, infosMain, infosDocs, f); err != nil {
		return "", err
	}

	return f.Name(), nil
}

func getReleaseNotesDocsTempDirAndName(version string, final bool) (string, string) {
	if final {
		return hugoFilepath("temp"), fmt.Sprintf("%s-relnotes-ready.md", version)
	}
	return hugoFilepath("temp"), fmt.Sprintf("%s-relnotes.md", version)
}

func getReleaseNotesDocsTempFilename(version string, final bool) string {
	return filepath.Join(getReleaseNotesDocsTempDirAndName(version, final))
}

func (r *ReleaseHandler) releaseNotesState(version string) (releaseNotesState, error) {
	docsTempPath, name := getReleaseNotesDocsTempDirAndName(version, false)
	_, err := os.Stat(filepath.Join(docsTempPath, name))

	if err == nil {
		return releaseNotesCreated, nil
	}

	docsTempPath, name = getReleaseNotesDocsTempDirAndName(version, true)
	_, err = os.Stat(filepath.Join(docsTempPath, name))

	if err == nil {
		return releaseNotesReady, nil
	}

	if !os.IsNotExist(err) {
		return releaseNotesNone, err
	}

	return releaseNotesNone, nil

}

func (r *ReleaseHandler) writeReleaseNotesToTemp(version string, isPatch bool, infosMain, infosDocs gitInfos) (string, error) {

	docsTempPath, name := getReleaseNotesDocsTempDirAndName(version, isPatch)

	var (
		w io.WriteCloser
	)

	if !r.try {
		os.Mkdir(docsTempPath, os.ModePerm)

		f, err := os.Create(filepath.Join(docsTempPath, name))
		if err != nil {
			return "", err
		}

		name = f.Name()

		defer f.Close()

		w = f

	} else {
		w = os.Stdout
	}

	if err := writeReleaseNotes(version, infosMain, infosDocs, w); err != nil {
		return "", err
	}

	return name, nil

}

func (r *ReleaseHandler) writeReleaseNotesToDocs(title, description, sourceFilename string) (string, error) {
	targetFilename := "index.md"
	bundleDir := strings.TrimSuffix(filepath.Base(sourceFilename), "-ready.md")
	contentDir := hugoFilepath("docs/content/en/news/" + bundleDir)
	targetFullFilename := filepath.Join(contentDir, targetFilename)

	if r.try {
		fmt.Printf("Write release notes to /docs: Bundle %q Dir: %q\n", bundleDir, contentDir)
		return targetFullFilename, nil
	}

	if err := os.MkdirAll(contentDir, os.ModePerm); err != nil {
		return "", nil
	}

	b, err := ioutil.ReadFile(sourceFilename)
	if err != nil {
		return "", err
	}

	f, err := os.Create(targetFullFilename)
	if err != nil {
		return "", err
	}
	defer f.Close()

	fmTail := ""
	if !strings.HasSuffix(title, ".0") {
		// Bug fix release
		fmTail = `
images:
- images/blog/hugo-bug-poster.png
`
	}

	if _, err := f.WriteString(fmt.Sprintf(`
---
date: %s
title: %q
description: %q
categories: ["Releases"]%s
---

	`, time.Now().Format("2006-01-02"), title, description, fmTail)); err != nil {
		return "", err
	}

	if _, err := f.Write(b); err != nil {
		return "", err
	}

	return targetFullFilename, nil

}
<commit_msg>Remove credit (#7347)<commit_after>// Copyright 2017-present The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package releaser implements a set of utilities and a wrapper around Goreleaser
// to help automate the Hugo release process.
package releaser

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"text/template"
	"time"
)

const (
	issueLinkTemplate                        = "[#%d](https://github.com/gohugoio/hugo/issues/%d)"
	linkTemplate                             = "[%s](%s)"
	releaseNotesMarkdownTemplatePatchRelease = `
{{ if eq (len .All) 1 }}
This is a bug-fix release with one important fix.
{{ else }}
This is a bug-fix release with a couple of important fixes.
{{ end }}
{{ range .All }}
{{- if .GitHubCommit -}}
* {{ .Subject }} {{ . | commitURL }} {{ . | authorURL }} {{ range .Issues }}{{ . | issue }}{{ end }}
{{ else -}}
* {{ .Subject }} {{ range .Issues }}{{ . | issue }}{{ end }}
{{ end -}}
{{- end }}

`
	releaseNotesMarkdownTemplate = `
{{- $contribsPerAuthor := .All.ContribCountPerAuthor -}}
{{- $docsContribsPerAuthor := .Docs.ContribCountPerAuthor -}}

This release represents **{{ len .All }} contributions by {{ len $contribsPerAuthor }} contributors** to the main Hugo code base.

{{- if  gt (len $contribsPerAuthor) 3 -}}
{{- $u1 := index $contribsPerAuthor 0 -}}
{{- $u2 := index $contribsPerAuthor 1 -}}
{{- $u3 := index $contribsPerAuthor 2 -}}
{{- $u4 := index $contribsPerAuthor 3 -}}
{{- $u1.AuthorLink }} leads the Hugo development with a significant amount of contributions, but also a big shoutout to {{ $u2.AuthorLink }}, {{ $u3.AuthorLink }}, and {{ $u4.AuthorLink }} for their ongoing contributions.
And a big thanks to [@digitalcraftsman](https://github.com/digitalcraftsman) for his relentless work on keeping the themes site in pristine condition and to [@davidsneighbour](https://github.com/davidsneighbour), [@coliff](https://github.com/coliff) and [@kaushalmodi](https://github.com/kaushalmodi) for all the great work on the documentation site.
{{ end }}
Many have also been busy writing and fixing the documentation in [hugoDocs](https://github.com/gohugoio/hugoDocs),
which has received **{{ len .Docs }} contributions by {{ len $docsContribsPerAuthor }} contributors**.
{{- if  gt (len $docsContribsPerAuthor) 3 -}}
{{- $u1 := index $docsContribsPerAuthor 0 -}}
{{- $u2 := index $docsContribsPerAuthor 1 -}}
{{- $u3 := index $docsContribsPerAuthor 2 -}}
{{- $u4 := index $docsContribsPerAuthor 3 }} A special thanks to {{ $u1.AuthorLink }}, {{ $u2.AuthorLink }}, {{ $u3.AuthorLink }}, and {{ $u4.AuthorLink }} for their work on the documentation site.
{{ end }}

Hugo now has:

{{ with .Repo -}}
* {{ .Stars }}+ [stars](https://github.com/gohugoio/hugo/stargazers)
* {{ len .Contributors }}+ [contributors](https://github.com/gohugoio/hugo/graphs/contributors)
{{- end -}}
{{ with .ThemeCount }}
* {{ . }}+ [themes](http://themes.gohugo.io/)
{{ end }}
{{ with .Notes }}
## Notes
{{ template "change-section" . }}
{{- end -}}
## Enhancements
{{ template "change-headers"  .Enhancements -}}
## Fixes
{{ template "change-headers"  .Fixes -}}

{{ define "change-headers" }}
{{ $tmplChanges := index . "templateChanges" -}}
{{- $outChanges := index . "outChanges" -}}
{{- $coreChanges := index . "coreChanges" -}}
{{- $otherChanges := index . "otherChanges" -}}
{{- with $tmplChanges -}}
### Templates
{{ template "change-section" . }}
{{- end -}}
{{- with $outChanges -}}
### Output
{{ template "change-section"  . }}
{{- end -}}
{{- with $coreChanges -}}
### Core
{{ template "change-section" . }}
{{- end -}}
{{- with $otherChanges -}}
### Other
{{ template "change-section"  . }}
{{- end -}}
{{ end }}

{{ define "change-section" }}
{{ range . }}
{{- if .GitHubCommit -}}
* {{ .Subject }} {{ . | commitURL }} {{ . | authorURL }} {{ range .Issues }}{{ . | issue }}{{ end }}
{{ else -}}
* {{ .Subject }} {{ range .Issues }}{{ . | issue }}{{ end }}
{{ end -}}
{{- end }}
{{ end }}
`
)

var templateFuncs = template.FuncMap{
	"isPatch": func(c changeLog) bool {
		return !strings.HasSuffix(c.Version, "0")
	},
	"issue": func(id int) string {
		return fmt.Sprintf(issueLinkTemplate, id, id)
	},
	"commitURL": func(info gitInfo) string {
		if info.GitHubCommit.HTMLURL == "" {
			return ""
		}
		return fmt.Sprintf(linkTemplate, info.Hash, info.GitHubCommit.HTMLURL)
	},
	"authorURL": func(info gitInfo) string {
		if info.GitHubCommit.Author.Login == "" {
			return ""
		}
		return fmt.Sprintf(linkTemplate, "@"+info.GitHubCommit.Author.Login, info.GitHubCommit.Author.HTMLURL)
	},
}

func writeReleaseNotes(version string, infosMain, infosDocs gitInfos, to io.Writer) error {
	client := newGitHubAPI("hugo")
	changes := gitInfosToChangeLog(infosMain, infosDocs)
	changes.Version = version
	repo, err := client.fetchRepo()
	if err == nil {
		changes.Repo = &repo
	}
	themeCount, err := fetchThemeCount()
	if err == nil {
		changes.ThemeCount = themeCount
	}

	mtempl := releaseNotesMarkdownTemplate

	if !strings.HasSuffix(version, "0") {
		mtempl = releaseNotesMarkdownTemplatePatchRelease
	}

	tmpl, err := template.New("").Funcs(templateFuncs).Parse(mtempl)
	if err != nil {
		return err
	}

	err = tmpl.Execute(to, changes)
	if err != nil {
		return err
	}

	return nil

}

func fetchThemeCount() (int, error) {
	resp, err := http.Get("https://raw.githubusercontent.com/gohugoio/hugoThemes/master/.gitmodules")
	if err != nil {
		return 0, err
	}
	defer resp.Body.Close()

	b, _ := ioutil.ReadAll(resp.Body)
	return bytes.Count(b, []byte("submodule")), nil
}

func writeReleaseNotesToTmpFile(version string, infosMain, infosDocs gitInfos) (string, error) {
	f, err := ioutil.TempFile("", "hugorelease")
	if err != nil {
		return "", err
	}

	defer f.Close()

	if err := writeReleaseNotes(version, infosMain, infosDocs, f); err != nil {
		return "", err
	}

	return f.Name(), nil
}

func getReleaseNotesDocsTempDirAndName(version string, final bool) (string, string) {
	if final {
		return hugoFilepath("temp"), fmt.Sprintf("%s-relnotes-ready.md", version)
	}
	return hugoFilepath("temp"), fmt.Sprintf("%s-relnotes.md", version)
}

func getReleaseNotesDocsTempFilename(version string, final bool) string {
	return filepath.Join(getReleaseNotesDocsTempDirAndName(version, final))
}

func (r *ReleaseHandler) releaseNotesState(version string) (releaseNotesState, error) {
	docsTempPath, name := getReleaseNotesDocsTempDirAndName(version, false)
	_, err := os.Stat(filepath.Join(docsTempPath, name))

	if err == nil {
		return releaseNotesCreated, nil
	}

	docsTempPath, name = getReleaseNotesDocsTempDirAndName(version, true)
	_, err = os.Stat(filepath.Join(docsTempPath, name))

	if err == nil {
		return releaseNotesReady, nil
	}

	if !os.IsNotExist(err) {
		return releaseNotesNone, err
	}

	return releaseNotesNone, nil

}

func (r *ReleaseHandler) writeReleaseNotesToTemp(version string, isPatch bool, infosMain, infosDocs gitInfos) (string, error) {

	docsTempPath, name := getReleaseNotesDocsTempDirAndName(version, isPatch)

	var (
		w io.WriteCloser
	)

	if !r.try {
		os.Mkdir(docsTempPath, os.ModePerm)

		f, err := os.Create(filepath.Join(docsTempPath, name))
		if err != nil {
			return "", err
		}

		name = f.Name()

		defer f.Close()

		w = f

	} else {
		w = os.Stdout
	}

	if err := writeReleaseNotes(version, infosMain, infosDocs, w); err != nil {
		return "", err
	}

	return name, nil

}

func (r *ReleaseHandler) writeReleaseNotesToDocs(title, description, sourceFilename string) (string, error) {
	targetFilename := "index.md"
	bundleDir := strings.TrimSuffix(filepath.Base(sourceFilename), "-ready.md")
	contentDir := hugoFilepath("docs/content/en/news/" + bundleDir)
	targetFullFilename := filepath.Join(contentDir, targetFilename)

	if r.try {
		fmt.Printf("Write release notes to /docs: Bundle %q Dir: %q\n", bundleDir, contentDir)
		return targetFullFilename, nil
	}

	if err := os.MkdirAll(contentDir, os.ModePerm); err != nil {
		return "", nil
	}

	b, err := ioutil.ReadFile(sourceFilename)
	if err != nil {
		return "", err
	}

	f, err := os.Create(targetFullFilename)
	if err != nil {
		return "", err
	}
	defer f.Close()

	fmTail := ""
	if !strings.HasSuffix(title, ".0") {
		// Bug fix release
		fmTail = `
images:
- images/blog/hugo-bug-poster.png
`
	}

	if _, err := f.WriteString(fmt.Sprintf(`
---
date: %s
title: %q
description: %q
categories: ["Releases"]%s
---

	`, time.Now().Format("2006-01-02"), title, description, fmTail)); err != nil {
		return "", err
	}

	if _, err := f.Write(b); err != nil {
		return "", err
	}

	return targetFullFilename, nil

}
<|endoftext|>


<commit_before>package storagepacker

import (
	"context"
	"crypto/md5"
	"fmt"
	"strconv"
	"strings"

	"github.com/golang/protobuf/proto"
	"github.com/hashicorp/errwrap"
	log "github.com/hashicorp/go-hclog"
	"github.com/hashicorp/vault/sdk/helper/compressutil"
	"github.com/hashicorp/vault/sdk/helper/locksutil"
	"github.com/hashicorp/vault/sdk/logical"
)

const (
	bucketCount                = 256
	StoragePackerBucketsPrefix = "packer/buckets/"
)

// StoragePacker packs items into a specific number of buckets by hashing
// its identifier and indexing on it. Currently this supports only 256 bucket entries and
// hence relies on the first byte of the hash value for indexing.
type StoragePacker struct {
	view         logical.Storage
	logger       log.Logger
	storageLocks []*locksutil.LockEntry
	viewPrefix   string
}

// View returns the storage view configured to be used by the packer
func (s *StoragePacker) View() logical.Storage {
	return s.view
}

// Get returns a bucket for a given key
func (s *StoragePacker) GetBucket(key string) (*Bucket, error) {
	if key == "" {
		return nil, fmt.Errorf("missing bucket key")
	}

	lock := locksutil.LockForKey(s.storageLocks, key)
	lock.RLock()
	defer lock.RUnlock()

	// Read from storage
	storageEntry, err := s.view.Get(context.Background(), key)
	if err != nil {
		return nil, errwrap.Wrapf("failed to read packed storage entry: {{err}}", err)
	}
	if storageEntry == nil {
		return nil, nil
	}

	uncompressedData, notCompressed, err := compressutil.Decompress(storageEntry.Value)
	if err != nil {
		return nil, errwrap.Wrapf("failed to decompress packed storage entry: {{err}}", err)
	}
	if notCompressed {
		uncompressedData = storageEntry.Value
	}

	var bucket Bucket
	err = proto.Unmarshal(uncompressedData, &bucket)
	if err != nil {
		return nil, errwrap.Wrapf("failed to decode packed storage entry: {{err}}", err)
	}

	return &bucket, nil
}

// upsert either inserts a new item into the bucket or updates an existing one
// if an item with a matching key is already present.
func (s *Bucket) upsert(item *Item) error {
	if s == nil {
		return fmt.Errorf("nil storage bucket")
	}

	if item == nil {
		return fmt.Errorf("nil item")
	}

	if item.ID == "" {
		return fmt.Errorf("missing item ID")
	}

	// Look for an item with matching key and don't modify the collection while
	// iterating
	foundIdx := -1
	for itemIdx, bucketItems := range s.Items {
		if bucketItems.ID == item.ID {
			foundIdx = itemIdx
			break
		}
	}

	// If there is no match, append the item, otherwise update it
	if foundIdx == -1 {
		s.Items = append(s.Items, item)
	} else {
		s.Items[foundIdx] = item
	}

	return nil
}

// BucketKey returns the storage key of the bucket where the given item will be
// stored.
func (s *StoragePacker) BucketKey(itemID string) string {
	hf := md5.New()
	hf.Write([]byte(itemID))
	index := uint8(hf.Sum(nil)[0])
	return s.viewPrefix + strconv.Itoa(int(index))
}

// DeleteItem removes the item from the respective bucket
func (s *StoragePacker) DeleteItem(_ context.Context, itemID string) error {
	if itemID == "" {
		return fmt.Errorf("empty item ID")
	}

	bucketKey := s.BucketKey(itemID)

	// Read from storage
	storageEntry, err := s.view.Get(context.Background(), bucketKey)
	if err != nil {
		return errwrap.Wrapf("failed to read packed storage value: {{err}}", err)
	}
	if storageEntry == nil {
		return nil
	}

	uncompressedData, notCompressed, err := compressutil.Decompress(storageEntry.Value)
	if err != nil {
		return errwrap.Wrapf("failed to decompress packed storage value: {{err}}", err)
	}
	if notCompressed {
		uncompressedData = storageEntry.Value
	}

	var bucket Bucket
	err = proto.Unmarshal(uncompressedData, &bucket)
	if err != nil {
		return errwrap.Wrapf("failed decoding packed storage entry: {{err}}", err)
	}

	// Look for a matching storage entry
	foundIdx := -1
	for itemIdx, item := range bucket.Items {
		if item.ID == itemID {
			foundIdx = itemIdx
			break
		}
	}

	// If there is a match, remove it from the collection and persist the
	// resulting collection
	if foundIdx != -1 {
		bucket.Items = append(bucket.Items[:foundIdx], bucket.Items[foundIdx+1:]...)

		// Persist bucket entry only if there is an update
		err = s.putBucket(&bucket)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *StoragePacker) putBucket(bucket *Bucket) error {
	if bucket == nil {
		return fmt.Errorf("nil bucket entry")
	}

	if bucket.Key == "" {
		return fmt.Errorf("missing key")
	}

	if !strings.HasPrefix(bucket.Key, s.viewPrefix) {
		return fmt.Errorf("incorrect prefix; bucket entry key should have %q prefix", s.viewPrefix)
	}

	marshaledBucket, err := proto.Marshal(bucket)
	if err != nil {
		return errwrap.Wrapf("failed to marshal bucket: {{err}}", err)
	}

	compressedBucket, err := compressutil.Compress(marshaledBucket, &compressutil.CompressionConfig{
		Type: compressutil.CompressionTypeSnappy,
	})
	if err != nil {
		return errwrap.Wrapf("failed to compress packed bucket: {{err}}", err)
	}

	// Store the compressed value
	err = s.view.Put(context.Background(), &logical.StorageEntry{
		Key:   bucket.Key,
		Value: compressedBucket,
	})
	if err != nil {
		return errwrap.Wrapf("failed to persist packed storage entry: {{err}}", err)
	}

	return nil
}

// GetItem fetches the storage entry for a given key from its corresponding
// bucket.
func (s *StoragePacker) GetItem(itemID string) (*Item, error) {
	if itemID == "" {
		return nil, fmt.Errorf("empty item ID")
	}

	bucketKey := s.BucketKey(itemID)

	// Fetch the bucket entry
	bucket, err := s.GetBucket(bucketKey)
	if err != nil {
		return nil, errwrap.Wrapf("failed to read packed storage item: {{err}}", err)
	}
	if bucket == nil {
		return nil, nil
	}

	// Look for a matching storage entry in the bucket items
	for _, item := range bucket.Items {
		if item.ID == itemID {
			return item, nil
		}
	}

	return nil, nil
}

// PutItem stores the given item in its respective bucket
func (s *StoragePacker) PutItem(_ context.Context, item *Item) error {
	if item == nil {
		return fmt.Errorf("nil item")
	}

	if item.ID == "" {
		return fmt.Errorf("missing ID in item")
	}

	var err error
	bucketKey := s.BucketKey(item.ID)

	bucket := &Bucket{
		Key: bucketKey,
	}

	// In this case, we persist the storage entry regardless of the read
	// storageEntry below is nil or not. Hence, directly acquire write lock
	// even to read the entry.
	lock := locksutil.LockForKey(s.storageLocks, bucketKey)
	lock.Lock()
	defer lock.Unlock()

	// Check if there is an existing bucket for a given key
	storageEntry, err := s.view.Get(context.Background(), bucketKey)
	if err != nil {
		return errwrap.Wrapf("failed to read packed storage bucket entry: {{err}}", err)
	}

	if storageEntry == nil {
		// If the bucket entry does not exist, this will be the only item the
		// bucket that is going to be persisted.
		bucket.Items = []*Item{
			item,
		}
	} else {
		uncompressedData, notCompressed, err := compressutil.Decompress(storageEntry.Value)
		if err != nil {
			return errwrap.Wrapf("failed to decompress packed storage entry: {{err}}", err)
		}
		if notCompressed {
			uncompressedData = storageEntry.Value
		}

		err = proto.Unmarshal(uncompressedData, bucket)
		if err != nil {
			return errwrap.Wrapf("failed to decode packed storage entry: {{err}}", err)
		}

		err = bucket.upsert(item)
		if err != nil {
			return errwrap.Wrapf("failed to update entry in packed storage entry: {{err}}", err)
		}
	}

	return s.putBucket(bucket)
}

// NewStoragePacker creates a new storage packer for a given view
func NewStoragePacker(view logical.Storage, logger log.Logger, viewPrefix string) (*StoragePacker, error) {
	if view == nil {
		return nil, fmt.Errorf("nil view")
	}

	if viewPrefix == "" {
		viewPrefix = StoragePackerBucketsPrefix
	}

	if !strings.HasSuffix(viewPrefix, "/") {
		viewPrefix = viewPrefix + "/"
	}

	// Create a new packer object for the given view
	packer := &StoragePacker{
		view:         view,
		viewPrefix:   viewPrefix,
		logger:       logger,
		storageLocks: locksutil.CreateLocks(),
	}

	return packer, nil
}
<commit_msg>Make linter happy (#6693)<commit_after>package storagepacker

import (
	"context"
	"crypto/md5"
	"fmt"
	"strconv"
	"strings"

	"github.com/golang/protobuf/proto"
	"github.com/hashicorp/errwrap"
	log "github.com/hashicorp/go-hclog"
	"github.com/hashicorp/vault/sdk/helper/compressutil"
	"github.com/hashicorp/vault/sdk/helper/locksutil"
	"github.com/hashicorp/vault/sdk/logical"
)

const (
	bucketCount = 256
	// StoragePackerBucketsPrefix is the default storage key prefix under which
	// bucket data will be stored.
	StoragePackerBucketsPrefix = "packer/buckets/"
)

// StoragePacker packs items into a specific number of buckets by hashing
// its identifier and indexing on it. Currently this supports only 256 bucket entries and
// hence relies on the first byte of the hash value for indexing.
type StoragePacker struct {
	view         logical.Storage
	logger       log.Logger
	storageLocks []*locksutil.LockEntry
	viewPrefix   string
}

// View returns the storage view configured to be used by the packer
func (s *StoragePacker) View() logical.Storage {
	return s.view
}

// GetBucket returns a bucket for a given key
func (s *StoragePacker) GetBucket(key string) (*Bucket, error) {
	if key == "" {
		return nil, fmt.Errorf("missing bucket key")
	}

	lock := locksutil.LockForKey(s.storageLocks, key)
	lock.RLock()
	defer lock.RUnlock()

	// Read from storage
	storageEntry, err := s.view.Get(context.Background(), key)
	if err != nil {
		return nil, errwrap.Wrapf("failed to read packed storage entry: {{err}}", err)
	}
	if storageEntry == nil {
		return nil, nil
	}

	uncompressedData, notCompressed, err := compressutil.Decompress(storageEntry.Value)
	if err != nil {
		return nil, errwrap.Wrapf("failed to decompress packed storage entry: {{err}}", err)
	}
	if notCompressed {
		uncompressedData = storageEntry.Value
	}

	var bucket Bucket
	err = proto.Unmarshal(uncompressedData, &bucket)
	if err != nil {
		return nil, errwrap.Wrapf("failed to decode packed storage entry: {{err}}", err)
	}

	return &bucket, nil
}

// upsert either inserts a new item into the bucket or updates an existing one
// if an item with a matching key is already present.
func (s *Bucket) upsert(item *Item) error {
	if s == nil {
		return fmt.Errorf("nil storage bucket")
	}

	if item == nil {
		return fmt.Errorf("nil item")
	}

	if item.ID == "" {
		return fmt.Errorf("missing item ID")
	}

	// Look for an item with matching key and don't modify the collection while
	// iterating
	foundIdx := -1
	for itemIdx, bucketItems := range s.Items {
		if bucketItems.ID == item.ID {
			foundIdx = itemIdx
			break
		}
	}

	// If there is no match, append the item, otherwise update it
	if foundIdx == -1 {
		s.Items = append(s.Items, item)
	} else {
		s.Items[foundIdx] = item
	}

	return nil
}

// BucketKey returns the storage key of the bucket where the given item will be
// stored.
func (s *StoragePacker) BucketKey(itemID string) string {
	hf := md5.New()
	input := []byte(itemID)
	n, err := hf.Write(input)
	// Make linter happy
	if err != nil || n != len(input) {
		return ""
	}
	index := uint8(hf.Sum(nil)[0])
	return s.viewPrefix + strconv.Itoa(int(index))
}

// DeleteItem removes the item from the respective bucket
func (s *StoragePacker) DeleteItem(_ context.Context, itemID string) error {
	if itemID == "" {
		return fmt.Errorf("empty item ID")
	}

	bucketKey := s.BucketKey(itemID)

	// Read from storage
	storageEntry, err := s.view.Get(context.Background(), bucketKey)
	if err != nil {
		return errwrap.Wrapf("failed to read packed storage value: {{err}}", err)
	}
	if storageEntry == nil {
		return nil
	}

	uncompressedData, notCompressed, err := compressutil.Decompress(storageEntry.Value)
	if err != nil {
		return errwrap.Wrapf("failed to decompress packed storage value: {{err}}", err)
	}
	if notCompressed {
		uncompressedData = storageEntry.Value
	}

	var bucket Bucket
	err = proto.Unmarshal(uncompressedData, &bucket)
	if err != nil {
		return errwrap.Wrapf("failed decoding packed storage entry: {{err}}", err)
	}

	// Look for a matching storage entry
	foundIdx := -1
	for itemIdx, item := range bucket.Items {
		if item.ID == itemID {
			foundIdx = itemIdx
			break
		}
	}

	// If there is a match, remove it from the collection and persist the
	// resulting collection
	if foundIdx != -1 {
		bucket.Items = append(bucket.Items[:foundIdx], bucket.Items[foundIdx+1:]...)

		// Persist bucket entry only if there is an update
		err = s.putBucket(&bucket)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *StoragePacker) putBucket(bucket *Bucket) error {
	if bucket == nil {
		return fmt.Errorf("nil bucket entry")
	}

	if bucket.Key == "" {
		return fmt.Errorf("missing key")
	}

	if !strings.HasPrefix(bucket.Key, s.viewPrefix) {
		return fmt.Errorf("incorrect prefix; bucket entry key should have %q prefix", s.viewPrefix)
	}

	marshaledBucket, err := proto.Marshal(bucket)
	if err != nil {
		return errwrap.Wrapf("failed to marshal bucket: {{err}}", err)
	}

	compressedBucket, err := compressutil.Compress(marshaledBucket, &compressutil.CompressionConfig{
		Type: compressutil.CompressionTypeSnappy,
	})
	if err != nil {
		return errwrap.Wrapf("failed to compress packed bucket: {{err}}", err)
	}

	// Store the compressed value
	err = s.view.Put(context.Background(), &logical.StorageEntry{
		Key:   bucket.Key,
		Value: compressedBucket,
	})
	if err != nil {
		return errwrap.Wrapf("failed to persist packed storage entry: {{err}}", err)
	}

	return nil
}

// GetItem fetches the storage entry for a given key from its corresponding
// bucket.
func (s *StoragePacker) GetItem(itemID string) (*Item, error) {
	if itemID == "" {
		return nil, fmt.Errorf("empty item ID")
	}

	bucketKey := s.BucketKey(itemID)

	// Fetch the bucket entry
	bucket, err := s.GetBucket(bucketKey)
	if err != nil {
		return nil, errwrap.Wrapf("failed to read packed storage item: {{err}}", err)
	}
	if bucket == nil {
		return nil, nil
	}

	// Look for a matching storage entry in the bucket items
	for _, item := range bucket.Items {
		if item.ID == itemID {
			return item, nil
		}
	}

	return nil, nil
}

// PutItem stores the given item in its respective bucket
func (s *StoragePacker) PutItem(_ context.Context, item *Item) error {
	if item == nil {
		return fmt.Errorf("nil item")
	}

	if item.ID == "" {
		return fmt.Errorf("missing ID in item")
	}

	var err error
	bucketKey := s.BucketKey(item.ID)

	bucket := &Bucket{
		Key: bucketKey,
	}

	// In this case, we persist the storage entry regardless of the read
	// storageEntry below is nil or not. Hence, directly acquire write lock
	// even to read the entry.
	lock := locksutil.LockForKey(s.storageLocks, bucketKey)
	lock.Lock()
	defer lock.Unlock()

	// Check if there is an existing bucket for a given key
	storageEntry, err := s.view.Get(context.Background(), bucketKey)
	if err != nil {
		return errwrap.Wrapf("failed to read packed storage bucket entry: {{err}}", err)
	}

	if storageEntry == nil {
		// If the bucket entry does not exist, this will be the only item the
		// bucket that is going to be persisted.
		bucket.Items = []*Item{
			item,
		}
	} else {
		uncompressedData, notCompressed, err := compressutil.Decompress(storageEntry.Value)
		if err != nil {
			return errwrap.Wrapf("failed to decompress packed storage entry: {{err}}", err)
		}
		if notCompressed {
			uncompressedData = storageEntry.Value
		}

		err = proto.Unmarshal(uncompressedData, bucket)
		if err != nil {
			return errwrap.Wrapf("failed to decode packed storage entry: {{err}}", err)
		}

		err = bucket.upsert(item)
		if err != nil {
			return errwrap.Wrapf("failed to update entry in packed storage entry: {{err}}", err)
		}
	}

	return s.putBucket(bucket)
}

// NewStoragePacker creates a new storage packer for a given view
func NewStoragePacker(view logical.Storage, logger log.Logger, viewPrefix string) (*StoragePacker, error) {
	if view == nil {
		return nil, fmt.Errorf("nil view")
	}

	if viewPrefix == "" {
		viewPrefix = StoragePackerBucketsPrefix
	}

	if !strings.HasSuffix(viewPrefix, "/") {
		viewPrefix = viewPrefix + "/"
	}

	// Create a new packer object for the given view
	packer := &StoragePacker{
		view:         view,
		viewPrefix:   viewPrefix,
		logger:       logger,
		storageLocks: locksutil.CreateLocks(),
	}

	return packer, nil
}
<|endoftext|>